From 12418190b984f3f94da08588276ef3d6440bd454 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Mon, 10 Aug 2020 14:49:49 +1000 Subject: sb/track: Add a command to track build sets. - Process a build set for a range of hosts and output a dependency tree, the used build set and configuration files. - Output the configuration files that are no referenced Closes #4036 --- bare/config/devel/libusb-1.0.18-1.cfg | 3 + source-builder/config/autoconf-2-1.cfg | 6 +- source-builder/config/automake-1-1.cfg | 6 +- source-builder/config/libusb-1-1.cfg | 4 +- source-builder/sb-track | 27 ++ source-builder/sb/build.py | 15 +- source-builder/sb/cmd-track.py | 29 ++ source-builder/sb/config.py | 87 +++-- source-builder/sb/getsources.py | 583 ++--------------------------- source-builder/sb/simhost.py | 657 +++++++++++++++++++++++++++++++++ source-builder/sb/track.py | 254 +++++++++++++ 11 files changed, 1068 insertions(+), 603 deletions(-) create mode 100755 source-builder/sb-track create mode 100755 source-builder/sb/cmd-track.py create mode 100644 source-builder/sb/simhost.py create mode 100644 source-builder/sb/track.py diff --git a/bare/config/devel/libusb-1.0.18-1.cfg b/bare/config/devel/libusb-1.0.18-1.cfg index b47855b..494afbe 100644 --- a/bare/config/devel/libusb-1.0.18-1.cfg +++ b/bare/config/devel/libusb-1.0.18-1.cfg @@ -15,6 +15,9 @@ Name: libusb-%{libusb_version}-%{_host}-%{release} %description LibUSB for host %{_host}. +%hash sha512 libusb-%{libusb_version}.tar.bz2 \ + u8PXXLkfTmoCRUnCusOO6FrtqFKaRVP9NryOilwiPNieuVLtpx/MAHHWxsgYoGMdTMoIT+1p1Jhu7l3PmofWYg== + # # The Libuxb build instructions. We use 1.xx Release 1. # diff --git a/source-builder/config/autoconf-2-1.cfg b/source-builder/config/autoconf-2-1.cfg index 86f5d1e..2b9466c 100644 --- a/source-builder/config/autoconf-2-1.cfg +++ b/source-builder/config/autoconf-2-1.cfg @@ -6,9 +6,9 @@ %ifn %{defined _internal_autotools} %define _internal_autotools no - %ifn %{defined _internal_autotools_path} - %define _internal_autotools_path %{nil} - %endif +%endif +%ifn %{defined _internal_autotools_path} + %define _internal_autotools_path %{_prefix} %endif Name: autoconf-%{autoconf_version}-%{_host}-%{release} diff --git a/source-builder/config/automake-1-1.cfg b/source-builder/config/automake-1-1.cfg index 48beb51..83473de 100644 --- a/source-builder/config/automake-1-1.cfg +++ b/source-builder/config/automake-1-1.cfg @@ -6,9 +6,9 @@ %ifn %{defined _internal_autotools} %define _internal_autotools no - %ifn %{defined _internal_autotools_path} - %define _internal_autotools_path %{nil} - %endif +%endif +%ifn %{defined _internal_autotools_path} + %define _internal_autotools_path %{_prefix} %endif Name: automake-%{automake_version}-%{_host}-%{release} diff --git a/source-builder/config/libusb-1-1.cfg b/source-builder/config/libusb-1-1.cfg index 4b250a3..803f326 100644 --- a/source-builder/config/libusb-1-1.cfg +++ b/source-builder/config/libusb-1-1.cfg @@ -33,8 +33,10 @@ URL: http://libusb.org/ cd libusb-%{libusb_version} %if "%{_build}" != "%{_host}" - CFLAGS_FOR_BUILD="-g -O2 -Wall" \ + LIBUSB_CFLAGS_FOR_BUILD="-g -O2 -Wall" %endif + + CFLAGS_FOR_BUILD=${LIBUSB_CFLAGS_FOR_BUILD} \ CFLAGS="$SB_CFLAGS" \ ./configure \ --build=%{_build} --host=%{_host} \ diff --git a/source-builder/sb-track b/source-builder/sb-track new file mode 100755 index 0000000..a739978 --- /dev/null +++ b/source-builder/sb-track @@ -0,0 +1,27 @@ +#! /bin/sh +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2019 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +set -e +base=$(dirname $0) +PYTHON_CMD=${base}/sb/cmd-track.py +if test -f ${base}/sb/python-wrapper.sh; then + . ${base}/sb/python-wrapper.sh +fi +echo "error: python wrapper not found" diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index ceb179a..cfb7d8c 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -635,6 +635,10 @@ class build: return 0 return package.get_size('installed') + def includes(self): + if self.config: + return self.config.includes() + def get_configs(opts): def _scan(_path, ext): @@ -648,10 +652,17 @@ def get_configs(opts): return configs configs = { 'paths': [], 'files': [] } - for cp in opts.defaults.expand('%{_configdir}').split(':'): + paths = opts.defaults.expand('%{_configdir}').split(':') + root = path.host(os.path.commonprefix(paths)) + configs['root'] = root + configs['localpaths'] = [lp[len(root):] for lp in paths] + for cp in paths: hcp = path.host(path.abspath(cp)) configs['paths'] += [hcp] - configs['files'] += _scan(hcp, ['.cfg', '.bset']) + hpconfigs = sorted(set(_scan(hcp, ['.cfg', '.bset']))) + hcplocal = hcp[len(root):] + configs[hcplocal] = [path.join(hcplocal, c) for c in hpconfigs] + configs['files'] += hpconfigs configs['files'] = sorted(set(configs['files'])) return configs diff --git a/source-builder/sb/cmd-track.py b/source-builder/sb/cmd-track.py new file mode 100755 index 0000000..82615a5 --- /dev/null +++ b/source-builder/sb/cmd-track.py @@ -0,0 +1,29 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2010-2019 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import print_function + +import sys, os + +try: + import track + track.run() +except ImportError: + print("Incorrect Source Builder installation", file = sys.stderr) + sys.exit(1) diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py index afcac84..f8854a5 100644 --- a/source-builder/sb/config.py +++ b/source-builder/sb/config.py @@ -283,6 +283,7 @@ class file: return s def _reset(self, name): + self.parent = 'root' self.name = name self.load_depth = 0 self.configpath = [] @@ -430,7 +431,8 @@ class file: if len(shell_macro) > 3: e = execute.capture_execution() if options.host_windows: - shell_cmd = ''.join([c if c != '"' else '\\' + c for c in shell_macro[2:-1]]) + shell_cmd = \ + ''.join([c if c != '"' else '\\' + c for c in shell_macro[2:-1]]) cmd = '%s -c "%s"' % (self.macros.expand('%{__sh}'), shell_cmd) else: cmd = shell_macro[2:-1] @@ -458,7 +460,8 @@ class file: if braces > 0: braces -= 1 else: - shell_cmd = '%(' + self._shell(line[pos + 2:p], nesting + 1) + ')' + shell_cmd = '%(' + \ + self._shell(line[pos + 2:p], nesting + 1) + ')' line = line[:pos] + _exec(shell_cmd) + line[p + 1:] updating = True break @@ -472,9 +475,10 @@ class file: ('with_download' in self.macros and self.macros['with_download'] == '1'): return '0' ok = False - log.trace('pkgconfig: check: crossc=%d pkg_crossc=%d prefix=%s' % ( self._cross_compile(), - self.pkgconfig_crosscompile, - self.pkgconfig_prefix)) + log.trace('pkgconfig: check: crossc=%d pkg_crossc=%d prefix=%s' + % ( self._cross_compile(), + self.pkgconfig_crosscompile, + self.pkgconfig_prefix)) log.trace('pkgconfig: check: test=%s' % (test)) if type(test) == str: test = test.split() @@ -594,7 +598,8 @@ class file: elif m.startswith('%{expand'): colon = m.find(':') if colon < 8: - log.warning(self._name_line_msg('malformed expand macro, no colon found')) + log.warning(self._name_line_msg('malformed expand macro, ' \ + 'no colon found')) else: e = self._expand(m[colon + 1:-1].strip()) s = s.replace(m, self._label(e)) @@ -861,7 +866,8 @@ class file: dir, info, data = self._process_directive(r, dir, info, data) else: if in_dir != dir: - self._error('directives cannot change scope across if statements') + self._error('directives cannot change' \ + ' scope across if statements') return data if r[1] == '%else': @@ -904,22 +910,25 @@ class file: elif cls[0] == '&&': join_op = 'and' cls = cls[1:] - log.trace('config: %s: %3d: _if[%i]: joining: %s' % (self.name, self.lc, - self.if_depth, - join_op)) + log.trace('config: %s: %3d: _if[%i]: joining: %s' % \ + (self.name, self.lc, + self.if_depth, + join_op)) ori = 0 andi = 0 i = len(cls) if '||' in cls: ori = cls.index('||') - log.trace('config: %s: %3d: _if[%i}: OR found at %i' % (self.name, self.lc, - self.if_depth, - ori)) + log.trace('config: %s: %3d: _if[%i}: OR found at %i' % \ + (self.name, self.lc, + self.if_depth, + ori)) if '&&' in cls: andi = cls.index('&&') - log.trace('config: %s: %3d: _if[%i]: AND found at %i' % (self.name, self.lc, - self.if_depth, - andi)) + log.trace('config: %s: %3d: _if[%i]: AND found at %i' % \ + (self.name, self.lc, + self.if_depth, + andi)) if ori > 0 or andi > 0: if ori == 0: i = andi @@ -929,9 +938,10 @@ class file: i = andi else: i = andi - log.trace('config: %s: %3d: _if[%i]: next OP found at %i' % (self.name, self.lc, - self.if_depth, - i)) + log.trace('config: %s: %3d: _if[%i]: next OP found at %i' % \ + (self.name, self.lc, + self.if_depth, + i)) ls = cls[:i] if len(ls) == 0: self._error('invalid if expression: ' + reduce(add, sls, '')) @@ -1226,7 +1236,8 @@ class file: log.trace('config: %s: %3d: _parse: directive: %s' % \ (self.name, self.lc, ls[0].strip())) return ('directive', ls[0].strip(), ls[1:]) - log.warning(self._name_line_msg("unknown directive: '" + ls[0] + "'")) + log.warning(self._name_line_msg("unknown directive: '" + \ + ls[0] + "'")) return ('data', [lo]) else: return ('data', [lo]) @@ -1247,7 +1258,8 @@ class file: _package = results[2][0] else: if results[2][0].strip() != '-n': - log.warning(self._name_line_msg("unknown directive option: '%s'" % (' '.join(results[2])))) + log.warning(self._name_line_msg("unknown directive option: '%s'" % \ + (' '.join(results[2])))) _package = results[2][1].strip() self._set_package(_package) if directive and directive != results[1]: @@ -1257,7 +1269,8 @@ class file: return (directive, info, data) def _process_data(self, results, directive, info, data): - log.trace('config: %s: %3d: _process_data: result=#%r# directive=#%s# info=#%r# data=#%r#' % \ + log.trace('config: %s: %3d: _process_data: result=#%r# ' \ + 'directive=#%s# info=#%r# data=#%r#' % \ (self.name, self.lc, results, directive, info, data)) new_data = [] for l in results[1]: @@ -1284,10 +1297,12 @@ class file: if info is not None: self._info_append(info, info_data) else: - log.warning(self._name_line_msg("invalid format: '%s'" % (info_data[:-1]))) + log.warning(self._name_line_msg("invalid format: '%s'" % \ + (info_data[:-1]))) else: l = self._expand(l) - log.trace('config: %s: %3d: _data: %s %s' % (self.name, self.lc, l, new_data)) + log.trace('config: %s: %3d: _data: %s %s' % \ + (self.name, self.lc, l, new_data)) new_data.append(l) return (directive, info, data + new_data) @@ -1303,7 +1318,8 @@ class file: self.package = _package def _directive_extend(self, dir, data): - log.trace('config: %s: %3d: _directive_extend: %s: %r' % (self.name, self.lc, dir, data)) + log.trace('config: %s: %3d: _directive_extend: %s: %r' % \ + (self.name, self.lc, dir, data)) self._packages[self.package].directive_extend(dir, data) def _info_append(self, info, data): @@ -1328,7 +1344,6 @@ class file: return end if self.load_depth == 0: - self._reset(name) self._packages[self.package] = package(self.package, self.define('%{_arch}'), self) @@ -1336,6 +1351,7 @@ class file: self.load_depth += 1 save_name = self.name + save_parent = self.parent save_lc = self.lc # @@ -1382,7 +1398,9 @@ class file: raise error.general('error opening config file: %s' % (path.host(configname))) self.configpath += [configname] - self._includes += [configname] + + self._includes += [configname + ':' + self.parent] + self.parent = configname self.name = self._relative_path(configname) self.lc = 0 @@ -1413,13 +1431,12 @@ class file: except: config.close() raise - - config.close() - - self.name = save_name - self.lc = save_lc - - self.load_depth -= 1 + finally: + config.close() + self.name = save_name + self.parent = save_parent + self.lc = save_lc + self.load_depth -= 1 def defined(self, name): return name in self.macros @@ -1456,7 +1473,7 @@ class file: raise error.general('package "' + _package + '" not found') if name not in self._packages[_package].directives: raise error.general('directive "' + name + \ - '" not found in package "' + _package + '"') + '" not found in package "' + _package + '"') return self._packages[_package].directives[name] def abspath(self, rpath): diff --git a/source-builder/sb/getsources.py b/source-builder/sb/getsources.py index 523a3a5..7d76038 100644 --- a/source-builder/sb/getsources.py +++ b/source-builder/sb/getsources.py @@ -32,13 +32,9 @@ import sys try: import build - import check import error - import git import log - import macros - import path - import sources + import simhost import version except KeyboardInterrupt: print('abort: user terminated', file = sys.stderr) @@ -47,548 +43,6 @@ except: print('error: unknown application load error', file = sys.stderr) sys.exit(1) -# -# Define host profiles so it can simulated on another host. -# -host_profiles = { - 'darwin': { '_os': ('none', 'none', 'darwin'), - '_host': ('triplet', 'required', 'x86_64-apple-darwin18.5.0'), - '_host_vendor': ('none', 'none', 'apple'), - '_host_os': ('none', 'none', 'darwin'), - '_host_os_version': ('none', 'none', '18.5.0'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'freebsd': { '_os': ('none', 'none', 'freebsd'), - '_host': ('triplet', 'required', 'x86_64-freebsd12.0-RELEASE-p3'), - '_host_vendor': ('none', 'none', 'pc'), - '_host_os': ('none', 'none', 'freebsd'), - '_host_os_version': ('none', 'none', '12.0-RELEASE-p3'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'linux': { '_os': ('none', 'none', 'linux'), - '_host': ('triplet', 'required', 'x86_64-linux-gnu'), - '_host_vendor': ('none', 'none', 'gnu'), - '_host_os': ('none', 'none', 'linux'), - '_host_os_version': ('none', 'none', '4.18.0-16'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'netbsd': { '_os': ('none', 'none', 'netbsd'), - '_host': ('triplet', 'required', 'x86_64-netbsd8.0'), - '_host_vendor': ('none', 'none', 'pc'), - '_host_os': ('none', 'none', 'netbsd'), - '_host_os_version': ('none', 'none', '8.0'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'solaris': { '_os': ('none', 'none', 'solaris'), - '_host': ('triplet', 'required', 'x86_64-pc-solaris2'), - '_host_vendor': ('none', 'none', 'pc'), - '_host_os': ('none', 'none', 'solaris'), - '_host_os_version': ('none', 'none', '2'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'win32': { '_os': ('none', 'none', 'win32'), - '_windows_os': ('none', 'none', 'mingw32'), - '_host': ('triplet', 'required', 'x86_64-w64-mingw32'), - '_host_vendor': ('none', 'none', 'pc'), - '_host_os': ('none', 'none', 'win32'), - '_host_os_version': ('none', 'none', '10'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, - 'cygwin': { '_os': ('none', 'none', 'win32'), - '_windows_os': ('none', 'none', 'cygwin'), - '_host': ('triplet', 'required', 'x86_64-w64-cygwin'), - '_host_vendor': ('none', 'none', 'microsoft'), - '_host_os': ('none', 'none', 'win32'), - '_host_os_version': ('none', 'none', '10'), - '_host_cpu': ('none', 'none', 'x86_64'), - '_host_alias': ('none', 'none', '%{nil}'), - '_host_arch': ('none', 'none', 'x86_64'), - '_usr': ('dir', 'optional', '/usr/local'), - '_var': ('dir', 'optional', '/usr/local/var') }, -} - -class log_capture(object): - def __init__(self): - self.log = [] - log.capture = self.capture - - def __str__(self): - return os.linesep.join(self.log) - - def capture(self, text): - self.log += [l for l in text.replace(chr(13), '').splitlines()] - - def get(self): - return self.log - - def clear(self): - self.log = [] - -# -# A skinny options command line class to get the configs to load. -# -class options(object): - def __init__(self, argv, argopts, defaults): - command_path = path.dirname(path.abspath(argv[1])) - if len(command_path) == 0: - command_path = '.' - self.command_path = command_path - self.command_name = path.basename(argv[0]) - extras = ['--dry-run', - '--with-download', - '--quiet', - '--without-log', - '--without-error-report', - '--without-release-url'] - self.argv = argv - self.args = argv[1:] + extras - self.defaults = macros.macros(name = defaults, - sbdir = command_path) - self.load_overrides() - self.opts = { 'params' : extras } - self.sb_git() - self.rtems_bsp() - if argopts.download_dir is not None: - self.defaults['_sourcedir'] = ('dir', - 'optional', - path.abspath(argopts.download_dir)) - self.defaults['_patchdir'] = ('dir', - 'optional', - path.abspath(argopts.download_dir)) - - def load_overrides(self): - overrides = None - if os.name == 'nt': - try: - import windows - overrides = windows.load() - host_windows = True - host_posix = False - except: - raise error.general('failed to load Windows host support') - elif os.name == 'posix': - uname = os.uname() - try: - if uname[0].startswith('MINGW64_NT'): - import windows - overrides = windows.load() - host_windows = True - elif uname[0].startswith('CYGWIN_NT'): - import windows - overrides = windows.load() - elif uname[0] == 'Darwin': - import darwin - overrides = darwin.load() - elif uname[0] == 'FreeBSD': - import freebsd - overrides = freebsd.load() - elif uname[0] == 'NetBSD': - import netbsd - overrides = netbsd.load() - elif uname[0] == 'Linux': - import linux - overrides = linux.load() - elif uname[0] == 'SunOS': - import solaris - overrides = solaris.load() - except error.general as ge: - raise error.general('failed to load %s host support: %s' % (uname[0], ge)) - except: - raise error.general('failed to load %s host support' % (uname[0])) - else: - raise error.general('unsupported host type; please add') - if overrides is None: - raise error.general('no hosts defaults found; please add') - for k in overrides: - self.defaults[k] = overrides[k] - - def parse_args(self, arg, error = True, extra = True): - for a in range(0, len(self.args)): - if self.args[a].startswith(arg): - lhs = None - rhs = None - if '=' in self.args[a]: - eqs = self.args[a].split('=') - lhs = eqs[0] - if len(eqs) > 2: - rhs = '='.join(eqs[1:]) - else: - rhs = eqs[1] - elif extra: - lhs = self.args[a] - a += 1 - if a < len(self.args): - rhs = self.args[a] - return [lhs, rhs] - a += 1 - return None - - def rtems_bsp(self): - self.defaults['rtems_version'] = str(version.version()) - self.defaults['_target'] = 'arch-rtems' - self.defaults['rtems_host'] = 'rtems-arch' - self.defaults['with_rtems_bsp'] = 'rtems-bsp' - - def sb_git(self): - repo = git.repo(self.defaults.expand('%{_sbdir}'), self) - repo_mail = None - if repo.valid(): - repo_valid = '1' - repo_head = repo.head() - repo_clean = not repo.dirty() - repo_remotes = '%{nil}' - remotes = repo.remotes() - if 'origin' in remotes: - repo_remotes = '%s/origin' % (remotes['origin']['url']) - repo_id = repo_head - if not repo_clean: - repo_id += '-modified' - repo_mail = repo.email() - else: - repo_valid = '0' - repo_head = '%{nil}' - repo_clean = '%{nil}' - repo_remotes = '%{nil}' - repo_id = 'no-repo' - self.defaults['_sbgit_valid'] = repo_valid - self.defaults['_sbgit_head'] = repo_head - self.defaults['_sbgit_clean'] = str(repo_clean) - self.defaults['_sbgit_remotes'] = str(repo_remotes) - self.defaults['_sbgit_id'] = repo_id - if repo_mail is not None: - self.defaults['_sbgit_mail'] = repo_mail - - def get_arg(self, arg): - if self.optargs is None or arg not in self.optargs: - return None - return self.parse_args(arg) - - def with_arg(self, label, default = 'not-found'): - # the default if there is no option for without. - result = default - for pre in ['with', 'without']: - arg_str = '--%s-%s' % (pre, label) - arg_label = '%s_%s' % (pre, label) - arg = self.parse_args(arg_str, error = False, extra = False) - if arg is not None: - if arg[1] is None: - result = 'yes' - else: - result = arg[1] - break - return [arg_label, result] - - def dry_run(self): - return True - - def keep_going(self): - return False - - def quiet(self): - return True - - def no_clean(self): - return True - - def always_clean(self): - return False - - def no_install(self): - return True - - def download_disabled(self): - return False - - def disable_install(self): - return True - - def urls(self): - return None - - def info(self): - s = ' Command Line: %s%s' % (' '.join(self.argv), os.linesep) - s += ' Python: %s' % (sys.version.replace('\n', '')) - return s - -class buildset: - """Build a set builds a set of packages.""" - - def __init__(self, bset, _configs, opts, macros = None): - log.trace('_bset: %s: init' % (bset)) - self.configs = _configs - self.opts = opts - if macros is None: - self.macros = copy.copy(opts.defaults) - else: - self.macros = copy.copy(macros) - self.macros.define('_rsb_getting_source') - log.trace('_bset: %s: macro defaults' % (bset)) - log.trace(str(self.macros)) - self.bset = bset - _target = self.macros.expand('%{_target}') - if len(_target): - pkg_prefix = _target - else: - pkg_prefix = self.macros.expand('%{_host}') - self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset) - self.build_failure = None - - def build_package(self, _config, _build): - if not _build.disabled(): - _build.make() - - def parse(self, bset): - - # - # Ouch, this is a copy of the setbuilder.py code. - # - - def _clean(line): - line = line[0:-1] - b = line.find('#') - if b >= 0: - line = line[1:b] - return line.strip() - - bsetname = bset - - if not path.exists(bsetname): - for cp in self.macros.expand('%{_configdir}').split(':'): - configdir = path.abspath(cp) - bsetname = path.join(configdir, bset) - if path.exists(bsetname): - break - bsetname = None - if bsetname is None: - raise error.general('no build set file found: %s' % (bset)) - try: - log.trace('_bset: %s: open: %s' % (self.bset, bsetname)) - bset = open(path.host(bsetname), 'r') - except IOError as err: - raise error.general('error opening bset file: %s' % (bsetname)) - - configs = [] - - try: - lc = 0 - for l in bset: - lc += 1 - l = _clean(l) - if len(l) == 0: - continue - log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l)) - ls = l.split() - if ls[0][-1] == ':' and ls[0][:-1] == 'package': - self.bset_pkg = ls[1].strip() - self.macros['package'] = self.bset_pkg - elif ls[0][0] == '%': - def err(msg): - raise error.general('%s:%d: %s' % (self.bset, lc, msg)) - if ls[0] == '%define': - if len(ls) > 2: - self.macros.define(ls[1].strip(), - ' '.join([f.strip() for f in ls[2:]])) - else: - self.macros.define(ls[1].strip()) - elif ls[0] == '%undefine': - if len(ls) > 2: - raise error.general('%s:%d: %undefine requires just the name' % \ - (self.bset, lc)) - self.macros.undefine(ls[1].strip()) - elif ls[0] == '%include': - configs += self.parse(ls[1].strip()) - elif ls[0] in ['%patch', '%source']: - sources.process(ls[0][1:], ls[1:], self.macros, err) - elif ls[0] == '%hash': - sources.hash(ls[1:], self.macros, err) - else: - l = l.strip() - c = build.find_config(l, self.configs) - if c is None: - raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l)) - configs += [c] - except: - bset.close() - raise - - bset.close() - - return configs - - def load(self): - # - # If the build set file ends with .cfg the user has passed to the - # buildset builder a configuration so we just return it. - # - if self.bset.endswith('.cfg'): - configs = [self.bset] - else: - exbset = self.macros.expand(self.bset) - self.macros['_bset'] = exbset - self.macros['_bset_tmp'] = build.short_name(exbset) - root, ext = path.splitext(exbset) - if exbset.endswith('.bset'): - bset = exbset - else: - bset = '%s.bset' % (exbset) - configs = self.parse(bset) - return configs - - def set_host_details(self, host, opts, macros): - if host not in host_profiles: - raise error.general('invalid host: ' + host) - for m in host_profiles[host]: - opts.defaults[m] = host_profiles[host][m] - macros[m] = host_profiles[host][m] - macros_to_copy = [('%{_build}', '%{_host}'), - ('%{_build_alias}', '%{_host_alias}'), - ('%{_build_arch}', '%{_host_arch}'), - ('%{_build_cpu}', '%{_host_cpu}'), - ('%{_build_os}', '%{_host_os}'), - ('%{_build_vendor}', '%{_host_vendor}')] - for m in macros_to_copy: - opts.defaults[m[0]] = opts.defaults[m[1]] - macros[m[0]] = macros[m[1]] - # - # Look for a valid cc and cxx. - # - for cc in ['/usr/bin/cc', '/usr/bin/clang', '/usr/bin/gcc']: - if check.check_exe(cc, cc): - opts.defaults['__cc'] = cc - macros['__cc'] = cc - break - if not macros.defined('__cc'): - raise error.general('no valid cc found') - for cxx in ['/usr/bin/c++', '/usr/bin/clang++', '/usr/bin/g++']: - if check.check_exe(cxx, cxx): - opts.defaults['__cxx'] = cxx - macros['__cxx'] = cxx - if not macros.defined('__cxx'): - raise error.general('no valid c++ found') - - def build(self, host, nesting_count = 0): - - build_error = False - - nesting_count += 1 - - log.trace('_bset: %s for %s: make' % (self.bset, host)) - log.notice('Build Set: %s for %s' % (self.bset, host)) - - mail_subject = '%s on %s' % (self.bset, - self.macros.expand('%{_host}')) - - current_path = os.environ['PATH'] - - start = datetime.datetime.now() - - have_errors = False - - try: - configs = self.load() - - log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs))) - - sizes_valid = False - builds = [] - for s in range(0, len(configs)): - b = None - try: - # - # Each section of the build set gets a separate set of - # macros so we do not contaminate one configuration with - # another. - # - opts = copy.copy(self.opts) - macros = copy.copy(self.macros) - self.set_host_details(host, opts, macros) - if configs[s].endswith('.bset'): - log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75)) - bs = buildset(configs[s], self.configs, opts, macros) - bs.build(host, nesting_count) - del bs - elif configs[s].endswith('.cfg'): - log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75)) - try: - b = build.build(configs[s], - False, - opts, - macros) - except: - build_error = True - raise - self.build_package(configs[s], b) - builds += [b] - # - # Dump post build macros. - # - log.trace('_bset: macros post-build') - log.trace(str(macros)) - else: - raise error.general('invalid config type: %s' % (configs[s])) - except error.general as gerr: - have_errors = True - if b is not None: - if self.build_failure is None: - self.build_failure = b.name() - raise - # - # Clear out the builds ... - # - for b in builds: - del b - except error.general as gerr: - if not build_error: - log.stderr(str(gerr)) - raise - except KeyboardInterrupt: - raise - except: - self.build_failure = 'RSB general failure' - raise - finally: - end = datetime.datetime.now() - os.environ['PATH'] = current_path - build_time = str(end - start) - log.notice('Build Set: Time %s' % (build_time)) - -def list_bset_files(opts, configs): - ext = '.bset' - for p in configs['paths']: - print('Examining: %s' % (os.path.relpath(p))) - for c in configs['files']: - if c.endswith(ext): - print(' %s' % (c[:c.rfind('.')])) - -def load_log(logfile): - log.default = log.log(streams = [logfile]) - -def log_default(): - return 'rsb-log-getsource-%s.txt' % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) - -def load_options(argv, argopts, defaults = '%{_sbdir}/defaults.mc'): - opts = options(argv, argopts, defaults) - opts.defaults['rtems_version'] = str(argopts.rtems_version) - return opts - def run(args = sys.argv): ec = 0 get_sources_error = True @@ -617,22 +71,24 @@ def run(args = sys.argv): action = 'store_true') argsp.add_argument('--log', help = 'Log file.', type = str, - default = log_default()) + default = simhost.log_default('getsource')) argsp.add_argument('--trace', help = 'Enable trace logging for debugging.', action = 'store_true') argsp.add_argument('bsets', nargs='*', help = 'Build sets.') argopts = argsp.parse_args(args[2:]) - load_log(argopts.log) + simhost.load_log(argopts.log) log.notice('RTEMS Source Builder - Get Sources, %s' % (version.string())) log.tracing = argopts.trace - opts = load_options(args, argopts) + opts = simhost.load_options(args, argopts, extras = ['--with-download']) configs = build.get_configs(opts) - if argopts.list_bsets: - list_bset_files(opts, configs) + if argopts.list_hosts: + simhost.list_hosts() + elif argopts.list_bsets: + simhost.list_bset_files(opts, configs) else: if argopts.clean: if argopts.download_dir is None: @@ -640,14 +96,23 @@ def run(args = sys.argv): if path.exists(argopts.download_dir): log.notice('Cleaning source directory: %s' % (argopts.download_dir)) path.removeall(argopts.download_dir) + all_bsets = simhost.get_bset_files(configs) if len(argopts.bsets) == 0: - raise error.general('no build sets provided on the command line') - for bset in argopts.bsets: - get_sources_error = True - b = buildset(bset, configs, opts) - get_sources_error = False - for host in host_profiles: - b.build(host) + bsets = all_bsets + else: + bsets = argopts.bsets + for bset in bsets: + b = None + try: + for host in simhost.profiles: + get_sources_error = True + b = simhost.buildset(bset, configs, opts) + get_sources_error = False + b.build(host) + del b + except error.general as gerr: + log.stderr(str(gerr)) + log.stderr('Build FAILED') b = None except error.general as gerr: if get_sources_error: diff --git a/source-builder/sb/simhost.py b/source-builder/sb/simhost.py new file mode 100644 index 0000000..13ab157 --- /dev/null +++ b/source-builder/sb/simhost.py @@ -0,0 +1,657 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2010-2020 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# +# This code builds a package compiler tool suite given a tool set. A tool +# set lists the various tools. These are specific tool configurations. +# + +from __future__ import print_function + +import copy +import datetime +import os + +try: + import build + import check + import error + import git + import log + import macros + import path + import sources + import version +except KeyboardInterrupt: + print('abort: user terminated', file = sys.stderr) + sys.exit(1) +except: + print('error: unknown application load error', file = sys.stderr) + sys.exit(1) + +# +# Define host profiles so it can simulated on another host. +# +profiles = { + 'darwin': { '_os': ('none', 'none', 'darwin'), + '_host': ('triplet', 'required', 'x86_64-apple-darwin18.5.0'), + '_host_vendor': ('none', 'none', 'apple'), + '_host_os': ('none', 'none', 'darwin'), + '_host_os_version': ('none', 'none', '18.5.0'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'freebsd': { '_os': ('none', 'none', 'freebsd'), + '_host': ('triplet', 'required', 'x86_64-freebsd12.0-RELEASE-p3'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'freebsd'), + '_host_os_version': ('none', 'none', '12.0-RELEASE-p3'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'linux': { '_os': ('none', 'none', 'linux'), + '_host': ('triplet', 'required', 'x86_64-linux-gnu'), + '_host_vendor': ('none', 'none', 'gnu'), + '_host_os': ('none', 'none', 'linux'), + '_host_os_version': ('none', 'none', '4.18.0-16'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'netbsd': { '_os': ('none', 'none', 'netbsd'), + '_host': ('triplet', 'required', 'x86_64-netbsd8.0'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'netbsd'), + '_host_os_version': ('none', 'none', '8.0'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'solaris': { '_os': ('none', 'none', 'solaris'), + '_host': ('triplet', 'required', 'x86_64-pc-solaris2'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'solaris'), + '_host_os_version': ('none', 'none', '2'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'win32': { '_os': ('none', 'none', 'win32'), + '_windows_os': ('none', 'none', 'mingw32'), + '_host': ('triplet', 'required', 'x86_64-w64-mingw32'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'win32'), + '_host_os_version': ('none', 'none', '10'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'cygwin': { '_os': ('none', 'none', 'win32'), + '_windows_os': ('none', 'none', 'cygwin'), + '_host': ('triplet', 'required', 'x86_64-w64-cygwin'), + '_host_vendor': ('none', 'none', 'microsoft'), + '_host_os': ('none', 'none', 'win32'), + '_host_os_version': ('none', 'none', '10'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, +} + + +class log_capture(object): + def __init__(self): + self.log = [] + log.capture = self.capture + + def __str__(self): + return os.linesep.join(self.log) + + def capture(self, text): + self.log += [l for l in text.replace(chr(13), '').splitlines()] + + def get(self): + return self.log + + def clear(self): + self.log = [] + +def find_bset_config(bset_config, macros): + '''Find the build set or config file using the macro config defined path.''' + name = bset_config + if not path.exists(name): + for cp in macros.expand('%{_configdir}').split(':'): + configdir = path.abspath(cp) + name = path.join(configdir, bset_config) + if path.exists(name): + break + name = None + if name is None: + raise error.general('no build set file found: %s' % (bset_config)) + return name + +# +# A skinny options command line class to get the configs to load. +# +class options(object): + def __init__(self, argv, argopts, defaults, extras): + command_path = path.dirname(path.abspath(argv[1])) + if len(command_path) == 0: + command_path = '.' + self.command_path = command_path + self.command_name = path.basename(argv[0]) + extras += ['--dry-run', + '--quiet', + '--without-log', + '--without-error-report', + '--without-release-url'] + self.argv = argv + self.args = argv[1:] + extras + self.defaults = macros.macros(name = defaults, + sbdir = command_path) + self.load_overrides() + self.opts = { 'params' : extras } + self.sb_git() + self.rtems_bsp() + if 'download_dir' in argopts and argopts.download_dir is not None: + self.defaults['_sourcedir'] = ('dir', + 'optional', + path.abspath(argopts.download_dir)) + self.defaults['_patchdir'] = ('dir', + 'optional', + path.abspath(argopts.download_dir)) + + def load_overrides(self): + overrides = None + if os.name == 'nt': + try: + import windows + overrides = windows.load() + host_windows = True + host_posix = False + except: + raise error.general('failed to load Windows host support') + elif os.name == 'posix': + uname = os.uname() + try: + if uname[0].startswith('MINGW64_NT'): + import windows + overrides = windows.load() + host_windows = True + elif uname[0].startswith('CYGWIN_NT'): + import windows + overrides = windows.load() + elif uname[0] == 'Darwin': + import darwin + overrides = darwin.load() + elif uname[0] == 'FreeBSD': + import freebsd + overrides = freebsd.load() + elif uname[0] == 'NetBSD': + import netbsd + overrides = netbsd.load() + elif uname[0] == 'Linux': + import linux + overrides = linux.load() + elif uname[0] == 'SunOS': + import solaris + overrides = solaris.load() + except error.general as ge: + raise error.general('failed to load %s host support: %s' % (uname[0], ge)) + except: + raise error.general('failed to load %s host support' % (uname[0])) + else: + raise error.general('unsupported host type; please add') + if overrides is None: + raise error.general('no hosts defaults found; please add') + for k in overrides: + self.defaults[k] = overrides[k] + + def parse_args(self, arg, error = True, extra = True): + for a in range(0, len(self.args)): + if self.args[a].startswith(arg): + lhs = None + rhs = None + if '=' in self.args[a]: + eqs = self.args[a].split('=') + lhs = eqs[0] + if len(eqs) > 2: + rhs = '='.join(eqs[1:]) + else: + rhs = eqs[1] + elif extra: + lhs = self.args[a] + a += 1 + if a < len(self.args): + rhs = self.args[a] + return [lhs, rhs] + a += 1 + return None + + def rtems_bsp(self): + self.defaults['rtems_version'] = str(version.version()) + self.defaults['_target'] = 'arch-rtems' + self.defaults['rtems_host'] = 'rtems-arch' + self.defaults['with_rtems_bsp'] = 'rtems-bsp' + + def sb_git(self): + repo = git.repo(self.defaults.expand('%{_sbdir}'), self) + repo_mail = None + if repo.valid(): + repo_valid = '1' + repo_head = repo.head() + repo_clean = not repo.dirty() + repo_remotes = '%{nil}' + remotes = repo.remotes() + if 'origin' in remotes: + repo_remotes = '%s/origin' % (remotes['origin']['url']) + repo_id = repo_head + if not repo_clean: + repo_id += '-modified' + repo_mail = repo.email() + else: + repo_valid = '0' + repo_head = '%{nil}' + repo_clean = '%{nil}' + repo_remotes = '%{nil}' + repo_id = 'no-repo' + self.defaults['_sbgit_valid'] = repo_valid + self.defaults['_sbgit_head'] = repo_head + self.defaults['_sbgit_clean'] = str(repo_clean) + self.defaults['_sbgit_remotes'] = str(repo_remotes) + self.defaults['_sbgit_id'] = repo_id + if repo_mail is not None: + self.defaults['_sbgit_mail'] = repo_mail + + def get_arg(self, arg): + if self.optargs is None or arg not in self.optargs: + return None + return self.parse_args(arg) + + def with_arg(self, label, default = 'not-found'): + # the default if there is no option for without. + result = default + for pre in ['with', 'without']: + arg_str = '--%s-%s' % (pre, label) + arg_label = '%s_%s' % (pre, label) + arg = self.parse_args(arg_str, error = False, extra = False) + if arg is not None: + if arg[1] is None: + result = 'yes' + else: + result = arg[1] + break + return [arg_label, result] + + def dry_run(self): + return True + + def keep_going(self): + return False + + def quiet(self): + return True + + def no_clean(self): + return True + + def always_clean(self): + return False + + def no_install(self): + return True + + def download_disabled(self): + return False + + def disable_install(self): + return True + + def urls(self): + return None + + def info(self): + s = ' Command Line: %s%s' % (' '.join(self.argv), os.linesep) + s += ' Python: %s' % (sys.version.replace('\n', '')) + return s + +class buildset: + """Build a set builds a set of packages.""" + + def __init__(self, bset, _configs, opts, macros = None): + log.trace('_bset: %s: init' % (bset)) + self.parent = 'root' + self._includes = [] + self._errors = [] + self.configs = _configs + self.opts = opts + if macros is None: + self.macros = copy.copy(opts.defaults) + else: + self.macros = copy.copy(macros) + self.macros.define('_rsb_getting_source') + log.trace('_bset: %s: macro defaults' % (bset)) + log.trace(str(self.macros)) + self.bset = bset + _target = self.macros.expand('%{_target}') + if len(_target): + pkg_prefix = _target + else: + pkg_prefix = self.macros.expand('%{_host}') + self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset) + self.build_failure = None + + def _add_includes(self, includes, parent = None): + if parent is None: + parent = self.parent + if not isinstance(includes, list): + includes = [includes] + self._includes += [i + ':' + parent for i in includes] + + def _rebase_includes(self, includes, parent): + if not isinstance(includes, list): + includes = [includes] + rebased = [] + for i in includes: + if i.split(':', 2)[1] == 'root': + rebased += [i.split(':', 2)[0] + ':' + parent] + else: + rebased += [i] + return rebased + + def includes(self): + return sorted(list(set(self._includes))) + + def errors(self): + return sorted(list(set(self._errors))) + + def build_package(self, _config, _build): + if not _build.disabled(): + _build.make() + + def parse(self, bset): + + # + # Ouch, this is a copy of the setbuilder.py code. + # + + def _clean(line): + line = line[0:-1] + b = line.find('#') + if b >= 0: + line = line[1:b] + return line.strip() + + bsetname = find_bset_config(bset, self.macros) + + try: + log.trace('_bset: %s: open: %s' % (self.bset, bsetname)) + bsetf = open(path.host(bsetname), 'r') + except IOError as err: + raise error.general('error opening bset file: %s' % (bsetname)) + + self._add_includes(bsetname) + parent = self.parent + self.parent = bsetname + + configs = [] + + try: + lc = 0 + for l in bsetf: + lc += 1 + l = _clean(l) + if len(l) == 0: + continue + log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l)) + ls = l.split() + if ls[0][-1] == ':' and ls[0][:-1] == 'package': + self.bset_pkg = ls[1].strip() + self.macros['package'] = self.bset_pkg + elif ls[0][0] == '%': + def err(msg): + raise error.general('%s:%d: %s' % (self.bset, lc, msg)) + if ls[0] == '%define': + if len(ls) > 2: + self.macros.define(ls[1].strip(), + ' '.join([f.strip() for f in ls[2:]])) + else: + self.macros.define(ls[1].strip()) + elif ls[0] == '%undefine': + if len(ls) > 2: + raise error.general('%s:%d: %undefine requires just the name' \ + % (self.bset, lc)) + self.macros.undefine(ls[1].strip()) + elif ls[0] == '%include': + configs += self.parse(ls[1].strip()) + elif ls[0] in ['%patch', '%source']: + sources.process(ls[0][1:], ls[1:], self.macros, err) + elif ls[0] == '%hash': + sources.hash(ls[1:], self.macros, err) + else: + l = l.strip() + c = build.find_config(l, self.configs) + if c is None: + raise error.general('%s:%d: cannot find file: %s' + % (self.bset, lc, l)) + configs += [c + ':' + self.parent] + finally: + bsetf.close() + self.parent = parent + + return configs + + def load(self): + # + # If the build set file ends with .cfg the user has passed to the + # buildset builder a configuration so we just return it. + # + if self.bset.endswith('.cfg'): + self._add_includes(self.bset) + configs = [self.bset] + else: + exbset = self.macros.expand(self.bset) + self.macros['_bset'] = exbset + self.macros['_bset_tmp'] = build.short_name(exbset) + root, ext = path.splitext(exbset) + if exbset.endswith('.bset'): + bset = exbset + else: + bset = '%s.bset' % (exbset) + configs = self.parse(bset) + return configs + + def set_host_details(self, host, opts, macros): + if host not in profiles: + raise error.general('invalid host: ' + host) + for m in profiles[host]: + opts.defaults[m] = profiles[host][m] + macros[m] = profiles[host][m] + macros_to_copy = [('%{_build}', '%{_host}'), + ('%{_build_alias}', '%{_host_alias}'), + ('%{_build_arch}', '%{_host_arch}'), + ('%{_build_cpu}', '%{_host_cpu}'), + ('%{_build_os}', '%{_host_os}'), + ('%{_build_vendor}', '%{_host_vendor}')] + for m in macros_to_copy: + opts.defaults[m[0]] = opts.defaults[m[1]] + macros[m[0]] = macros[m[1]] + # + # Look for a valid cc and cxx. + # + for cc in ['/usr/bin/cc', '/usr/bin/clang', '/usr/bin/gcc']: + if check.check_exe(cc, cc): + opts.defaults['__cc'] = cc + macros['__cc'] = cc + break + if not macros.defined('__cc'): + raise error.general('no valid cc found') + for cxx in ['/usr/bin/c++', '/usr/bin/clang++', '/usr/bin/g++']: + if check.check_exe(cxx, cxx): + opts.defaults['__cxx'] = cxx + macros['__cxx'] = cxx + if not macros.defined('__cxx'): + raise error.general('no valid c++ found') + + def build(self, host, nesting_count = 0): + + build_error = False + + nesting_count += 1 + + log.trace('_bset: %s for %s: make' % (self.bset, host)) + log.notice('Build Set: %s for %s' % (self.bset, host)) + + mail_subject = '%s on %s' % (self.bset, + self.macros.expand('%{_host}')) + + current_path = os.environ['PATH'] + + start = datetime.datetime.now() + + have_errors = False + + try: + configs = self.load() + + log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs))) + + sizes_valid = False + builds = [] + for s in range(0, len(configs)): + bs = None + b = None + try: + # + # Each section of the build set gets a separate set of + # macros so we do not contaminate one configuration with + # another. + # + opts = copy.copy(self.opts) + macros = copy.copy(self.macros) + self.set_host_details(host, opts, macros) + config, parent = configs[s].split(':', 2) + if config.endswith('.bset'): + log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75)) + bs = buildset(config, self.configs, opts, macros) + bs.build(host, nesting_count) + self._includes += \ + self._rebase_includes(bs.includes(), parent) + del bs + elif config.endswith('.cfg'): + log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75)) + try: + b = build.build(config, + False, + opts, + macros) + self._includes += \ + self._rebase_includes(b.includes(), parent) + except: + build_error = True + raise + self.build_package(config, b) + builds += [b] + # + # Dump post build macros. + # + log.trace('_bset: macros post-build') + log.trace(str(macros)) + else: + raise error.general('invalid config type: %s' % (config)) + except error.general as gerr: + have_errors = True + if b is not None: + if self.build_failure is None: + self.build_failure = b.name() + self._includes += b.includes() + self._errors += [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes + raise + # + # Clear out the builds ... + # + for b in builds: + del b + except error.general as gerr: + if not build_error: + log.stderr(str(gerr)) + raise + except KeyboardInterrupt: + raise + except: + self.build_failure = 'RSB general failure' + raise + finally: + end = datetime.datetime.now() + os.environ['PATH'] = current_path + build_time = str(end - start) + log.notice('Build Set: Time %s' % (build_time)) + +def list_hosts(): + hosts = sorted(profiles.keys()) + max_os_len = max(len(h) for h in hosts) + max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts) + for h in hosts: + print('%*s: %-*s %s' % (max_os_len, h, max_host_len, + profiles[h]['_host'][2], + profiles[h]['_host'][2])) + +def get_files(configs, ext, localpath): + files = [] + if localpath: + for cp in configs['localpaths']: + files += [c for c in configs[cp] if c.endswith(ext)] + else: + files = [c for c in configs['files'] if c.endswith(ext)] + return files + +def get_config_files(configs, localpath = False): + return get_files(configs, '.cfg', localpath) + +def get_bset_files(configs, localpath = False): + return get_files(configs, '.bset', localpath) + +def get_root(configs): + return configs['root'] + +def list_bset_files(opts, configs): + for p in configs['paths']: + print('Examining: %s' % (os.path.relpath(p))) + for b in get_bset_files(configs): + print(' %s' % (b[:b.rfind('.')])) + +def load_log(logfile): + log.default = log.log(streams = [logfile]) + +def log_default(name): + return 'rsb-log-%s-%s.txt' % (name, datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + +def load_options(argv, argopts, defaults = '%{_sbdir}/defaults.mc', extras = []): + opts = options(argv, argopts, defaults, extras) + opts.defaults['rtems_version'] = str(argopts.rtems_version) + return opts diff --git a/source-builder/sb/track.py b/source-builder/sb/track.py new file mode 100644 index 0000000..ef4ce8b --- /dev/null +++ b/source-builder/sb/track.py @@ -0,0 +1,254 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2020 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# +# This code builds a package compiler tool suite given a tool set. A tool +# set lists the various tools. These are specific tool configurations. +# + +from __future__ import print_function + +import argparse +import copy +import datetime +import os +import sys + +try: + import build + import error + import git + import log + import simhost + import version +except KeyboardInterrupt: + print('abort: user terminated', file = sys.stderr) + sys.exit(1) +except: + print('error: unknown application load error', file = sys.stderr) + sys.exit(1) + +def unique(l): + return sorted(list(set(l))) + +def filter_deps(deps, ext): + rdeps = [] + for d in deps: + ds = d.split(':', 2) + if ds[0].endswith(ext): + rdeps += [ds[0] + ':' + ds[1]] + return sorted(rdeps) + +def normalise_paths(includes, root): + normalised = [] + for inc in unique(includes): + config, parent = inc.split(':', 2) + if config.startswith(root): + config = config[len(root):] + if parent.startswith(root): + parent = parent[len(root):] + normalised += [config + ':' + parent] + return normalised + +def process_dependencies(includes): + deps = {} + incs = [i.split(':', 2) for i in includes] + for config, parent in incs: + if parent not in deps: + deps[parent] = [] + for inc in incs: + if inc[1] == parent: + deps[parent] += [inc[0]] + for d in deps: + deps[d] = unique(deps[d]) + return deps + +def includes_str(includes): + o = [] + deps = [i.split(':', 2) for i in includes] + ll = max([len(d[1]) for d in deps]) + for d in deps: + o += ['%*s %s' % (ll, d[1], d[0])] + return o + +def deps_str(deps): + def print_node(deps, node, level = 0, prefix = '', indent = ''): + o = [] + if node != 'root': + level += 1 + if level == 1: + o += [''] + o += [prefix + '+-- ' + node] + if node in deps: + prefix += indent + for c, child in enumerate(deps[node], start = 1): + if c < len(deps[node]) and level > 1: + indent = '| ' + else: + indent = ' ' + o += print_node(deps, child, level, prefix, indent) + return o + return print_node(deps, 'root') + +def run(args = sys.argv): + ec = 0 + output = [] + try: + # + # The RSB options support cannot be used because it loads the defaults + # for the host which we cannot do here. + # + description = 'RTEMS Track Dependencies a build set has for all hosts.' + + argsp = argparse.ArgumentParser(prog = 'sb-dep-check', + description = description) + argsp.add_argument('--rtems-version', help = 'Set the RTEMS version.', + type = str, + default = version.version()) + argsp.add_argument('--list-hosts', help = 'List the hosts.', + action = 'store_true') + argsp.add_argument('--list-bsets', help = 'List the hosts.', + action = 'store_true') + argsp.add_argument('--output', help = 'Output file.', + type = str, + default = None) + argsp.add_argument('--log', help = 'Log file.', + type = str, + default = simhost.log_default('trackdeps')) + argsp.add_argument('--trace', help = 'Enable trace logging for debugging.', + action = 'store_true') + argsp.add_argument('--not-referenced', + help = 'Write out the list of not config files not referenced.', + action = 'store_true') + argsp.add_argument('bsets', nargs='*', help = 'Build sets.') + + argopts = argsp.parse_args(args[2:]) + + simhost.load_log(argopts.log) + log.notice('RTEMS Source Builder - Track Dependencies, %s' % (version.string())) + log.tracing = argopts.trace + + opts = simhost.load_options(args, argopts, extras = ['---keep-going']) + configs = build.get_configs(opts) + + if argopts.list_hosts: + simhost.list_hosts() + elif argopts.list_bsets: + simhost.list_bset_files(opts, configs) + else: + all_bsets = simhost.get_bset_files(configs) + if len(argopts.bsets) == 0: + bsets = all_bsets + else: + bsets = argopts.bsets + includes = [] + errors = [] + for bset in bsets: + b = None + try: + for host in simhost.profiles: + b = simhost.buildset(bset, configs, opts) + b.build(host) + includes += b.includes() + errors += b.errors() + del b + except error.general as gerr: + log.stderr(str(gerr)) + log.stderr('Build FAILED') + if b: + includes += b.includes() + errors += b.errors() + b = None + root = simhost.get_root(configs) + all_configs = simhost.get_config_files(configs, True) + includes = normalise_paths(includes, root) + bsets = filter_deps(includes, '.bset') + configs = filter_deps(includes, '.cfg') + deps_tree = deps_str(process_dependencies(bsets + configs)) + bsets = unique([b.split(':', 2)[0] for b in bsets]) + configs = unique([i.split(':', 2)[0] for i in configs]) + not_used_configs = [c for c in all_configs if c not in configs] + if len(errors) > 0: + errors = [e.split(':', 2)[0] for e in normalise_paths(errors, root)] + errs = [] + for e in errors: + if e not in bsets + configs: + errs += [e] + errors = errs + if argopts.not_referenced: + output = not_used_configs + else: + output = ['RSB Dependency Tracker', + '', + 'Total buildsets: %d' % (len(all_bsets)), + 'Total configs: %d' % (len(all_configs)), + ''] + if len(errors) > 0: + output += ['Errored File Set (%d):' % (len(errors)), + ''] + \ + errors + \ + [''] + if len(configs) > 0: + output += ['Include Tree(s):', + ''] + \ + deps_tree + \ + [''] + if len(bsets) > 0: + output += ['Buildsets (%d):' % (len(bsets)), + ''] + \ + bsets + \ + [''] + if len(configs) > 0: + output += ['Configurations (%d):' % (len(configs)), + ''] + \ + configs + \ + [''] + if len(not_used_configs) > 0: + output += ['Not referenced (%d): ' % (len(not_used_configs)), + ''] + \ + not_used_configs + output = os.linesep.join(output) + if argopts.output: + o = open(argopts.output, "w") + o.write(output) + o.close + else: + print() + print(output) + except error.general as gerr: + log.stderr(str(gerr)) + log.stderr('Build FAILED') + ec = 1 + except error.internal as ierr: + log.stderr(str(ierr)) + log.stderr('Internal Build FAILED') + ec = 1 + except error.exit as eerr: + pass + except KeyboardInterrupt: + log.notice('abort: user terminated') + ec = 1 + except: + raise + log.notice('abort: unknown error') + ec = 1 + sys.exit(ec) + +if __name__ == "__main__": + run() -- cgit v1.2.3