diff options
Diffstat (limited to 'source-builder/sb/simhost.py')
| -rw-r--r-- | source-builder/sb/simhost.py | 720 |
1 files changed, 720 insertions, 0 deletions
diff --git a/source-builder/sb/simhost.py b/source-builder/sb/simhost.py new file mode 100644 index 0000000..9196151 --- /dev/null +++ b/source-builder/sb/simhost.py @@ -0,0 +1,720 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2010-2020 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# +# This code builds a package compiler tool suite given a tool set. A tool +# set lists the various tools. These are specific tool configurations. +# + +from __future__ import print_function + +import copy +import datetime +import os + +try: + import build + import check + import error + import git + import log + import macros + import path + import shell + import sources + import version +except KeyboardInterrupt: + print('abort: user terminated', file = sys.stderr) + sys.exit(1) +except: + raise + +# +# Define host profiles so it can simulated on another host. +# +profiles = { + 'darwin': { '_os': ('none', 'none', 'darwin'), + '_host': ('triplet', 'required', 'x86_64-apple-darwin18.5.0'), + '_host_vendor': ('none', 'none', 'apple'), + '_host_os': ('none', 'none', 'darwin'), + '_host_os_version': ('none', 'none', '18.5.0'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'freebsd': { '_os': ('none', 'none', 'freebsd'), + '_host': ('triplet', 'required', 'x86_64-freebsd12.0-RELEASE-p3'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'freebsd'), + '_host_os_version': ('none', 'none', '12.0-RELEASE-p3'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'linux': { '_os': ('none', 'none', 'linux'), + '_host': ('triplet', 'required', 'x86_64-linux-gnu'), + '_host_vendor': ('none', 'none', 'gnu'), + '_host_os': ('none', 'none', 'linux'), + '_host_os_version': ('none', 'none', '4.18.0-16'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'netbsd': { '_os': ('none', 'none', 'netbsd'), + '_host': ('triplet', 'required', 'x86_64-netbsd8.0'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'netbsd'), + '_host_os_version': ('none', 'none', '8.0'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'solaris': { '_os': ('none', 'none', 'solaris'), + '_host': ('triplet', 'required', 'x86_64-pc-solaris2'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'solaris'), + '_host_os_version': ('none', 'none', '2'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'win32': { '_os': ('none', 'none', 'win32'), + '_windows_os': ('none', 'none', 'mingw32'), + '_host': ('triplet', 'required', 'x86_64-w64-mingw32'), + '_host_vendor': ('none', 'none', 'pc'), + '_host_os': ('none', 'none', 'win32'), + '_host_os_version': ('none', 'none', '10'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, + 'cygwin': { '_os': ('none', 'none', 'win32'), + '_windows_os': ('none', 'none', 'cygwin'), + '_host': ('triplet', 'required', 'x86_64-w64-cygwin'), + '_host_vendor': ('none', 'none', 'microsoft'), + '_host_os': ('none', 'none', 'win32'), + '_host_os_version': ('none', 'none', '10'), + '_host_cpu': ('none', 'none', 'x86_64'), + '_host_alias': ('none', 'none', '%{nil}'), + '_host_arch': ('none', 'none', 'x86_64'), + '_usr': ('dir', 'optional', '/usr/local'), + '_var': ('dir', 'optional', '/usr/local/var') }, +} + +class log_capture(object): + def __init__(self): + self.log = [] + log.capture = self.capture + + def __str__(self): + return os.linesep.join(self.log) + + def capture(self, text): + self.log += [l for l in text.replace(chr(13), '').splitlines()] + + def get(self): + return self.log + + def clear(self): + self.log = [] + +def find_bset_config(bset_config, macros): + '''Find the build set or config file using the macro config defined path.''' + name = bset_config + if not path.exists(name): + for cp in macros.expand('%{_configdir}').split(':'): + configdir = path.abspath(cp) + name = path.join(configdir, bset_config) + if path.exists(name): + break + name = None + if name is None: + raise error.general('no build set file found: %s' % (bset_config)) + return name + +def macro_expand(macros, _str): + cstr = None + while cstr != _str: + cstr = _str + _str = macros.expand(_str) + _str = shell.expand(macros, _str) + return _str + +def strip_common_prefix(files): + commonprefix = os.path.commonprefix(files) + return sorted(list(set([f[len(commonprefix):] for f in files]))) + +# +# A skinny options command line class to get the configs to load. +# +class options(object): + def __init__(self, argv, argopts, defaults, extras): + command_path = path.dirname(path.abspath(argv[0])) + if len(command_path) == 0: + command_path = '.' + self.command_path = command_path + self.command_name = path.basename(argv[0]) + extras += ['--dry-run', + '--quiet', + '--without-log', + '--without-error-report', + '--without-release-url'] + self.argv = argv + self.args = argv[1:] + extras + self.defaults = macros.macros(name = defaults, + sbdir = command_path) + self.load_overrides() + self.opts = { 'params' : extras } + self.sb_git() + self.rtems_bsp() + if 'download_dir' in argopts and argopts.download_dir is not None: + self.defaults['_sourcedir'] = ('dir', + 'optional', + path.abspath(argopts.download_dir)) + self.defaults['_patchdir'] = ('dir', + 'optional', + path.abspath(argopts.download_dir)) + + def load_overrides(self): + overrides = None + if os.name == 'nt': + try: + from . import windows + overrides = windows.load() + host_windows = True + host_posix = False + except: + raise error.general('failed to load Windows host support') + elif os.name == 'posix': + uname = os.uname() + try: + if uname[0].startswith('MINGW64_NT'): + import windows + overrides = windows.load() + host_windows = True + elif uname[0].startswith('CYGWIN_NT'): + import windows + overrides = windows.load() + elif uname[0] == 'Darwin': + import darwin + overrides = darwin.load() + elif uname[0] == 'FreeBSD': + import freebsd + overrides = freebsd.load() + elif uname[0] == 'NetBSD': + import netbsd + overrides = netbsd.load() + elif uname[0] == 'Linux': + import linux + overrides = linux.load() + elif uname[0] == 'SunOS': + import solaris + overrides = solaris.load() + except error.general as ge: + raise error.general('failed to load %s host support: %s' % (uname[0], ge)) + except: + raise error.general('failed to load %s host support' % (uname[0])) + else: + raise error.general('unsupported host type; please add') + if overrides is None: + raise error.general('no hosts defaults found; please add') + for k in overrides: + self.defaults[k] = overrides[k] + + def parse_args(self, arg, error = True, extra = True): + for a in range(0, len(self.args)): + if self.args[a].startswith(arg): + lhs = None + rhs = None + if '=' in self.args[a]: + eqs = self.args[a].split('=') + lhs = eqs[0] + if len(eqs) > 2: + rhs = '='.join(eqs[1:]) + else: + rhs = eqs[1] + elif extra: + lhs = self.args[a] + a += 1 + if a < len(self.args): + rhs = self.args[a] + return [lhs, rhs] + a += 1 + return None + + def rtems_bsp(self, arch='arch'): + self.defaults['rtems_version'] = str(version.version()) + self.defaults['_target'] = arch + '-rtems' + self.defaults['rtems_host'] = 'rtems-' + arch + self.defaults['with_rtems_bsp'] = 'rtems-bsp' + + def sb_git(self): + repo = git.repo(self.defaults.expand('%{_sbdir}'), self) + repo_mail = None + if repo.valid(): + repo_valid = '1' + repo_head = repo.head() + repo_clean = not repo.dirty() + repo_remotes = '%{nil}' + remotes = repo.remotes() + if 'origin' in remotes: + repo_remotes = '%s/origin' % (remotes['origin']['url']) + repo_id = repo_head + if not repo_clean: + repo_id += '-modified' + repo_mail = repo.email() + else: + repo_valid = '0' + repo_head = '%{nil}' + repo_clean = '%{nil}' + repo_remotes = '%{nil}' + repo_id = 'no-repo' + self.defaults['_sbgit_valid'] = repo_valid + self.defaults['_sbgit_head'] = repo_head + self.defaults['_sbgit_clean'] = str(repo_clean) + self.defaults['_sbgit_remotes'] = str(repo_remotes) + self.defaults['_sbgit_id'] = repo_id + if repo_mail is not None: + self.defaults['_sbgit_mail'] = repo_mail + + def get_arg(self, arg): + if self.optargs is None or arg not in self.optargs: + return None + return self.parse_args(arg) + + def with_arg(self, label, default = 'not-found'): + # the default if there is no option for without. + result = default + for pre in ['with', 'without']: + arg_str = '--%s-%s' % (pre, label) + arg_label = '%s_%s' % (pre, label) + arg = self.parse_args(arg_str, error = False, extra = False) + if arg is not None: + if arg[1] is None: + result = 'yes' + else: + result = arg[1] + break + return [arg_label, result] + + def dry_run(self): + return True + + def keep_going(self): + return False + + def quiet(self): + return True + + def no_clean(self): + return True + + def always_clean(self): + return False + + def no_install(self): + return True + + def download_disabled(self): + return False + + def disable_install(self): + return True + + def urls(self): + return None + + def info(self): + s = ' Command Line: %s%s' % (' '.join(self.argv), os.linesep) + s += ' Python: %s' % (sys.version.replace('\n', '')) + return s + +class buildset: + """Build a set builds a set of packages.""" + + def __init__(self, bset, _configs, opts, macros = None): + log.trace('_bset: %s: init' % (bset)) + self.parent = 'root' + self._includes = [] + self._errors = [] + self.configs = _configs + self.opts = opts + if macros is None: + self.macros = copy.copy(opts.defaults) + else: + self.macros = copy.copy(macros) + self.macros.define('_rsb_getting_source') + log.trace('_bset: %s: macro defaults' % (bset)) + log.trace(str(self.macros)) + self.bset = bset + _target = self.macros.expand('%{_target}') + if len(_target): + pkg_prefix = _target + else: + pkg_prefix = self.macros.expand('%{_host}') + self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset) + self.build_failure = None + + def _add_includes(self, includes, parent = None): + if parent is None: + parent = self.parent + if not isinstance(includes, list): + includes = [includes] + self._includes += [i + ':' + parent for i in includes] + + def _rebase_includes(self, includes, parent): + if not isinstance(includes, list): + includes = [includes] + rebased = [] + for i in includes: + if i.split(':', 2)[1] == 'root': + rebased += [i.split(':', 2)[0] + ':' + parent] + else: + rebased += [i] + return rebased + + def root(self): + for i in self._includes: + si = i.split(':') + if len(si) == 2: + if si[1] == 'root': + return si[0] + return None + + def includes(self): + return [i for i in self._includes if not i.endswith(':root')] + + def deps(self): + return strip_common_prefix([i.split(':')[0] for i in self.includes()]) + + def errors(self): + return sorted(list(set(self._errors))) + + def build_package(self, _config, _build): + if not _build.disabled(): + _build.make() + + def parse(self, bset, expand=True): + + # + # Ouch, this is a copy of the setbuilder.py code. + # + + def _clean(line): + line = line[0:-1] + b = line.find('#') + if b >= 0: + line = line[1:b] + return line.strip() + + bsetname = find_bset_config(bset, self.macros) + + try: + log.trace('_bset: %s: open: %s %s' % (self.bset, bsetname, expand)) + bsetf = open(path.host(bsetname), 'r') + except IOError as err: + raise error.general('error opening bset file: %s' % (bsetname)) + + self._add_includes(bsetname) + parent = self.parent + self.parent = bsetname + + configs = [] + + try: + lc = 0 + for l in bsetf: + lc += 1 + l = _clean(l) + if len(l) == 0: + continue + log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l)) + ls = l.split() + if ls[0][-1] == ':' and ls[0][:-1] == 'package': + self.bset_pkg = ls[1].strip() + self.macros['package'] = self.bset_pkg + elif ls[0][0] == '%' and (len(ls[0]) > 1 and ls[0][1] != '{'): + def err(msg): + raise error.general('%s:%d: %s' % (self.bset, lc, msg)) + if ls[0] == '%define' or ls[0] == '%defineifnot' : + name = ls[1].strip() + value = None + if len(ls) > 2: + value = ' '.join([f.strip() for f in ls[2:]]) + if ls[0] == '%defineifnot': + if self.macros.defined(name): + name = None + if name is not None: + if value is not None: + self.macros.define(name, value) + else: + self.macros.define(name) + elif ls[0] == '%undefine': + if len(ls) > 2: + raise error.general('%s:%d: %undefine requires ' \ + 'just the name' % (self.bset, lc)) + self.macros.undefine(ls[1].strip()) + elif ls[0] == '%include': + configs += self.parse(ls[1].strip()) + elif ls[0] in ['%patch', '%source']: + sources.process(ls[0][1:], ls[1:], self.macros, err) + elif ls[0] == '%hash': + sources.hash(ls[1:], self.macros, err) + else: + try: + l = macro_expand(self.macros, l.strip()) + except: + if expand: + raise + l = None + if l is not None: + c = build.find_config(l, self.configs) + if c is None: + raise error.general('%s:%d: cannot find file: %s' + % (self.bset, lc, l)) + configs += [c + ':' + self.parent] + finally: + bsetf.close() + self.parent = parent + + return configs + + def load(self): + # + # If the build set file ends with .cfg the user has passed to the + # buildset builder a configuration so we just return it. + # + if self.bset.endswith('.cfg'): + self._add_includes(self.bset) + configs = [self.bset] + else: + exbset = self.macros.expand(self.bset) + self.macros['_bset'] = exbset + self.macros['_bset_tmp'] = build.short_name(exbset) + root, ext = path.splitext(exbset) + if exbset.endswith('.bset'): + bset = exbset + else: + bset = '%s.bset' % (exbset) + configs = self.parse(bset) + return configs + + def set_host_details(self, host, opts, macros): + if host not in profiles: + raise error.general('invalid host: ' + host) + for m in profiles[host]: + opts.defaults[m] = profiles[host][m] + macros[m] = profiles[host][m] + macros_to_copy = [('%{_build}', '%{_host}'), + ('%{_build_alias}', '%{_host_alias}'), + ('%{_build_arch}', '%{_host_arch}'), + ('%{_build_cpu}', '%{_host_cpu}'), + ('%{_build_os}', '%{_host_os}'), + ('%{_build_vendor}', '%{_host_vendor}')] + for m in macros_to_copy: + opts.defaults[m[0]] = opts.defaults[m[1]] + macros[m[0]] = macros[m[1]] + # + # Look for a valid cc and cxx. + # + for cc in ['/usr/bin/cc', '/usr/bin/clang', '/usr/bin/gcc']: + if check.check_exe(cc, cc): + opts.defaults['__cc'] = cc + macros['__cc'] = cc + break + if not macros.defined('__cc'): + raise error.general('no valid cc found') + for cxx in ['/usr/bin/c++', '/usr/bin/clang++', '/usr/bin/g++']: + if check.check_exe(cxx, cxx): + opts.defaults['__cxx'] = cxx + macros['__cxx'] = cxx + if not macros.defined('__cxx'): + raise error.general('no valid c++ found') + + def build(self, host, nesting_count = 0): + + build_error = False + + nesting_count += 1 + + log.trace('_bset: %2d: %s for %s: make' % (nesting_count, self.bset, host)) + log.notice('Build Set: %s for %s' % (self.bset, host)) + + mail_subject = '%s on %s' % (self.bset, + self.macros.expand('%{_host}')) + + current_path = os.environ['PATH'] + + start = datetime.datetime.now() + + have_errors = False + + try: + configs = self.load() + + log.trace('_bset: %2d: %s: configs: %s' % (nesting_count, self.bset, ','.join(configs))) + + sizes_valid = False + builds = [] + for s in range(0, len(configs)): + bs = None + b = None + try: + # + # Each section of the build set gets a separate set of + # macros so we do not contaminate one configuration with + # another. + # + opts = copy.copy(self.opts) + macros = copy.copy(self.macros) + self.set_host_details(host, opts, macros) + config, parent = configs[s].split(':', 2) + if config.endswith('.bset'): + log.trace('_bset: %2d: %s' % (nesting_count + 1, '=' * 75)) + bs = buildset(config, self.configs, opts, macros) + bs.build(host, nesting_count) + self._includes += \ + self._rebase_includes(bs.includes(), parent) + del bs + elif config.endswith('.cfg'): + log.trace('_bset: %2d: %s' % (nesting_count + 1, '-' * 75)) + try: + b = build.build(config, + False, + opts, + macros) + self._includes += \ + self._rebase_includes(b.includes(), parent) + except: + build_error = True + raise + self.build_package(config, b) + builds += [b] + # + # Dump post build macros. + # + log.trace('_bset: %2d: macros post-build' % (nesting_count)) + log.trace(str(macros)) + else: + raise error.general('invalid config type: %s' % (config)) + except error.general as gerr: + have_errors = True + if b is not None: + if self.build_failure is None: + self.build_failure = b.name() + self._includes += b.includes() + self._errors += \ + [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes + raise + # + # Clear out the builds ... + # + for b in builds: + del b + self._includes += \ + [find_bset_config(c.split(':')[0], self.macros) + ':' + self.bset for c in configs] + except error.general as gerr: + if not build_error: + log.stderr(str(gerr)) + raise + except KeyboardInterrupt: + raise + except: + self.build_failure = 'RSB general failure' + raise + finally: + end = datetime.datetime.now() + os.environ['PATH'] = current_path + build_time = str(end - start) + log.notice('Build Set: Time %s' % (build_time)) + +def list_hosts(): + hosts = sorted(profiles.keys()) + max_os_len = max(len(h) for h in hosts) + max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts) + for h in hosts: + log.notice('%*s: %-*s %s' % (max_os_len, h, max_host_len, + profiles[h]['_host'][2], + profiles[h]['_host'][2])) + +def get_files(configs, ext, localpath): + files = [] + if localpath: + for cp in configs['localpaths']: + files += [c for c in configs[cp] if c.endswith(ext)] + else: + files = [c for c in configs['files'] if c.endswith(ext)] + return files + +def get_config_files(configs, localpath = False): + return get_files(configs, '.cfg', localpath) + +def get_bset_files(configs, localpath = False): + return get_files(configs, '.bset', localpath) + +def get_config_bset_files(opts, configs): + cbs = get_config_files(configs) + get_bset_files(configs) + return strip_common_prefix([find_bset_config(cb, opts.defaults) for cb in cbs]) + +def get_root_bset_files(opts, configs, localpath = False): + bsets = get_bset_files(configs, localpath) + incs = {} + for bs in bsets: + bset = buildset(bs, configs, opts) + cfgs = [find_bset_config(c.split(':')[0], bset.macros) for c in bset.parse(bs, False)] + incs[bset.root()] = bset.includes() + cfgs + roots = sorted(incs.keys()) + for inc in incs: + for i in incs[inc]: + si = i.split(':') + if len(si) > 0 and si[0] in roots: + roots.remove(si[0]) + return roots + +def get_root(configs): + return configs['root'] + +def list_root_bset_files(opts, configs): + for p in configs['paths']: + log.notice('Examining: %s' % (os.path.relpath(p))) + for r in strip_common_prefix(get_root_bset_files(opts, configs)): + log.notice(' %s' % (r)) + +def list_bset_files(opts, configs): + for p in configs['paths']: + log.notice('Examining: %s' % (os.path.relpath(p))) + for b in get_bset_files(configs): + log.notice(' %s' % (b[:b.rfind('.')])) + +def load_log(logfile): + log.default = log.log(streams = [logfile]) + +def log_default(name): + return 'rsb-log-%s-%s.txt' % (name, datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + +def load_options(argv, argopts, defaults = '%{_sbdir}/defaults.mc', extras = []): + opts = options(argv, argopts, defaults, extras) + opts.defaults['rtems_version'] = str(argopts.rtems_version) + return opts |