path: root/source-builder/sb/simhost.py

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2020 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

#
# This code builds a package compiler tool suite given a tool set. A tool
# set lists the various tools. These are specific tool configurations.
#

from __future__ import print_function

import copy
import datetime
import os

try:
    from . import build
    from . import check
    from . import error
    from . import git
    from . import log
    from . import macros
    from . import path
    from . import shell
    from . import sources
    from . import version
except KeyboardInterrupt:
    print('abort: user terminated', file = sys.stderr)
    sys.exit(1)
except:
    raise

#
# Define host profiles so it can simulated on another host.
#
profiles = {
    'darwin':  { '_os':              ('none',    'none',     'darwin'),
                 '_host':            ('triplet', 'required', 'x86_64-apple-darwin18.5.0'),
                 '_host_vendor':     ('none',    'none',     'apple'),
                 '_host_os':         ('none',    'none',     'darwin'),
                 '_host_os_version': ('none',    'none',     '18.5.0'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'freebsd': { '_os':              ('none',    'none',     'freebsd'),
                 '_host':            ('triplet', 'required', 'x86_64-freebsd12.0-RELEASE-p3'),
                 '_host_vendor':     ('none',    'none',     'pc'),
                 '_host_os':         ('none',    'none',     'freebsd'),
                 '_host_os_version': ('none',    'none',     '12.0-RELEASE-p3'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'linux':   { '_os':              ('none',    'none',     'linux'),
                 '_host':            ('triplet', 'required', 'x86_64-linux-gnu'),
                 '_host_vendor':     ('none',    'none',     'gnu'),
                 '_host_os':         ('none',    'none',     'linux'),
                 '_host_os_version': ('none',    'none',     '4.18.0-16'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'netbsd':  { '_os':              ('none',    'none',     'netbsd'),
                 '_host':            ('triplet', 'required', 'x86_64-netbsd8.0'),
                 '_host_vendor':     ('none',    'none',     'pc'),
                 '_host_os':         ('none',    'none',     'netbsd'),
                 '_host_os_version': ('none',    'none',     '8.0'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'solaris': { '_os':              ('none',    'none',     'solaris'),
                 '_host':            ('triplet', 'required', 'x86_64-pc-solaris2'),
                 '_host_vendor':     ('none',    'none',     'pc'),
                 '_host_os':         ('none',    'none',     'solaris'),
                 '_host_os_version': ('none',    'none',     '2'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'win32':   { '_os':              ('none',    'none',     'win32'),
                 '_windows_os':      ('none',    'none',     'mingw32'),
                 '_host':            ('triplet', 'required', 'x86_64-w64-mingw32'),
                 '_host_vendor':     ('none',    'none',     'pc'),
                 '_host_os':         ('none',    'none',     'win32'),
                 '_host_os_version': ('none',    'none',     '10'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
    'cygwin':  { '_os':              ('none',    'none',     'win32'),
                 '_windows_os':      ('none',    'none',     'cygwin'),
                 '_host':            ('triplet', 'required', 'x86_64-w64-cygwin'),
                 '_host_vendor':     ('none',    'none',     'microsoft'),
                 '_host_os':         ('none',    'none',     'win32'),
                 '_host_os_version': ('none',    'none',     '10'),
                 '_host_cpu':        ('none',    'none',     'x86_64'),
                 '_host_alias':      ('none',    'none',     '%{nil}'),
                 '_host_arch':       ('none',    'none',     'x86_64'),
                 '_usr':             ('dir',     'optional', '/usr/local'),
                 '_var':             ('dir',     'optional', '/usr/local/var') },
}

class log_capture(object):
    def __init__(self):
        self.log = []
        log.capture = self.capture

    def __str__(self):
        return os.linesep.join(self.log)

    def capture(self, text):
        self.log += [l for l in text.replace(chr(13), '').splitlines()]

    def get(self):
        return self.log

    def clear(self):
        self.log = []

def find_bset_config(bset_config, macros):
    '''Find the build set or config file using the macro config defined path.'''
    name = bset_config
    if not path.exists(name):
        for cp in macros.expand('%{_configdir}').split(':'):
            configdir = path.abspath(cp)
            name = path.join(configdir, bset_config)
            if path.exists(name):
                break
            name = None
        if name is None:
            raise error.general('no build set file found: %s' % (bset_config))
    return name

def macro_expand(macros, _str):
    cstr = None
    while cstr != _str:
        cstr = _str
        _str = macros.expand(_str)
        _str = shell.expand(macros, _str)
    return _str

def strip_common_prefix(files):
    commonprefix = os.path.commonprefix(files)
    return sorted(list(set([f[len(commonprefix):] for f in files])))

#
# A skinny options command line class to get the configs to load.
#
class options(object):
    def __init__(self, argv, argopts, defaults, extras):
        command_path = path.dirname(path.abspath(argv[0]))
        if len(command_path) == 0:
            command_path = '.'
        self.command_path = command_path
        self.command_name = path.basename(argv[0])
        extras += ['--dry-run',
                   '--quiet',
                   '--without-log',
                   '--without-error-report',
                   '--without-release-url']
        self.argv = argv
        self.args = argv[1:] + extras
        self.defaults = macros.macros(name = defaults,
                                      sbdir = command_path)
        self.load_overrides()
        self.opts = { 'params' :  extras }
        self.sb_git()
        self.rtems_bsp()
        if 'download_dir' in argopts and argopts.download_dir is not None:
            self.defaults['_sourcedir'] = ('dir',
                                           'optional',
                                           path.abspath(argopts.download_dir))
            self.defaults['_patchdir'] = ('dir',
                                          'optional',
                                          path.abspath(argopts.download_dir))

    def load_overrides(self):
        overrides = None
        if os.name == 'nt':
            try:
                from . import windows
                overrides = windows.load()
                host_windows = True
                host_posix = False
            except:
                raise error.general('failed to load Windows host support')
        elif os.name == 'posix':
            uname = os.uname()
            try:
                if uname[0].startswith('MINGW64_NT'):
                    from . import windows
                    overrides = windows.load()
                    host_windows = True
                elif uname[0].startswith('CYGWIN_NT'):
                    from . import windows
                    overrides = windows.load()
                elif uname[0] == 'Darwin':
                    from . import darwin
                    overrides = darwin.load()
                elif uname[0] == 'FreeBSD':
                    from . import freebsd
                    overrides = freebsd.load()
                elif uname[0] == 'NetBSD':
                    from . import netbsd
                    overrides = netbsd.load()
                elif uname[0] == 'Linux':
                    from . import linux
                    overrides = linux.load()
                elif uname[0] == 'SunOS':
                    from . import solaris
                    overrides = solaris.load()
            except error.general as ge:
                raise error.general('failed to load %s host support: %s' % (uname[0], ge))
            except:
                raise error.general('failed to load %s host support' % (uname[0]))
        else:
            raise error.general('unsupported host type; please add')
        if overrides is None:
            raise error.general('no hosts defaults found; please add')
        for k in overrides:
            self.defaults[k] = overrides[k]

    def parse_args(self, arg, error = True, extra = True):
        for a in range(0, len(self.args)):
            if self.args[a].startswith(arg):
                lhs = None
                rhs = None
                if '=' in self.args[a]:
                    eqs = self.args[a].split('=')
                    lhs = eqs[0]
                    if len(eqs) > 2:
                        rhs = '='.join(eqs[1:])
                    else:
                        rhs = eqs[1]
                elif extra:
                    lhs = self.args[a]
                    a += 1
                    if a < len(self.args):
                        rhs = self.args[a]
                return [lhs, rhs]
            a += 1
        return None

    def rtems_bsp(self, arch='arch'):
        self.defaults['rtems_version'] = str(version.version())
        self.defaults['_target'] = arch + '-rtems'
        self.defaults['rtems_host'] = 'rtems-' + arch
        self.defaults['with_rtems_bsp'] = 'rtems-bsp'

    def sb_git(self):
        repo = git.repo(self.defaults.expand('%{_sbdir}'), self)
        repo_mail = None
        if repo.valid():
            repo_valid = '1'
            repo_head = repo.head()
            repo_clean = not repo.dirty()
            repo_remotes = '%{nil}'
            remotes = repo.remotes()
            if 'origin' in remotes:
                repo_remotes = '%s/origin' % (remotes['origin']['url'])
                repo_id = repo_head
            if not repo_clean:
                repo_id += '-modified'
                repo_mail = repo.email()
        else:
            repo_valid = '0'
            repo_head = '%{nil}'
            repo_clean = '%{nil}'
            repo_remotes = '%{nil}'
            repo_id = 'no-repo'
        self.defaults['_sbgit_valid'] = repo_valid
        self.defaults['_sbgit_head']  = repo_head
        self.defaults['_sbgit_clean'] = str(repo_clean)
        self.defaults['_sbgit_remotes'] = str(repo_remotes)
        self.defaults['_sbgit_id']    = repo_id
        if repo_mail is not None:
            self.defaults['_sbgit_mail'] = repo_mail

    def get_arg(self, arg):
        if self.optargs is None or arg not in self.optargs:
            return None
        return self.parse_args(arg)

    def with_arg(self, label, default = 'not-found'):
        # the default if there is no option for without.
        result = default
        for pre in ['with', 'without']:
            arg_str = '--%s-%s' % (pre, label)
            arg_label = '%s_%s' % (pre, label)
            arg = self.parse_args(arg_str, error = False, extra = False)
            if arg is not None:
                if arg[1] is  None:
                    result = 'yes'
                else:
                    result = arg[1]
                break
        return [arg_label, result]

    def dry_run(self):
        return True

    def keep_going(self):
        return False

    def quiet(self):
            return True

    def no_clean(self):
        return True

    def always_clean(self):
        return False

    def no_install(self):
        return True

    def download_disabled(self):
        return False

    def disable_install(self):
        return True

    def urls(self):
        return None

    def info(self):
        s = ' Command Line: %s%s' % (' '.join(self.argv), os.linesep)
        s += ' Python: %s' % (sys.version.replace('\n', ''))
        return s

class buildset:
    """Build a set builds a set of packages."""

    def __init__(self, bset, _configs, opts, macros = None):
        log.trace('_bset: %s: init' % (bset))
        self.parent = 'root'
        self._includes = []
        self._errors = []
        self.configs = _configs
        self.opts = opts
        if macros is None:
            self.macros = copy.copy(opts.defaults)
        else:
            self.macros = copy.copy(macros)
        self.macros.define('_rsb_getting_source')
        log.trace('_bset: %s: macro defaults' % (bset))
        log.trace(str(self.macros))
        self.bset = bset
        _target = self.macros.expand('%{_target}')
        if len(_target):
            pkg_prefix = _target
        else:
            pkg_prefix = self.macros.expand('%{_host}')
        self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
        self.build_failure = None

    def _add_includes(self, includes, parent = None):
        if parent is None:
            parent = self.parent
        if not isinstance(includes, list):
            includes = [includes]
        self._includes += [i + ':' + parent for i in includes]

    def _rebase_includes(self, includes, parent):
        if not isinstance(includes, list):
            includes = [includes]
        rebased = []
        for i in includes:
            if i.split(':', 2)[1] == 'root':
                rebased += [i.split(':', 2)[0] + ':' + parent]
            else:
                rebased += [i]
        return rebased

    def root(self):
        for i in self._includes:
            si = i.split(':')
            if len(si) == 2:
                if si[1] == 'root':
                    return si[0]
        return None

    def includes(self):
        return [i for i in self._includes if not i.endswith(':root')]

    def deps(self):
        return strip_common_prefix([i.split(':')[0] for i in self.includes()])

    def errors(self):
        return sorted(list(set(self._errors)))

    def build_package(self, _config, _build):
        if not _build.disabled():
            _build.make()

    def parse(self, bset, expand=True):

        #
        # Ouch, this is a copy of the setbuilder.py code.
        #

        def _clean(line):
            line = line[0:-1]
            b = line.find('#')
            if b >= 0:
                line = line[1:b]
            return line.strip()

        bsetname = find_bset_config(bset, self.macros)

        try:
            log.trace('_bset: %s: open: %s %s' % (self.bset, bsetname, expand))
            bsetf = open(path.host(bsetname), 'r')
        except IOError as err:
            raise error.general('error opening bset file: %s' % (bsetname))

        self._add_includes(bsetname)
        parent = self.parent
        self.parent = bsetname

        configs = []

        try:
            lc = 0
            for l in bsetf:
                lc += 1
                l = _clean(l)
                if len(l) == 0:
                    continue
                log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
                ls = l.split()
                if ls[0][-1] == ':' and ls[0][:-1] == 'package':
                    self.bset_pkg = ls[1].strip()
                    self.macros['package'] = self.bset_pkg
                elif ls[0][0] == '%' and (len(ls[0]) > 1 and ls[0][1] != '{'):
                    def err(msg):
                        raise error.general('%s:%d: %s' % (self.bset, lc, msg))
                    if ls[0] == '%define' or ls[0] == '%defineifnot' :
                        name = ls[1].strip()
                        value = None
                        if len(ls) > 2:
                            value = ' '.join([f.strip() for f in ls[2:]])
                        if ls[0] == '%defineifnot':
                            if self.macros.defined(name):
                                name = None
                        if name is not None:
                            if value is not None:
                                self.macros.define(name, value)
                            else:
                                self.macros.define(name)
                    elif ls[0] == '%undefine':
                        if len(ls) > 2:
                            raise error.general('%s:%d: %undefine requires ' \
                                                'just the name' % (self.bset, lc))
                        self.macros.undefine(ls[1].strip())
                    elif ls[0] == '%include':
                        configs += self.parse(ls[1].strip())
                    elif ls[0] in ['%patch', '%source']:
                        sources.process(ls[0][1:], ls[1:], self.macros, err)
                    elif ls[0] == '%hash':
                        sources.hash(ls[1:], self.macros, err)
                else:
                    try:
                        l = macro_expand(self.macros, l.strip())
                    except:
                        if expand:
                            raise
                        l = None
                    if l is not None:
                        c = build.find_config(l, self.configs)
                        if c is None:
                            raise error.general('%s:%d: cannot find file: %s'
                                                % (self.bset, lc, l))
                        configs += [c + ':' + self.parent]
        finally:
            bsetf.close()
            self.parent = parent

        return configs

    def load(self):
        #
        # If the build set file ends with .cfg the user has passed to the
        # buildset builder a configuration so we just return it.
        #
        if self.bset.endswith('.cfg'):
            self._add_includes(self.bset)
            configs = [self.bset]
        else:
            exbset = self.macros.expand(self.bset)
            self.macros['_bset'] = exbset
            self.macros['_bset_tmp'] = build.short_name(exbset)
            root, ext = path.splitext(exbset)
            if exbset.endswith('.bset'):
                bset = exbset
            else:
                bset = '%s.bset' % (exbset)
            configs = self.parse(bset)
        return configs

    def set_host_details(self, host, opts, macros):
        if host not in profiles:
            raise error.general('invalid host: ' + host)
        for m in profiles[host]:
            opts.defaults[m] = profiles[host][m]
            macros[m] = profiles[host][m]
        macros_to_copy = [('%{_build}',        '%{_host}'),
                          ('%{_build_alias}',  '%{_host_alias}'),
                          ('%{_build_arch}',   '%{_host_arch}'),
                          ('%{_build_cpu}',    '%{_host_cpu}'),
                          ('%{_build_os}',     '%{_host_os}'),
                          ('%{_build_vendor}', '%{_host_vendor}')]
        for m in macros_to_copy:
            opts.defaults[m[0]] = opts.defaults[m[1]]
            macros[m[0]] = macros[m[1]]
        #
        # Look for a valid cc and cxx.
        #
        for cc in ['/usr/bin/cc', '/usr/bin/clang', '/usr/bin/gcc']:
            if check.check_exe(cc, cc):
                opts.defaults['__cc'] = cc
                macros['__cc'] = cc
                break
        if not macros.defined('__cc'):
            raise error.general('no valid cc found')
        for cxx in ['/usr/bin/c++', '/usr/bin/clang++', '/usr/bin/g++']:
            if check.check_exe(cxx, cxx):
                opts.defaults['__cxx'] = cxx
                macros['__cxx'] = cxx
        if not macros.defined('__cxx'):
            raise error.general('no valid c++ found')

    def build(self, host, nesting_count = 0):

        build_error = False

        nesting_count += 1

        log.trace('_bset: %2d: %s for %s: make' % (nesting_count, self.bset, host))
        log.notice('Build Set: %s for %s' % (self.bset, host))

        mail_subject = '%s on %s' % (self.bset,
                                     self.macros.expand('%{_host}'))

        current_path = os.environ['PATH']

        start = datetime.datetime.now()

        have_errors = False

        try:
            configs = self.load()

            log.trace('_bset: %2d: %s: configs: %s'  % (nesting_count, self.bset, ','.join(configs)))

            sizes_valid = False
            builds = []
            for s in range(0, len(configs)):
                bs = None
                b = None
                try:
                    #
                    # Each section of the build set gets a separate set of
                    # macros so we do not contaminate one configuration with
                    # another.
                    #
                    opts = copy.copy(self.opts)
                    macros = copy.copy(self.macros)
                    self.set_host_details(host, opts, macros)
                    config, parent = configs[s].split(':', 2)
                    if config.endswith('.bset'):
                        log.trace('_bset: %2d: %s' % (nesting_count + 1, '=' * 75))
                        bs = buildset(config, self.configs, opts, macros)
                        bs.build(host, nesting_count)
                        self._includes += \
                            self._rebase_includes(bs.includes(), parent)
                        del bs
                    elif config.endswith('.cfg'):
                        log.trace('_bset: %2d: %s' % (nesting_count + 1, '-' * 75))
                        try:
                            b = build.build(config,
                                            False,
                                            opts,
                                            macros)
                            self._includes += \
                                self._rebase_includes(b.includes(), parent)
                        except:
                            build_error = True
                            raise
                        self.build_package(config, b)
                        builds += [b]
                        #
                        # Dump post build macros.
                        #
                        log.trace('_bset: %2d: macros post-build' % (nesting_count))
                        log.trace(str(macros))
                    else:
                        raise error.general('invalid config type: %s' % (config))
                except error.general as gerr:
                    have_errors = True
                    if b is not None:
                        if self.build_failure is None:
                            self.build_failure = b.name()
                        self._includes += b.includes()
                    self._errors += \
                        [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
                    raise
            #
            # Clear out the builds ...
            #
            for b in builds:
                del b
            self._includes += \
                [find_bset_config(c.split(':')[0], self.macros) + ':' + self.bset for c in configs]
        except error.general as gerr:
            if not build_error:
                log.stderr(str(gerr))
            raise
        except KeyboardInterrupt:
            raise
        except:
            self.build_failure = 'RSB general failure'
            raise
        finally:
            end = datetime.datetime.now()
            os.environ['PATH'] = current_path
            build_time = str(end - start)
            log.notice('Build Set: Time %s' % (build_time))

def list_hosts():
    hosts = sorted(profiles.keys())
    max_os_len = max(len(h) for h in hosts)
    max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts)
    for h in hosts:
        log.notice('%*s: %-*s %s' % (max_os_len, h, max_host_len,
                                     profiles[h]['_host'][2],
                                     profiles[h]['_host'][2]))

def get_files(configs, ext, localpath):
    files = []
    if localpath:
        for cp in configs['localpaths']:
            files += [c for c in configs[cp] if c.endswith(ext)]
    else:
        files = [c for c in configs['files'] if c.endswith(ext)]
    return files

def get_config_files(configs, localpath = False):
    return get_files(configs, '.cfg', localpath)

def get_bset_files(configs, localpath = False):
    return get_files(configs, '.bset', localpath)

def get_config_bset_files(opts, configs):
    cbs = get_config_files(configs) + get_bset_files(configs)
    return strip_common_prefix([find_bset_config(cb, opts.defaults) for cb in cbs])

def get_root_bset_files(opts, configs, localpath = False):
    bsets = get_bset_files(configs, localpath)
    incs = {}
    for bs in bsets:
        bset = buildset(bs, configs, opts)
        cfgs = [find_bset_config(c.split(':')[0], bset.macros) for c in bset.parse(bs, False)]
        incs[bset.root()] = bset.includes() + cfgs
    roots = sorted(incs.keys())
    for inc in incs:
        for i in incs[inc]:
            si = i.split(':')
            if len(si) > 0 and si[0] in roots:
                roots.remove(si[0])
    return roots

def get_root(configs):
    return configs['root']

def list_root_bset_files(opts, configs):
    for p in configs['paths']:
        log.notice('Examining: %s' % (os.path.relpath(p)))
    for r in strip_common_prefix(get_root_bset_files(opts, configs)):
        log.notice(' %s' % (r))

def list_bset_files(opts, configs):
    for p in configs['paths']:
        log.notice('Examining: %s' % (os.path.relpath(p)))
    for b in get_bset_files(configs):
        log.notice(' %s' % (b[:b.rfind('.')]))

def load_log(logfile):
    log.default = log.log(streams = [logfile])

def log_default(name):
    return 'rsb-log-%s-%s.txt' % (name, datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))

def load_options(argv, argopts, defaults = '%{_sbdir}/defaults.mc', extras = []):
    opts = options(argv, argopts, defaults, extras)
    opts.defaults['rtems_version'] = str(argopts.rtems_version)
    return opts