path: root/source-builder/sb/build.py

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

#
# This code builds a package given a config file. It only builds to be
# installed not to be package unless you run a packager around this.
#

from __future__ import print_function

import copy
import getopt
import glob
import os
import shutil
import stat
import sys

try:
    from . import check
    from . import config
    from . import download
    from . import error
    from . import ereport
    from . import execute
    from . import log
    from . import options
    from . import path
    from . import sources
    from . import version
except KeyboardInterrupt:
    print('abort: user terminated')
    sys.exit(1)
except:
    raise

def humanize_number(num, suffix):
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "%5.3f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.3f%s%s" % (size, 'Y', suffix)

def short_name(name):
    #
    # If on Windows use short names to keep the build paths as short as possible.
    #
    if options.host_windows:
        buildname = ''
        add = True
        for n in name.split('-'):
            buildname += n[0]
        return buildname
    else:
        return name

class script:
    """Create and manage a shell script."""

    def __init__(self):
        self.reset()

    def reset(self):
        self.body = []
        self.lc = 0

    def append(self, text):
        is_str = False
        if type(text) is str:
            is_str = True
        try:
            if type(text) is unicode:
                is_str = True
        except:
            pass
        if is_str:
            text = text.splitlines()
        if not log.quiet:
            i = 0
            for l in text:
                i += 1
                log.output('script:%3d: %s' % (self.lc + i, l))
        self.lc += len(text)
        self.body.extend(text)

    def write(self, name, check_for_errors = False):
        s = None
        try:
            s = open(path.host(name), 'w')
            s.write('\n'.join(self.body))
            s.close()
            os.chmod(path.host(name), stat.S_IRWXU | \
                         stat.S_IRGRP | stat.S_IXGRP | \
                         stat.S_IROTH | stat.S_IXOTH)
        except IOError as err:
            raise error.general('creating script: ' + name)
        except:
            if s is not None:
                s.close()
            raise
        if s is not None:
            s.close()

class build:
    """Build a package given a config file."""

    def _generate_report_(self, header, footer = None):
        ereport.generate('rsb-report-%s.txt' % self.macros['name'],
                         self.opts, header, footer)

    def __init__(self, name, create_tar_files, opts, macros = None):
        try:
            self.opts = opts
            self.init_name = name
            self.init_macros = macros
            self.config = None
            self.create_tar_files = create_tar_files
            log.notice('config: ' + name)
            self.set_macros(macros)
            self.config = config.file(name, opts, self.macros)
            self.script_build = script()
            self.script_clean = script()
            self.macros['buildname'] = short_name(self.macros['name'])
        except error.general as gerr:
            log.notice(str(gerr))
            log.stderr('Build FAILED')
            raise
        except error.internal as ierr:
            log.notice(str(ierr))
            log.stderr('Internal Build FAILED')
            raise
        except:
            raise

    def copy_init_macros(self):
        return copy.copy(self.init_macros)

    def copy_macros(self):
        return copy.copy(self.macros)

    def set_macros(self, macros):
        if macros is None:
            self.macros = copy.copy(self.opts.defaults)
        else:
            self.macros = copy.copy(macros)
        if self.config:
            self.config.set_macros(self.macros)

    def rmdir(self, rmpath):
        log.output('removing: %s' % (path.host(rmpath)))
        if not self.opts.dry_run():
            if path.exists(rmpath):
                path.removeall(rmpath)

    def mkdir(self, mkpath):
        log.output('making dir: %s' % (path.host(mkpath)))
        if not self.opts.dry_run():
            path.mkdir(mkpath)

    def canadian_cross(self):
        _host = self.config.expand('%{_host}')
        _build = self.config.expand('%{_build}')
        _target = self.config.expand('%{_target}')
        _allowed = self.config.defined('%{allow_cxc}')
        if len(_host) and len(_build) and (_target) and \
           _allowed and _host != _build and _host != _target:
            return True
        return False

    def installable(self):
        _host = self.config.expand('%{_host}')
        _build = self.config.expand('%{_build}')
        _canadian_cross = self.canadian_cross()
        if self.macros.get('_disable_installing') and \
           self.config.expand('%{_disable_installing}') == 'yes':
            _disable_installing = True
        else:
            _disable_installing = False
        _no_install = self.opts.no_install()
        log.trace('_build: installable: host=%s build=%s ' \
                  'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
                  (_host, _build, _no_install, _canadian_cross, _disable_installing, \
                   self.disabled()))
        return len(_host) and len(_build) and \
            not self.disabled() and \
            not _disable_installing and \
            not _canadian_cross

    def source(self, name, strip_components, download_only):
        #
        # Return the list of sources. Merge in any macro defined sources as
        # these may be overridden by user loaded macros.
        #
        _map = 'source-%s' % (name)
        src_keys = [s for s in self.macros.map_keys(_map) if s != 'setup']
        if len(src_keys) == 0:
            raise error.general('no source set: %s (%s)' % (name, _map))
        srcs = []
        for s in src_keys:
            sm = self.macros.get(s, globals = False, maps = _map)
            if sm is None:
                raise error.internal('source macro not found: %s in %s (%s)' % \
                                         (s, name, _map))
            opts = []
            url = []
            for sp in sm[2].split():
                if len(url) == 0 and sp[0] == '-':
                    opts += [sp]
                else:
                    url += [sp]
            if len(url) == 0:
                raise error.general('source URL not found: %s' % (' '.join(args)))
            #
            # Look for --rsb-file as an option we use as a local file name.
            # This can be used if a URL has no reasonable file name the
            # download URL parser can figure out.
            #
            file_override = None
            if len(opts) > 0:
                for o in opts:
                    if o.startswith('--rsb-file'):
                       os_ = o.split('=')
                       if len(os_) != 2:
                           raise error.general('invalid --rsb-file option: %s' % \
                                               (' '.join(args)))
                       if os_[0] != '--rsb-file':
                           raise error.general('invalid --rsb-file option: %s' % \
                                               (' '.join(args)))
                       file_override = os_[1]
                opts = [o for o in opts if not o.startswith('--rsb-')]
            url = self.config.expand(' '.join(url))
            src = download.parse_url(url, '_sourcedir',
                                     self.config, self.opts, file_override)
            download.get_file(src['url'], src['local'], self.opts, self.config)
            if not download_only:
                if strip_components > 0:
                    tar_extract = '%%{__tar_extract} --strip-components %d' % \
                        (strip_components)
                else:
                    tar_extract = '%{__tar_extract}'
                if 'symlink' in src:
                    sname = name.replace('-', '_')
                    src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % \
                        (src['symlink'], sname)
                elif 'compressed' in src:
                    #
                    # Zip files unpack as well so do not use tar.
                    #
                    src['script'] = '%s %s' % (src['compressed'], src['local'])
                    if src['compressed-type'] != 'zip':
                        src['script'] += ' | %s -f -' % (tar_extract)
                else:
                    src['script'] = '%s -f %s' % (tar_extract, src['local'])
                srcs += [src]
        return srcs

    def source_setup(self, package, args):
        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
        setup_name = args[1]
        args = args[1:]
        try:
            opts, args = getopt.getopt(args[1:], 'qDcn:bas:gE')
        except getopt.GetoptError as ge:
            raise error.general('source setup error: %s' % str(ge))
        quiet = False
        unpack_before_chdir = True
        delete_before_unpack = True
        create_dir = False
        deleted_dir = False
        created_dir = False
        changed_dir = False
        no_errors = False
        strip_components = 0
        opt_name = None
        download_only = False
        for o in opts:
            if o[0] == '-q':
                quiet = True
            elif o[0] == '-D':
                delete_before_unpack = False
            elif o[0] == '-c':
                create_dir = True
            elif o[0] == '-n':
                opt_name = o[1]
            elif o[0] == '-b':
                unpack_before_chdir = True
            elif o[0] == '-a':
                unpack_before_chdir = False
            elif o[0] == '-E':
                no_errors = True
            elif o[0] == '-s':
                if not o[1].isdigit():
                    raise error.general('source setup error: invalid strip count: %s' % \
                                        (o[1]))
                strip_components = int(o[1])
            elif o[0] == '-g':
                download_only = True
        name = None
        for source in self.source(setup_name, strip_components, download_only):
            if name is None:
                if opt_name is None:
                    if source:
                        opt_name = source['name']
                    else:
                        raise error.general('setup source tag not found: %d' % \
                                            (source_tag))
                else:
                    name = opt_name
            if not download_only:
                self.script_build.append(self.config.expand('cd %{_builddir}'))
                if not deleted_dir and delete_before_unpack and name is not None:
                    self.script_build.append(self.config.expand('%{__rm} -rf ' + name))
                    deleted_dir = True
                if not created_dir and create_dir and name is not None:
                    self.script_build.append(self.config.expand('%{__mkdir_p} ' + name))
                    created_dir = True
                if not changed_dir and (not unpack_before_chdir or create_dir) and \
                   name is not None:
                    self.script_build.append(self.config.expand('cd ' + name))
                    changed_dir = True
                #
                # On Windows tar can fail on links if the link appears in the
                # tar file before the target of the link exists. We can assume the
                # tar file is correct, that is all files and links are valid,
                # so on error redo the untar a second time.
                #
                if options.host_windows or no_errors:
                    self.script_build.append('set +e')
                self.script_build.append(self.config.expand(source['script']))
                if options.host_windows or not no_errors:
                    self.script_build.append('tar_exit=$?')
                if options.host_windows or no_errors:
                    self.script_build.append('set -e')
                if options.host_windows:
                    if no_errors:
                        self.script_build.append(' set +e')
                        self.script_build.append(' ' + self.config.expand(source['script']))
                        self.script_build.append(' set -e')
                    else:
                        self.script_build.append('if test $tar_exit != 0; then')
                        self.script_build.append(' ' + self.config.expand(source['script']))
                        self.script_build.append('fi')
        if not changed_dir and (unpack_before_chdir and not create_dir) and \
           name is not None and not download_only:
            self.script_build.append(self.config.expand('cd ' + name))
            changed_dir = True
        self.script_build.append(self.config.expand('%{__setup_post}'))

    def patch_setup(self, package, args):
        name = args[1]
        args = args[2:]
        _map = 'patch-%s' % (name)
        default_opts = ' '.join(args)
        patch_keys = [p for p in self.macros.map_keys(_map) if p != 'setup']
        patches = []
        for p in patch_keys:
            pm = self.macros.get(p, globals = False, maps = _map)
            if pm is None:
                raise error.internal('patch macro not found: %s in %s (%s)' % \
                                         (p, name, _map))
            opts = []
            url = []
            for pp in pm[2].split():
                if len(url) == 0 and pp[0] == '-':
                    opts += [pp]
                else:
                    url += [pp]
            if len(url) == 0:
                raise error.general('patch URL not found: %s' % (' '.join(opts)))
            #
            # Look for --rsb-file as an option we use as a local file name.
            # This can be used if a URL has no reasonable file name the
            # download URL parser can figure out.
            #
            file_override = None
            if len(opts) > 0:
                for o in opts:
                    if o.startswith('--rsb-file'):
                       os_ = o.split('=')
                       if len(os_) != 2:
                           raise error.general('invalid --rsb-file option: %s' % \
                                               (' '.join(opts)))
                       if os_[0] != '--rsb-file':
                           raise error.general('invalid --rsb-file option: %s' % \
                                               (' '.join(opts)))
                       file_override = os_[1]
                opts = [o for o in opts if not o.startswith('--rsb-')]
            if len(opts) == 0:
                opts = default_opts
            else:
                opts = ' '.join(opts)
            opts = self.config.expand(opts)
            url = self.config.expand(' '.join(url))
            #
            # Parse the URL first in the source builder's patch directory.
            #
            patch = download.parse_url(url, '_patchdir', self.config,
                                       self.opts, file_override)
            #
            # Download the patch
            #
            download.get_file(patch['url'], patch['local'], self.opts, self.config)
            if 'compressed' in patch:
                patch['script'] = patch['compressed'] + ' ' +  patch['local']
            else:
                patch['script'] = '%{__cat} ' + patch['local']
            patch['script'] += ' | %%{__patch} %s' % (opts)
            self.script_build.append(self.config.expand(patch['script']))

    def run(self, command, shell_opts = '', cwd = None):
        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
        log.output('run: ' + cmd)
        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
        if exit_code != 0:
            log.output('shell cmd failed: %s' % (cmd))
            raise error.general('building %s' % (self.macros['buildname']))

    def builddir(self):
        builddir = self.config.abspath('_builddir')
        self.rmdir(builddir)
        if not self.opts.dry_run():
            self.mkdir(builddir)

    def prep(self, package):
        self.script_build.append('echo "==> %prep:"')
        _prep = package.prep()
        if _prep:
            for l in _prep:
                args = l.split()
                if len(args):
                    def err(msg):
                        raise error.general('%s: %s' % (package, msg))
                    if args[0] == '%setup':
                        if len(args) == 1:
                            raise error.general('invalid %%setup directive: %s' % \
                                                (' '.join(args)))
                        if args[1] == 'source':
                            self.source_setup(package, args[1:])
                        elif args[1] == 'patch':
                            self.patch_setup(package, args[1:])
                    elif args[0] in ['%patch', '%source']:
                        sources.process(args[0][1:], args[1:], self.macros, err)
                    elif args[0] == '%hash':
                        sources.hash(args[1:], self.macros, err)
                        self.hash(package, args)
                    else:
                        self.script_build.append(' '.join(args))

    def build(self, package):
        self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
        self.script_build.append('%s ${SB_BUILD_ROOT}' %
                                 (self.config.expand('%{__rmdir}')))
        self.script_build.append('%s ${SB_BUILD_ROOT}' %
                                 (self.config.expand('%{__mkdir_p}')))
        self.script_build.append('echo "==> %build:"')
        _build = package.build()
        if _build:
            for l in _build:
                self.script_build.append(l)

    def install(self, package):
        self.script_build.append('echo "==> %install:"')
        _install = package.install()
        if _install:
            for l in _install:
                args = l.split()
                self.script_build.append(' '.join(args))

    def files(self, package):
        if self.create_tar_files \
           and not self.macros.get('%{_disable_packaging'):
            self.script_build.append('echo "==> %files:"')
            inpath = path.abspath(self.config.expand('%{buildroot}'))
            tardir = path.abspath(self.config.expand('%{_tardir}'))
            self.script_build.append(self.config.expand('if test -d %s; then' % (inpath)))
            self.script_build.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
            self.script_build.append(self.config.expand('  cd ' + inpath))
            tar = path.join(tardir, package.long_name() + '.tar.bz2')
            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
            self.script_build.append(cmd)
            self.script_build.append(self.config.expand('  cd %{_builddir}'))
            self.script_build.append('fi')

    def clean(self, package):
        self.script_clean.reset()
        self.script_clean.append(self.config.expand('%{___build_template}'))
        self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"')
        self.script_clean.append('echo "==> %clean:"')
        _clean = package.clean()
        if _clean is not None:
            for l in _clean:
                args = l.split()
                self.script_clean.append(' '.join(args))

    def sizes(self, package):
        def _sizes(package, what, path):
            package.set_size(what, path)
            s = humanize_number(package.get_size(what), 'B')
            log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what)))
            return s
        s = {}
        for p in [('build', '%{_builddir}'),
                  ('build', '%{buildroot}'),
                  ('installed', '%{buildroot}')]:
            hs = _sizes(package, p[0], self.config.expand(p[1]))
            s[p[0]] = hs
        log.notice('sizes: %s: %s (installed: %s)' % (package.name(),
                                                      s['build'],
                                                      s['installed']))

    def build_package(self, package):
        if self.canadian_cross():
            if not self.config.defined('%{allow_cxc}'):
                raise error.general('Canadian Cross is not allowed')
            self.script_build.append('echo "==> Candian-cross build/target:"')
            self.script_build.append('SB_CXC="yes"')
        else:
            self.script_build.append('SB_CXC="no"')
        self.build(package)
        self.install(package)
        self.files(package)
        if not self.opts.no_clean():
            self.clean(package)

    def cleanup(self):
        package = self.main_package()
        if not package.disabled() and not self.opts.no_clean():
            buildroot = self.config.abspath('buildroot')
            builddir = self.config.abspath('_builddir')
            buildcxcdir = self.config.abspath('_buildcxcdir')
            tmproot = self.config.abspath('_tmproot')
            log.trace('cleanup: %s' % (buildroot))
            self.rmdir(buildroot)
            log.trace('cleanup: %s' % (builddir))
            self.rmdir(builddir)
            if self.canadian_cross():
                log.trace('cleanup: %s' % (buildcxcdir))
                self.rmdir(buildcxcdir)
            log.trace('cleanup: %s' % (tmproot))
            self.rmdir(tmproot)

    def main_package(self):
        packages = self.config.packages()
        return packages['main']

    def reload(self):
        self.config.load(self.init_name)

    def make(self):
        package = self.main_package()
        if package.disabled():
            log.notice('package: nothing to build')
        else:
            try:
                name = package.name()
                if self.canadian_cross():
                    cxc_label = '(Cxc) '
                else:
                    cxc_label = ''
                log.notice('package: %s%s' % (cxc_label, name))
                log.trace('---- macro maps %s' % ('-' * 55))
                log.trace('%s' % (str(self.config.macros)))
                log.trace('-' * 70)
                self.script_build.reset()
                self.script_build.append(self.config.expand('%{___build_template}'))
                self.script_build.append('echo "=> ' + name + ': BUILD"')
                self.prep(package)
                self.build_package(package)
                if not self.opts.dry_run():
                    self.builddir()
                    build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build')
                    log.output('write script: ' + build_sn)
                    self.script_build.write(build_sn)
                    clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean')
                    log.output('write script: ' + clean_sn)
                    self.script_clean.write(clean_sn)
                    log.notice('building: %s%s' % (cxc_label, name))
                    self.run(build_sn)
                    self.sizes(package)
                    log.notice('cleaning: %s%s' % (cxc_label, name))
                    self.run(clean_sn)
            except error.general as gerr:
                log.notice(str(gerr))
                log.stderr('Build FAILED')
                self._generate_report_('Build: %s' % (gerr))
                raise
            except error.internal as ierr:
                log.notice(str(ierr))
                log.stderr('Internal Build FAILED')
                self._generate_report_('Build: %s' % (ierr))
                raise
            except:
                raise
            if self.opts.dry_run():
                self._generate_report_('Build: dry run (no actual error)',
                                       'Build: dry run (no actual error)')

    def name(self):
        packages = self.config.packages()
        package = packages['main']
        return package.name()

    def disabled(self):
        packages = self.config.packages()
        package = packages['main']
        return package.disabled()

    def get_build_size(self):
        package = self.main_package()
        if package.disabled():
            return 0
        return package.get_size('build')

    def get_installed_size(self):
        package = self.main_package()
        if package.disabled():
            return 0
        return package.get_size('installed')

    def includes(self):
        if self.config:
            return self.config.includes()

def get_configs(opts):

    def _scan(_path, ext):
        configs = []
        for root, dirs, files in os.walk(_path):
            prefix = root[len(_path) + 1:]
            for file in files:
                for e in ext:
                    if file.endswith(e):
                        configs += [path.join(prefix, file)]
        return configs

    configs = { 'paths': [], 'files': [] }
    paths = opts.defaults.expand('%{_configdir}').split(':')
    root = path.host(os.path.commonprefix(paths))
    configs['root'] = root
    configs['localpaths'] = [lp[len(root):] for lp in paths]
    for cp in paths:
        hcp = path.host(path.abspath(cp))
        configs['paths'] += [hcp]
        hpconfigs = sorted(set(_scan(hcp, ['.cfg', '.bset'])))
        hcplocal = hcp[len(root):]
        configs[hcplocal] = [path.join(hcplocal, c) for c in hpconfigs]
        configs['files'] += hpconfigs
    configs['files'] = sorted(set(configs['files']))
    return configs

def find_config(config, configs):
    config_root, config_ext = path.splitext(config)
    if config_ext not in ['', '.bset', '.cfg']:
        config_root = config
        config_ext = ''
    for c in configs['files']:
        r, e = path.splitext(c)
        if config_root == r:
            if config_ext == '' or config_ext == e:
                return c
    return None

def run(args):
    ec = 0
    try:
        optargs = { '--list-configs': 'List available configurations' }
        opts = options.load(args, optargs)
        log.notice('RTEMS Source Builder, Package Builder, %s' % (version.string()))
        opts.log_info()
        if not check.host_setup(opts):
            if not opts.force():
                raise error.general('host build environment is not set up' +
                                    ' correctly (use --force to proceed)')
            log.notice('warning: forcing build with known host setup problems')
        if opts.get_arg('--list-configs'):
            configs = get_configs(opts)
            for p in configs['paths']:
                print('Examining: %s' % (os.path.relpath(p)))
                for c in configs['files']:
                    if c.endswith('.cfg'):
                        print('    %s' % (c))
        else:
            for config_file in opts.config_files():
                b = build(config_file, True, opts)
                b.make()
                b = None
    except error.general as gerr:
        log.stderr('Build FAILED')
        ec = 1
    except error.internal as ierr:
        log.stderr('Internal Build FAILED')
        ec = 1
    except error.exit as eerr:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        ec = 1
    sys.exit(ec)

if __name__ == "__main__":
    run(sys.argv)