diff options
Diffstat (limited to 'source-builder/sb/setbuilder.py')
| -rw-r--r-- | source-builder/sb/setbuilder.py | 340 |
1 files changed, 254 insertions, 86 deletions
diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py index 17b781a..b0e2b23 100644 --- a/source-builder/sb/setbuilder.py +++ b/source-builder/sb/setbuilder.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2018 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -33,22 +33,30 @@ import sys import textwrap try: - import build - import check - import error - import log - import mailer - import options - import path - import reports - import sources - import version + from . import build + from . import check + from . import error + from . import log + from . import mailer + from . import options + from . import path + from . import reports + from . import shell + from . import sources + from . import version except KeyboardInterrupt: print('abort: user terminated', file = sys.stderr) sys.exit(1) except: - print('error: unknown application load error', file = sys.stderr) - sys.exit(1) + raise + +def macro_expand(macros, _str): + cstr = None + while cstr != _str: + cstr = _str + _str = macros.expand(_str) + _str = shell.expand(macros, _str) + return _str class log_capture(object): def __init__(self): @@ -71,21 +79,21 @@ class buildset: """Build a set builds a set of packages.""" def __init__(self, bset, _configs, opts, macros = None): - log.trace('_bset: %s: init' % (bset)) + log.trace('_bset: : %s: init' % (bset)) self.configs = _configs self.opts = opts if macros is None: self.macros = copy.copy(opts.defaults) else: self.macros = copy.copy(macros) - log.trace('_bset: %s: macro defaults' % (bset)) + log.trace('_bset: : %s: macro defaults' % (bset)) log.trace(str(self.macros)) self.bset = bset - _target = self.macros.expand('%{_target}') + _target = macro_expand(self.macros, '%{_target}') if len(_target): pkg_prefix = _target else: - pkg_prefix = self.macros.expand('%{_host}') + pkg_prefix = macro_expand(self.macros, '%{_host}') self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset) self.mail_header = '' self.mail_report = '' @@ -120,6 +128,25 @@ class buildset: def get_mail_report(self): return self.mail_report + def mail_single_report(self): + return self.macros.get('%{mail_single_report}') != 0 + + def mail_active(self, mail, nesting_count = 1): + return mail is not None and not (self.mail_single_report() and nesting_count > 1) + + def mail_send(self, mail): + if True: #not self.opts.dry_run(): + mail_subject = '%s on %s' % (self.bset, self.macros.expand('%{_host}')) + if mail['failure'] is not None: + mail_subject = 'FAILED %s (%s)' % (mail_subject, mail['failure']) + else: + mail_subject = 'PASSED %s' % (mail_subject) + mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'), + mail_subject) + body = mail['log'] + body += (os.linesep * 2).join(mail['reports']) + mail['mail'].send(mail['to'], mail_subject, body) + def copy(self, src, dst): log.output('copy: %s => %s' % (path.host(src), path.host(dst))) if not self.opts.dry_run(): @@ -152,7 +179,7 @@ class buildset: else: raise error.general('invalid report format: %s' % (format)) buildroot = _build.config.abspath('%{buildroot}') - prefix = _build.macros.expand('%{_prefix}') + prefix = macro_expand(_build.macros, '%{_prefix}') name = _build.main_package().name() + ext log.notice('reporting: %s -> %s' % (_config, name)) if not _build.opts.get_arg('--no-report'): @@ -161,7 +188,7 @@ class buildset: outname = path.host(path.join(outpath, name)) else: outname = None - r = reports.report(format, self.configs, + r = reports.report(format, False, self.configs, copy.copy(opts), copy.copy(macros)) r.introduction(_build.config.file_name()) r.generate(_build.config.file_name()) @@ -171,7 +198,7 @@ class buildset: r.write(outname) del r if mail: - r = reports.report('text', self.configs, + r = reports.report('text', True, self.configs, copy.copy(opts), copy.copy(macros)) r.introduction(_build.config.file_name()) r.generate(_build.config.file_name()) @@ -182,17 +209,25 @@ class buildset: def root_copy(self, src, dst): what = '%s -> %s' % \ (os.path.relpath(path.host(src)), os.path.relpath(path.host(dst))) - log.trace('_bset: %s: collecting: %s' % (self.bset, what)) + log.trace('_bset: : %s: collecting: %s' % (self.bset, what)) self.copy(src, dst) - def install(self, name, buildroot, prefix): - dst = prefix - src = path.join(buildroot, prefix) - log.notice('installing: %s -> %s' % (name, path.host(dst))) + def install(self, mode, name, src, dst): + log.trace('_bset: : %s: copy %s -> %s' % (mode, src, dst)) + log.notice('%s: %s -> %s' % (mode, name, path.host(dst))) self.copy(src, dst) + def install_mode(self): + return macro_expand(self.macros, '%{install_mode}') + + def installing(self): + return self.install_mode() == 'installing' + + def staging(self): + return not self.installing() + def canadian_cross(self, _build): - log.trace('_bset: Cxc for build machine: _build => _host') + log.trace('_bset: : Cxc for build machine: _build => _host') macros_to_copy = [('%{_host}', '%{_build}'), ('%{_host_alias}', '%{_build_alias}'), ('%{_host_arch}', '%{_build_arch}'), @@ -204,7 +239,7 @@ class buildset: ('%{_builddir}', '%{_buildcxcdir}')] cxc_macros = _build.copy_init_macros() for m in macros_to_copy: - log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]])) + log.trace('_bset: : Cxc: %s <= %s' % (m[0], cxc_macros[m[1]])) cxc_macros[m[0]] = cxc_macros[m[1]] _build.set_macros(cxc_macros) _build.reload() @@ -229,12 +264,15 @@ class buildset: if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \ and not _build.macros.get('%{_disable_packaging}'): path.mkdir(tardir) - tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name()))) + tar = path.join(tardir, + _build.config.expand('%s.tar.bz2' % \ + (_build.main_package().name()))) log.notice('tarball: %s' % (os.path.relpath(path.host(tar)))) if not self.opts.dry_run(): tmproot = _build.config.expand('%{_tmproot}') cmd = _build.config.expand('"cd ' + tmproot + \ - ' && %{__tar} -cf - . | %{__bzip2} > ' + tar + '"') + ' && %{__tar} -cf - . | %{__bzip2} > ' + \ + tar + '"') _build.run(cmd, shell_opts = '-c', cwd = tmproot) def parse(self, bset): @@ -249,7 +287,7 @@ class buildset: bsetname = bset if not path.exists(bsetname): - for cp in self.macros.expand('%{_configdir}').split(':'): + for cp in macro_expand(self.macros, '%{_configdir}').split(':'): configdir = path.abspath(cp) bsetname = path.join(configdir, bset) if path.exists(bsetname): @@ -258,7 +296,7 @@ class buildset: if bsetname is None: raise error.general('no build set file found: %s' % (bset)) try: - log.trace('_bset: %s: open: %s' % (self.bset, bsetname)) + log.trace('_bset: : %s: open: %s' % (self.bset, bsetname)) bset = open(path.host(bsetname), 'r') except IOError as err: raise error.general('error opening bset file: %s' % (bsetname)) @@ -272,7 +310,7 @@ class buildset: l = _clean(l) if len(l) == 0: continue - log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l)) + log.trace('_bset: : %s: %03d: %s' % (self.bset, lc, l)) ls = l.split() if ls[0][-1] == ':' and ls[0][:-1] == 'package': self.bset_pkg = ls[1].strip() @@ -288,8 +326,8 @@ class buildset: self.macros.define(ls[1].strip()) elif ls[0] == '%undefine': if len(ls) > 2: - raise error.general('%s:%d: %undefine requires just the name' % \ - (self.bset, lc)) + raise error.general('%s:%d: %undefine requires ' \ + 'just the name' % (self.bset, lc)) self.macros.undefine(ls[1].strip()) elif ls[0] == '%include': configs += self.parse(ls[1].strip()) @@ -301,7 +339,8 @@ class buildset: l = l.strip() c = build.find_config(l, self.configs) if c is None: - raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l)) + raise error.general('%s:%d: cannot find file: %s' % (self.bset, + lc, l)) configs += [c] except: bset.close() @@ -320,8 +359,12 @@ class buildset: if self.bset.endswith('.cfg'): configs = [self.bset] else: - exbset = self.macros.expand(self.bset) + exbset = macro_expand(self.macros, self.bset) self.macros['_bset'] = exbset + bset_tmp = build.short_name(exbset) + if bset_tmp.endswith('.bset'): + bset_tmp = bset_tmp[:-5] + self.macros['_bset_tmp'] = bset_tmp root, ext = path.splitext(exbset) if exbset.endswith('.bset'): bset = exbset @@ -336,30 +379,59 @@ class buildset: nesting_count += 1 - if mail: + if self.mail_active(mail, nesting_count): mail['output'].clear() + mail['log'] = '' + mail['reports'] = [] + mail['failure'] = None - log.trace('_bset: %s: make' % (self.bset)) + log.trace('_bset: %2d: %s: make' % (nesting_count, self.bset)) log.notice('Build Set: %s' % (self.bset)) - mail_subject = '%s on %s' % (self.bset, - self.macros.expand('%{_host}')) - current_path = os.environ['PATH'] start = datetime.datetime.now() mail_report = False have_errors = False + interrupted = False - if mail: - mail['output'].clear() + # + # If this is the outter most buildset it's files are installed. Nested + # build sets staged their installed file. The staged files are install + # when the outtter most build finishes. + # + if nesting_count != 1: + if self.installing(): + self.macros['install_mode'] = 'staging' + + # + # Only the outter build set can have staging to install. Get the staging + # root via the config because it could require a valid config. + # + have_staging = False try: configs = self.load() - log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs))) + log.trace('_bset: %2d: %s: configs: %s' % (nesting_count, + self.bset, ', '.join(configs))) + + if nesting_count == 1 and len(configs) > 1: + # + # Prepend staging areas, bin directory to the + # path. Lets the later package depend on the earlier + # ones. + # + pathprepend = ['%{stagingroot}/bin'] + \ + macro_expand(self.macros, '%{_pathprepend}').split(':') + pathprepend = [pp for pp in pathprepend if len(pp)] + if len(pathprepend) == 1: + self.macros['_pathprepend'] = pathprepend[0] + else: + self.macros['_pathprepend'] = ':'.join(pathprepend) + sizes_valid = False builds = [] for s in range(0, len(configs)): b = None @@ -372,14 +444,20 @@ class buildset: opts = copy.copy(self.opts) macros = copy.copy(self.macros) if configs[s].endswith('.bset'): - log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75)) + log.trace('_bset: %2d: %s %s' % (nesting_count, + configs[s], + '=' * (74 - len(configs[s])))) bs = buildset(configs[s], self.configs, opts, macros) bs.build(deps, nesting_count, mail) + if self.installing(): + have_staging = True del bs elif configs[s].endswith('.cfg'): if mail: mail_report = True - log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75)) + log.trace('_bset: %2d: %s %s' % (nesting_count, + configs[s], + '=' * (74 - len(configs[s])))) try: b = build.build(configs[s], self.opts.get_arg('--pkg-tar-files'), @@ -410,8 +488,8 @@ class buildset: # # Dump post build macros. # - log.trace('_bset: macros post-build') - log.trace(str(macros)) + log.trace('_bset: : macros post-build') + log.trace(str(b.macros)) else: raise error.general('invalid config type: %s' % (configs[s])) except error.general as gerr: @@ -434,65 +512,151 @@ class buildset: else: raise # - # Installing ... + # Installing or staging ... # - log.trace('_bset: installing: deps:%r no-install:%r' % \ - (deps is None, self.opts.no_install())) - if deps is None \ - and not self.opts.no_install() \ - and not have_errors: + log.trace('_bset: %2d: %s: deps:%r no-install:%r' % \ + (nesting_count, self.install_mode(), + deps is None, self.opts.no_install())) + log.trace('_bset: %2d: %s: builds: %s' % \ + (nesting_count, self.install_mode(), + ', '.join([b.name() for b in builds]))) + if deps is None and not self.opts.no_install() and not have_errors: for b in builds: - log.trace('_bset: installing: %r' % b.installable()) + log.trace('_bset: : %s: %r' % (self.install_mode(), + b.installable())) if b.installable(): - self.install(b.name(), - b.config.expand('%{buildroot}'), - b.config.expand('%{_prefix}')) + prefix = b.config.expand('%{_prefix}') + buildroot = path.join(b.config.expand('%{buildroot}'), prefix) + if self.staging(): + prefix = b.config.expand('%{stagingroot}') + self.install(self.install_mode(), b.name(), buildroot, prefix) + # + # Sizes ... + # + if len(builds) > 1: + size_build = 0 + size_installed = 0 + size_build_max = 0 + for b in builds: + s = b.get_build_size() + size_build += s + if s > size_build_max: + size_build_max = s + size_installed += b.get_installed_size() + size_sources = 0 + for p in builds[0].config.expand('%{_sourcedir}').split(':'): + size_sources += path.get_size(p) + size_patches = 0 + for p in builds[0].config.expand('%{_patchdir}').split(':'): + size_patches += path.get_size(p) + size_total = size_sources + size_patches + size_installed + build_max_size_human = build.humanize_number(size_build_max + + size_installed, 'B') + build_total_size_human = build.humanize_number(size_total, 'B') + build_sources_size_human = build.humanize_number(size_sources, 'B') + build_patches_size_human = build.humanize_number(size_patches, 'B') + build_installed_size_human = build.humanize_number(size_installed, 'B') + build_size = 'usage: %s' % (build_max_size_human) + build_size += ' total: %s' % (build_total_size_human) + build_size += ' (sources: %s' % (build_sources_size_human) + build_size += ', patches: %s' % (build_patches_size_human) + build_size += ', installed %s)' % (build_installed_size_human) + sizes_valid = True + # + # Cleaning ... + # if deps is None and \ (not self.opts.no_clean() or self.opts.always_clean()): for b in builds: if not b.disabled(): log.notice('cleaning: %s' % (b.name())) b.cleanup() + # + # Log the build size message + # + if len(builds) > 1: + log.notice('Build Sizes: %s' % (build_size)) + # + # Clear out the builds ... + # for b in builds: del b + + # + # If builds have been staged install into the finaly prefix. + # + if have_staging and not self.opts.no_install() and not have_errors: + stagingroot = macro_expand(self.macros, '%{stagingroot}') + have_stagingroot = path.exists(stagingroot) + log.trace('_bset: %2d: install staging, present: %s' % \ + (nesting_count, have_stagingroot)) + if have_stagingroot: + prefix = macro_expand(self.macros, '%{_prefix}') + self.install(self.install_mode(), self.bset, stagingroot, prefix) + staging_size = path.get_size(stagingroot) + if not self.opts.no_clean() or self.opts.always_clean(): + log.notice('clean staging: %s' % (self.bset)) + log.trace('removing: %s' % (stagingroot)) + if not self.opts.dry_run(): + if path.exists(stagingroot): + path.removeall(stagingroot) + log.notice('Staging Size: %s' % \ + (build.humanize_number(staging_size, 'B'))) except error.general as gerr: if not build_error: log.stderr(str(gerr)) raise except KeyboardInterrupt: - mail_report = False + interrupted = True raise except: self.build_failure = 'RSB general failure' + interrupted = True raise finally: end = datetime.datetime.now() os.environ['PATH'] = current_path build_time = str(end - start) - if mail_report and not self.macros.defined('mail_disable'): - self.write_mail_header('Build Time: %s' % (build_time), True) - self.write_mail_header('', True) + if self.mail_single_report() and nesting_count == 1: + mail_report = True + if interrupted or self.macros.defined('mail_disable'): + mail_report = False + if mail_report and mail is not None: + if self.installing(): + self.write_mail_header('Build Time: %s' % (build_time), True) + self.write_mail_header('', True) + self.write_mail_header(mail['header'], True) + self.write_mail_header('') + log.notice('Mailing report: %s' % (mail['to'])) + mail['log'] += self.get_mail_header() + if sizes_valid: + mail['log'] += 'Sizes' + os.linesep + mail['log'] += '=====' + os.linesep + os.linesep + mail['log'] += \ + 'Maximum build usage: ' + build_max_size_human + os.linesep + mail['log'] += \ + 'Total size: ' + build_total_size_human + os.linesep + mail['log'] += \ + 'Installed : ' + build_installed_size_human + os.linesep + mail['log'] += 'Sources: ' + build_sources_size_human + os.linesep + mail['log'] += 'Patches: ' + build_patches_size_human + os.linesep + mail['log'] += os.linesep + mail['log'] += 'Output' + os.linesep + mail['log'] += '======' + os.linesep + os.linesep + mail['log'] += os.linesep.join(mail['output'].get()) + mail['log'] += os.linesep + os.linesep + mail['log'] += 'Report' + os.linesep + mail['log'] += '======' + os.linesep + os.linesep + mail['reports'] += [self.get_mail_report()] if self.build_failure is not None: - mail_subject = 'FAILED %s (%s)' % \ - (mail_subject, self.build_failure) - else: - mail_subject = 'PASSED %s' % (mail_subject) - mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'), - mail_subject) - self.write_mail_header(mail['header'], True) - self.write_mail_header('') - log.notice('Mailing report: %s' % (mail['to'])) - body = self.get_mail_header() - body += 'Output' + os.linesep - body += '======' + os.linesep + os.linesep - body += os.linesep.join(mail['output'].get()) - body += os.linesep + os.linesep - body += 'Report' + os.linesep - body += '======' + os.linesep + os.linesep - body += self.get_mail_report() - if not opts.dry_run(): - mail['mail'].send(mail['to'], mail_subject, body) + mail['failure'] = self.build_failure + if self.mail_active(mail, nesting_count): + try: + self.mail_send(mail) + except error.general as gerr: + log.notice('Mail Send Failure: %s' % (gerr)) + log.notice('Build Set: Time %s' % (build_time)) def list_bset_cfg_files(opts, configs): @@ -517,6 +681,7 @@ def run(): try: optargs = { '--list-configs': 'List available configurations', '--list-bsets': 'List available build sets', + '--list-configs': 'List available configuration files.', '--list-deps': 'List the dependent files.', '--bset-tar-file': 'Create a build set tar file', '--pkg-tar-files': 'Create package tar files', @@ -525,15 +690,18 @@ def run(): mailer.append_options(optargs) opts = options.load(sys.argv, optargs) if opts.get_arg('--mail'): - mail = { 'mail' : mailer.mail(opts), - 'output': log_capture() } + mail = { 'mail' : mailer.mail(opts), + 'output' : log_capture(), + 'log' : '', + 'reports': [], + 'failure': None } to_addr = opts.get_arg('--mail-to') if to_addr is not None: mail['to'] = to_addr[1] else: - mail['to'] = opts.defaults.expand('%{_mail_tools_to}') + mail['to'] = macro_expand(opts.defaults, '%{_mail_tools_to}') mail['from'] = mail['mail'].from_address() - log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str())) + log.notice('RTEMS Source Builder - Set Builder, %s' % (version.string())) opts.log_info() if not check.host_setup(opts): raise error.general('host build environment is not set up correctly') @@ -551,7 +719,7 @@ def run(): else: deps = None if not list_bset_cfg_files(opts, configs): - prefix = opts.defaults.expand('%{_prefix}') + prefix = macro_expand(opts.defaults, '%{_prefix}') if opts.canadian_cross(): opts.disable_install() |