diff options
Diffstat (limited to 'tester/rt/check.py')
| -rwxr-xr-x | tester/rt/check.py | 496 |
1 files changed, 496 insertions, 0 deletions
diff --git a/tester/rt/check.py b/tester/rt/check.py new file mode 100755 index 0000000..fcd7b55 --- /dev/null +++ b/tester/rt/check.py @@ -0,0 +1,496 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2016 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +from __future__ import print_function + +import argparse +import datetime +import operator +import os +import sys + +from rtemstoolkit import execute +from rtemstoolkit import error +from rtemstoolkit import log +from rtemstoolkit import path +from rtemstoolkit import version + +def rtems_version(): + return version.version() + +class warnings_counter: + + def __init__(self, rtems): + self.rtems = path.host(rtems) + self.reset() + + def report(self): + str = '' + sw = sorted(self.warnings.items(), key = operator.itemgetter(1), reverse = True) + for w in sw: + str += ' %5d %s%s' % (w[1], w[0], os.linesep) + return str + + def accumulate(self, total): + for w in self.warnings: + if w not in total.warnings: + total.warnings[w] = self.warnings[w] + else: + total.warnings[w] += self.warnings[w] + total.count += self.count + + def get(self): + return self.count + + def reset(self): + self.warnings = { } + self.count = 0 + + def output(self, text): + for l in text.splitlines(): + if ' warning:' in l: + self.count += 1 + ws = l.split(' ') + if len(ws) > 0: + ws = ws[0].split(':') + w = path.abspath(ws[0]) + w = w.replace(self.rtems, '') + if path.isabspath(w): + w = w[1:] + w = '%s:%s:%s' % (w, ws[1], ws[2]) + if w not in self.warnings: + self.warnings[w] = 0 + self.warnings[w] += 1 + log.output(text) + +class configuration: + + def __init__(self): + try: + import configparser + except: + import ConfigParser as configparser + self.config = configparser.ConfigParser() + self.name = None + self.archs = { } + self.builds = { } + self.profiles = { } + + def __str__(self): + import pprint + s = self.name + os.linesep + s += 'Archs:' + os.linesep + \ + pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep + s += 'Builds:' + os.linesep + \ + pprint.pformat(self.builds, indent = 1, width = 80) + os.linesep + s += 'Profiles:' + os.linesep + \ + pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep + return s + + def _get_item(self, section, label, err = True): + try: + rec = self.config.get(section, label).replace(os.linesep, ' ') + return rec + except: + if err: + raise error.general('config: no %s found in %s' % (label, section)) + return None + + def _comma_list(self, section, label, error = True): + items = self._get_item(section, label, error) + if items is None: + return [] + return sorted(set([a.strip() for a in items.split(',')])) + + def load(self, name): + if not path.exists(name): + raise error.general('config: cannot read configuration: %s' % (name)) + self.name = name + self.config.read(name) + archs = [] + self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False) + if len(self.profiles['profiles']) == 0: + self.profiles['profiles'] = ['tier_%d' % (t) for t in range(1,4)] + for p in self.profiles['profiles']: + profile = {} + profile['name'] = p + profile['archs'] = self._comma_list(profile['name'], 'archs') + archs += profile['archs'] + for arch in profile['archs']: + bsps = 'bsps_%s' % (arch) + profile[bsps] = self._comma_list(profile['name'], bsps) + self.profiles[profile['name']] = profile + for a in set(archs): + arch = {} + arch['excludes'] = self._comma_list(a, 'excludes', error = False) + arch['bsps'] = self._comma_list(a, 'bsps', error = False) + for b in arch['bsps']: + arch[b] = {} + arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False) + arch[b]['config'] = self._comma_list(a, 'config_%s' % (b), error = False) + self.archs[a] = arch + builds = {} + builds['default'] = self._get_item('builds', 'default').split() + builds['variations'] = self._comma_list('builds', 'variations') + builds['var_options'] = {} + for v in builds['variations']: + builds['var_options'][v] = self._get_item('builds', v).split() + self.builds = builds + + def variations(self): + return self.builds['variations'] + + def excludes(self, arch): + return self.archs[arch]['excludes'] + + def archs(self): + return sorted(self.archs.keys()) + + def arch_present(self, arch): + return arch in self.archs + + def arch_bsps(self, arch): + return sorted(self.archs[arch]['bsps']) + + def bsp_present(self, arch, bsp): + return bsp in self.archs[arch]['bsps'] + + def bspopts(self, arch, bsp): + return self.archs[arch][bsp]['bspopts'] + + def defaults(self): + return self.builds['default'] + + def variant_options(self, variant): + if variant in self.builds['var_options']: + self.builds['var_options'][variant] + return [] + + def profile_present(self, profile): + return profile in self.profiles + + def profile_archs(self, profile): + return self.profiles[profile]['archs'] + + def profile_arch_bsps(self, profile, arch): + return self.profiles[profile]['bsps_%s' % (arch)] + +class build: + + def __init__(self, config, version, prefix, tools, rtems, build_dir, options): + self.config = config + self.build_dir = build_dir + self.rtems_version = version + self.prefix = prefix + self.tools = tools + self.rtems = rtems + self.options = options + self.errors = { 'configure': 0, + 'build': 0, + 'tests': 0 } + self.counts = { 'h' : 0, + 'exes' : 0, + 'objs' : 0, + 'libs' : 0 } + self.warnings = warnings_counter(rtems) + if not path.exists(path.join(rtems, 'configure')) or \ + not path.exists(path.join(rtems, 'Makefile.in')) or \ + not path.exists(path.join(rtems, 'cpukit')): + raise error.general('RTEMS source path does not look like RTEMS') + + def _error_str(self): + return 'Status: configure:%d build:%d' % \ + (self.errors['configure'], self.errors['build']) + + def _path(self, arch, bsp): + return path.join(self.build_dir, arch, bsp) + + def _archs(self, build_data): + return sorted(build_data.keys()) + + def _bsps(self, arch): + return self.config.arch_bsps(arch) + + def _variations(self, arch, bsp): + vars = self.config.variations() + for v in self.config.excludes(arch): + if v in vars: + vars.remove(v) + return vars + + def _arch_bsp_dir_make(self, arch, bsp): + if not path.exists(self._path(arch, bsp)): + path.mkdir(self._path(arch, bsp)) + + def _arch_bsp_dir_clean(self, arch, bsp): + if path.exists(self._path(arch, bsp)): + path.removeall(self._path(arch, bsp)) + + def _config_command(self, commands, arch, bsp): + cmd = [path.join(self.rtems, 'configure')] + commands += self.config.bspopts(arch, bsp) + for c in commands: + c = c.replace('@PREFIX@', self.prefix) + c = c.replace('@RTEMS_VERSION@', self.rtems_version) + c = c.replace('@ARCH@', arch) + c = c.replace('@BSP@', bsp) + cmd += [c] + return ' '.join(cmd) + + def _build_set(self, variations): + build_set = { } + bs = self.config.defaults() + for var in variations: + build_set[var] = bs + self.config.variant_options(var) + return build_set + + def _build_dir(self, arch, bsp, build): + return path.join(self._path(arch, bsp), build) + + def _count_files(self, arch, bsp, build): + counts = { 'h' : 0, + 'exes' : 0, + 'objs' : 0, + 'libs' : 0 } + for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)): + for file in files: + if file.endswith('.exe'): + counts['exes'] += 1 + elif file.endswith('.o'): + counts['objs'] += 1 + elif file.endswith('.a'): + counts['libs'] += 1 + elif file.endswith('.h'): + counts['h'] += 1 + for f in self.counts: + if f in counts: + self.counts[f] = counts[f] + return counts + + def build_arch_bsp(self, arch, bsp): + if not self.config.bsp_present(arch, bsp): + raise error.general('BSP not found: %s/%s' % (arch, bsp)) + log.output('-' * 70) + log.notice('] BSP: %s/%s' % (arch, bsp)) + log.notice('. Creating: %s' % (self._path(arch, bsp))) + self._arch_bsp_dir_clean(arch, bsp) + self._arch_bsp_dir_make(arch, bsp) + variations = self._variations(arch, bsp) + build_set = self._build_set(variations) + bsp_start = datetime.datetime.now() + bsp_warnings = warnings_counter(self.rtems) + env_path = os.environ['PATH'] + os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \ + os.pathsep + os.environ['PATH'] + for bs in sorted(build_set.keys()): + warnings = warnings_counter(self.rtems) + start = datetime.datetime.now() + log.output('- ' * 35) + log.notice('. Configuring: %s' % (bs)) + try: + bpath = self._build_dir(arch, bsp, bs) + path.mkdir(bpath) + cmd = self._config_command(build_set[bs], arch, bsp) + e = execute.capture_execution(log = warnings) + log.output('run: ' + cmd) + if self.options['dry-run']: + exit_code = 0 + else: + exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath)) + if exit_code != 0: + self.errors['configure'] += 1 + log.notice('- Configure failed: %s' % (bs)) + log.output('cmd failed: %s' % (cmd)) + if self.options['stop-on-error']: + raise error.general('Configuring %s failed' % (bs)) + else: + log.notice('. Building: %s' % (bs)) + cmd = 'make' + if 'jobs' in self.options: + cmd += ' -j %s' % (self.options['jobs']) + log.output('run: ' + cmd) + if self.options['dry-run']: + exit_code = 0 + else: + exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath)) + if exit_code != 0: + self.errors['build'] += 1 + log.notice('- FAIL: %s: %s' % (bs, self._error_str())) + log.output('cmd failed: %s' % (cmd)) + if self.options['stop-on-error']: + raise error.general('Building %s failed' % (bs)) + files = self._count_files(arch, bsp, bs) + log.notice('+ Pass: %s: warnings:%d exes:%d objs:%s libs:%d' % \ + (bs, warnings.get(), + files['exes'], files['objs'], files['libs'])) + log.notice(' %s' % (self._error_str())) + finally: + end = datetime.datetime.now() + if not self.options['no-clean']: + log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs))) + path.removeall(self._build_dir(arch, bsp, bs)) + log.notice('^ Time %s' % (str(end - start))) + log.output('Warnings Report:') + log.output(warnings.report()) + warnings.accumulate(bsp_warnings) + warnings.accumulate(self.warnings) + bsp_end = datetime.datetime.now() + log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start))) + log.output('BSP Warnings Report:') + log.output(bsp_warnings.report()) + os.environ['PATH'] = env_path + + def build_arch(self, arch): + start = datetime.datetime.now() + log.output('=' * 70) + log.notice(']] Architecture: %s' % (arch)) + if not self.confif.arch_present(arch): + raise error.general('Architecture not found: %s' % (arch)) + for bsp in self._bsps(arch): + self.build_arch_bsp(arch, bsp) + log.notice('^ Architecture Time %s' % (str(end - start))) + log.notice(' warnings:%d exes:%d objs:%s libs:%d' % \ + self.warnings.get(), self.counts['exes'], + self.counts['objs'], self.counts['libs']) + log.output('Architecture Warnings:') + log.output(self.warnings.report()) + + def build(self): + for arch in self.config.archs(): + self.build_arch(arch) + log.notice('^ Profile Time %s' % (str(end - start))) + log.notice('+ warnings:%d exes:%d objs:%s libs:%d' % \ + self.warnings.get(), self.counts['exes'], + self.counts['objs'], self.counts['libs']) + log.output('Profile Warnings:') + log.output(self.warnings.report()) + + def build_profile(self, profile): + if not self.config.profile_present(profile): + raise error.general('BSP not found: %s/%s' % (arch, bsp)) + start = datetime.datetime.now() + log.notice(']] Profile: %s' % (profile)) + for arch in self.config.profile_archs(profile): + for bsp in self.config.profile_arch_bsps(profile, arch): + self.build_arch_bsp(arch, bsp) + end = datetime.datetime.now() + log.notice('^ Profile Time %s' % (str(end - start))) + log.notice(' warnings:%d exes:%d objs:%d libs:%d' % \ + (self.warnings.get(), self.counts['exes'], + self.counts['objs'], self.counts['libs'])) + log.output('Profile Warnings:') + log.output(self.warnings.report()) + +def run_args(args): + try: + # + # On Windows MSYS2 prepends a path to itself to the environment + # path. This means the RTEMS specific automake is not found and which + # breaks the bootstrap. We need to remove the prepended path. Also + # remove any ACLOCAL paths from the environment. + # + if os.name == 'nt': + cspath = os.environ['PATH'].split(os.pathsep) + if 'msys' in cspath[0] and cspath[0].endswith('bin'): + os.environ['PATH'] = os.pathsep.join(cspath[1:]) + + top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + profile = 'tier-1' + prefix = '/opt/rtems/%s' % (rtems_version()) + tools = prefix + build_dir = 'bsp-builds' + logf = 'bsp-build-%s.txt' % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + config_file = path.join(top, 'share', 'rtems', 'tester', 'rtems', 'rtems-bsps.ini') + + argsp = argparse.ArgumentParser() + argsp.add_argument('--prefix', help = 'Prefix to build the BSP.', type = str) + argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.', type = str) + argsp.add_argument('--rtems', help = 'The RTEMS source tree.', type = str) + argsp.add_argument('--build-path', help = 'Path to build in.', type = str) + argsp.add_argument('--log', help = 'Log file.', type = str) + argsp.add_argument('--stop-on-error', help = 'Stop on an error.', action = 'store_true') + argsp.add_argument('--no-clean', help = 'Do not clean the build output.', action = 'store_true') + argsp.add_argument('--arch', help = 'Build the specific architecture.', type = str) + argsp.add_argument('--bsp', help = 'Build the specific BSP.', type = str) + argsp.add_argument('--dry-run', help = 'Do not run the actual builds.', action = 'store_true') + + opts = argsp.parse_args(args[1:]) + if opts.log is not None: + logf = opts.log + log.default = log.log([logf]) + log.notice('RTEMS Tools Project - RTEMS Kernel Check, %s' % (version.str())) + if opts.rtems is None: + raise error.general('No RTEMS source provided on the command line') + if opts.prefix is not None: + prefix = path.shell(opts.prefix) + if opts.rtems_tools is not None: + tools = path.shell(opts.rtems_tools) + if opts.build_path is not None: + build_dir = path.shell(opts.build_path) + if opts.bsp is not None and opts.arch is None: + raise error.general('BSP provided but no architecture') + + config = configuration() + config.load(config_file) + + options = { 'stop-on-error' : opts.stop_on_error, + 'no-clean' : opts.no_clean, + 'dry-run' : opts.dry_run, + 'jobs' : 8 } + + b = build(config, rtems_version(), prefix, tools, path.shell(opts.rtems), build_dir, options) + if opts.arch is not None: + if opts.bsp is not None: + b.build_arch_bsp(opts.arch, opts.bsp) + else: + b.build_arch(opts.arch) + else: + b.build_profile(profile) + + except error.general as gerr: + print(gerr) + print('BSP Build FAILED', file = sys.stderr) + sys.exit(1) + except error.internal as ierr: + print(ierr) + print('BSP Build FAILED', file = sys.stderr) + sys.exit(1) + except error.exit as eerr: + pass + except KeyboardInterrupt: + log.notice('abort: user terminated') + sys.exit(1) + sys.exit(0) + +def run(): + run_args(sys.argv) + +if __name__ == "__main__": + run() |