path: root/tester/rt/check.py

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from __future__ import print_function

import argparse
import datetime
import operator
import os
import re
import sys
import textwrap

import pprint

try:
    import configparser
except:
    import ConfigParser as configparser

from rtemstoolkit import execute
from rtemstoolkit import error
from rtemstoolkit import host
from rtemstoolkit import log
from rtemstoolkit import path
from rtemstoolkit import textbox
from rtemstoolkit import version

def rtems_version():
    return version.version()

def wrap(line, lineend = '', indent = 0):
    if type(line) is tuple or type(line) is list:
        if len(line) >= 2:
            s1 = line[0]
        else:
            s1 = ''
        s2 = line[1:]
    elif type(line) is str:
        s1 = ''
        s2 = [line]
    else:
        raise error.internal('line is not a tuple, list or string')
    s = ''
    first = True
    for ss in s2:
        if type(ss) is not str and type(ss) is not unicode:
            raise error.internal('text needs to be a string')
        for l in textwrap.wrap(ss):
            s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
            if first and len(s1) > 0:
                s1 = ' ' * len(s1)
    if lineend != '':
        s = s[:0 - len(os.linesep) - 1] + os.linesep
    return s

def title():
    return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())

def command_line():
    return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')

class warnings_errors:

    def __init__(self, rtems):
        self.rtems = path.host(rtems)
        self.reset()
        self.groups = { 'groups'  : ['Shared', 'BSP', 'Network', 'Tests',
                                     'LibCPU', 'CPU Kit'],
                        'exclude' : '.*Makefile.*',
                        'CPU Kit' : '.*cpukit/.*',
                        'Network' : '.*libnetworking/.*',
                        'Tests'   : '.*testsuites/.*',
                        'BSP'     : '.*libbsp/.*',
                        'LibCPU'  : '.*libcpu/.*',
                        'Shared'  : '.*shared/.*' }
        self.arch = None
        self.bsp = None
        self.build = None

    def _opts(self, arch = None, bsp = None, build = None):
        if arch is None:
            arch = self.arch
        if bsp is None:
            bsp = self.bsp
        if build is None:
            build = self.build
        return arch, bsp, build

    def _key(self, arch, bsp, build):
        arch, bsp, build = self._opts(arch, bsp, build)
        return '%s/%s-%s' % (arch, bsp, build)

    def _get_warnings(self, arch = None, bsp = None, build = None):
        arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
        if arch is None:
            arch = '.*'
        if bsp is None:
            bsp = '.*'
        if build is None:
            build = '.*'
        selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
        warnings = [w for w in self.warnings if selector.match(w)]
        return sorted(warnings)

    def _total(self, archive):
        total = 0
        for a in archive:
            total += archive[a]
        return total

    def _analyze(self, warnings, exclude):
        def _group(data, category, name, warning, count, groups, group_regx):
            if 'groups' not in data:
                data['groups'] = { }
            if category not in data['groups']:
                data['groups'][category] = { 'totals' : { } }
            if name not in data['groups'][category]:
                data['groups'][category][name] = { }
            for group in groups:
                if group not in data['groups'][category]['totals']:
                    data['groups'][category]['totals'][group] = 0
                if group not in data['groups'][category][name]:
                    data['groups'][category][name][group] = 0
                if group_regx[group].match(warning):
                    data['groups'][category][name][group] += count
                    data['groups'][category]['totals'][group] += count
                    break

        def _update(data, category, name, warning, count, groups, group_regx):
            if category not in data:
                data[category] = { }
            if name not in data[category]:
                data[category][name] = { }
            if warning not in data[category][name]:
                data[category][name][warning] = 0
            data[category][name][warning] += count
            _group(data, category, name,  w, count, groups, group_regx)

        data = { }
        group_regx = { }
        for group in self.groups['groups']:
            group_regx[group] = re.compile(self.groups[group])
        exclude_regx = re.compile(exclude)
        for warning in warnings:
            arch = warning.split('/', 1)[0]
            arch_bsp = warning.split('-', 1)[0]
            build = warning.split('-', 1)[1]
            for w in self.warnings[warning]:
                if not exclude_regx.match(w):
                    count = self.warnings[warning][w]
                    _update(data, 'arch',     arch,     w, count,
                           self.groups['groups'], group_regx)
                    _update(data, 'arch_bsp', arch_bsp, w, count,
                           self.groups['groups'], group_regx)
                    _update(data, 'build',  build,  w, count,
                           self.groups['groups'], group_regx)
        for category in ['arch', 'arch_bsp', 'build']:
            common = {}
            for name in data[category]:
                for w in data[category][name]:
                    if w not in common:
                        for other in [n for n in data[category] if n != name]:
                            if w in data[category][other]:
                                common[w] = data[category][name][w]
                                _group(data, category, 'common', w, common[w],
                                       self.groups['groups'], group_regx)
            data[category]['common'] = common
        return data

    def _report_category(self, label, warnings, group_counts):
        width = 70
        cols_1 = [width]
        cols_2 = [8, width - 8]
        cols_4 = textbox.even_columns(4, width)
        cols_2_4 = textbox.merge_columns([cols_2, cols_4])
        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
        s += textbox.row(cols_1, [' ' + label], indent = 1)
        s += textbox.line(cols_1, marker = '+', indent = 1)
        builds = ['common'] + sorted([b for b in warnings if b != 'common'])
        common = warnings['common']
        for build in builds:
            build_warnings = warnings[build]
            if build is not 'common':
                build_warnings = [w for w in build_warnings if w not in common]
            s += textbox.row(cols_1,
                             [' %s : %d warning(s)' % (build,
                                                       len(build_warnings))],
                             indent = 1)
            if len(build_warnings) == 0:
                s += textbox.line(cols_1, marker = '+', indent = 1)
            else:
                s += textbox.line(cols_4, marker = '+', indent = 1)
                if build not in group_counts:
                    gs = [0 for group in self.groups['groups']]
                else:
                    gs = []
                    for g in range(0, len(self.groups['groups'])):
                        group = self.groups['groups'][g]
                        gs += ['%*s' % (cols_4[g % 4] - 2,
                                        '%s : %4d' % \
                                        (group,
                                         group_counts[build][group]))]
                    for row in range(0, len(self.groups['groups']), 4):
                        if row + 4 > len(self.groups['groups']):
                            d = gs[row:] + \
                                ['' for r in range(row,
                                                   len(self.groups['groups']))]
                        else:
                            d = gs[row:+4]
                        s += textbox.row(cols_4, d, indent = 1)
                s += textbox.line(cols_2_4, marker = '+', indent = 1)
                vw = sorted([(w, warnings[build][w]) for w in build_warnings],
                            key = operator.itemgetter(1),
                            reverse = True)
                for w in vw:
                    c1 = '%6d' % w[1]
                    for l in textwrap.wrap(' ' + w[0]):
                        s += textbox.row(cols_2, [c1, l], indent = 1)
                        c1 = ' ' * 6
                s += textbox.line(cols_2, marker = '+', indent = 1)
        return s

    def _report_warning_map(self):
        builds = self.messages['warnings']
        width = 70
        cols_1 = [width]
        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
        s += textbox.row(cols_1, [' Warning Map'], indent = 1)
        s += textbox.line(cols_1, marker = '+', indent = 1)
        for build in builds:
            messages = builds[build]
            s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
            s += textbox.line(cols_1, marker = '+', indent = 1)
            for msg in messages:
                for l in textwrap.wrap(msg, width = width - 3):
                    s += textbox.row(cols_1, [' ' + l], indent = 1)
                for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
                    s += textbox.row(cols_1, ['    ' + l], indent = 1)
            s += textbox.line(cols_1, marker = '+', indent = 1)
        return s

    def report(self):
        arch, bsp, build = self._opts()
        warnings = self._get_warnings(arch, bsp, build)
        total = 0
        for build in warnings:
            total += self._total(self.warnings[build])
        if total == 0:
            s = ' No warnings'
        else:
            data = self._analyze(warnings, self.groups['exclude'])
            s = self._report_category('By Architecture (total : %d)' % (total),
                                      data['arch'], data['groups']['arch'])
            s += os.linesep
            s += self._report_category('By BSP (total : %d)' % (total),
                                       data['arch_bsp'], data['groups']['arch_bsp'])
            s += os.linesep
            s += self._report_category('By Build (total : %d)' % (total),
                                       data['build'], data['groups']['build'])
            s += os.linesep
            s += self._report_warning_map()
            s += os.linesep

        return s

    def set_build(self, arch, bsp, build):
        self.arch = arch
        self.bsp = bsp
        self.build = build
        self.build_key = '%s/%s-%s' % (arch, bsp, build)
        if self.build_key not in self.warnings:
            self.warnings[self.build_key] = {}
        if self.build_key not in self.errors:
            self.errors[self.build_key] = {}

    def clear_build(self):
        self.arch = None
        self.bsp = None
        self.build = None
        self.build_key = None

    def get_warning_count(self):
        return self.warning_count

    def get_error_count(self):
        return self.error_count

    def reset(self):
        self.warnings = { }
        self.warning_count = 0
        self.errors = { }
        self.error_count = 0
        self.messages = { 'warnings' : { }, 'errors' : { } }

    def get_warning_messages(self, arch = None, bsp = None, build = None):
        messages = self.messages['warnings'][self._key(arch, bsp, build)]
        return ['%s %s' % (m, messages[m]) for m in messages]

    def get_error_messages(self, arch = None, bsp = None, build = None):
        messages = self.messages['errors'][self._key(arch, bsp, build)]
        return ['%s %s' % (m, messages[m]) for m in messages]

    def output(self, text):
        def _line_split(line, source_base):
            ls = line.split(' ', 1)
            fname = ls[0].split(':')
            #
            # Ignore compiler option warnings.
            #
            if len(fname) < 4:
                return None
            p = path.abspath(fname[0])
            p = p.replace(source_base, '')
            if path.isabspath(p):
                p = p[1:]
            return p, fname[1], fname[2], ls[1]

        if self.build_key is not None and \
           (' warning:' in text or ' error:' in text):
            for l in text.splitlines():
                if ' warning:' in l:
                    self.warning_count += 1
                    archive = self.warnings[self.build_key]
                    messages = 'warnings'
                elif ' error:' in l:
                    self.error_count += 1
                    archive = self.errors[self.build_key]
                    messages = 'errors'
                else:
                    continue
                line_parts = _line_split(l, self.rtems)
                if line_parts is not None:
                    src, line, pos, msg = line_parts
                    where = '%s:%s:%s' % (src, line, pos)
                    if where not in archive:
                        archive[where] = 1
                    else:
                        archive[where] += 1
                    if self.build_key not in self.messages[messages]:
                        self.messages[messages][self.build_key] = { }
                    self.messages[messages][self.build_key][where] = msg

        log.output(text)

class results:

    def __init__(self):
        self.passes = []
        self.fails = []

    def _arch_bsp(self, arch, bsp):
        return '%s/%s' % (arch, bsp)

    def add(self, good, arch, bsp, configure, warnings, error_messages):
        if good:
            self.passes += [(arch, bsp, configure, warnings, None)]
        else:
            self.fails += [(arch, bsp, configure, warnings, error_messages)]

    def report(self):
        log.notice('* Passes: %d   Failures: %d' %
                   (len(self.passes), len(self.fails)))
        log.output()
        log.output('Build Report')
        log.output('   Passes: %d   Failures: %d' %
                   (len(self.passes), len(self.fails)))
        log.output(' Failures:')
        if len(self.fails) == 0:
            log.output('  None')
        else:
            max_col = 0
            for f in self.fails:
                arch_bsp = self._arch_bsp(f[0], f[1])
                if len(arch_bsp) > max_col:
                    max_col = len(arch_bsp)
            for f in self.fails:
                config_cmd = f[2]
                config_at = config_cmd.find('configure')
                if config_at != -1:
                    config_cmd = config_cmd[config_at:]
                s1 = ' %*s:  ' % (max_col + 2, self._arch_bsp(f[0], f[1]))
                log.output(wrap([s1, config_cmd], lineend = '\\'))
                if f[4] is not None:
                    s1 = ' ' * len(s1)
                    for msg in f[4]:
                        log.output(wrap([s1, msg], lineend = '\\'))
        log.output(' Passes:')
        if len(self.passes) == 0:
            log.output('  None')
        else:
            max_col = 0
            for f in self.passes:
                arch_bsp = self._arch_bsp(f[0], f[1])
                if len(arch_bsp) > max_col:
                    max_col = len(arch_bsp)
            for f in self.passes:
                config_cmd = f[2]
                config_at = config_cmd.find('configure')
                if config_at != -1:
                    config_cmd = config_cmd[config_at:]
                log.output(wrap((' %*s:  %5d  ' % (max_col + 2,
                                                   self._arch_bsp(f[0], f[1]),
                                                   f[3]),
                                 config_cmd),
                                lineend = '\\'))

class configuration:

    def __init__(self):
        self.config = configparser.ConfigParser()
        self.name = None
        self.archs = { }
        self.builds_ = { }
        self.profiles = { }
        self.configurations = { }

    def __str__(self):
        import pprint
        s = self.name + os.linesep
        s += 'Archs:' + os.linesep + \
             pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
        s += 'Builds:' + os.linesep + \
             pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
        s += 'Profiles:' + os.linesep + \
             pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
        return s

    def _get_item(self, section, label, err = True):
        try:
            rec = self.config.get(section, label).replace(os.linesep, ' ')
            return rec
        except:
            if err:
                raise error.general('config: no "%s" found in "%s"' % (label, section))
        return None

    def _get_items(self, section, err = True):
        try:
            items = [(name, key.replace(os.linesep, ' ')) for name, key in self.config.items(section)]
            return items
        except:
            if err:
                raise error.general('config: section "%s" not found' % (section))
        return []

    def _comma_list(self, section, label, error = True):
        items = self._get_item(section, label, error)
        if items is None:
            return []
        return sorted(set([a.strip() for a in items.split(',')]))

    def _get_item_names(self, section, err = True):
        try:
            return [item[0] for item in self.config.items(section)]
        except:
            if err:
                raise error.general('config: section "%s" not found' % (section))
        return []

    def _build_options(self, build, nesting = 0):
        if ':' in build:
            section, name = build.split(':', 1)
            opts = [self._get_item(section, name)]
            return opts
        builds = self.builds_['builds']
        if build not in builds:
            raise error.general('build %s not found' % (build))
        if nesting > 20:
            raise error.general('nesting build %s' % (build))
        options = []
        for option in self.builds_['builds'][build]:
            if ':' in option:
                section, name = option.split(':', 1)
                opts = [self._get_item(section, name)]
            else:
                opts = self._options(option, nesting + 1)
            for opt in opts:
                if opt not in options:
                    options += [opt]
        return options

    def load(self, name, build):
        if not path.exists(name):
            raise error.general('config: cannot read configuration: %s' % (name))
        self.name = name
        try:
            self.config.read(name)
        except configparser.ParsingError as ce:
            raise error.general('config: %s' % (ce))
        archs = []
        self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
        if len(self.profiles['profiles']) == 0:
            self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
        for p in self.profiles['profiles']:
            profile = {}
            profile['name'] = p
            profile['archs'] = self._comma_list(profile['name'], 'archs')
            archs += profile['archs']
            for arch in profile['archs']:
                bsps = 'bsps_%s' % (arch)
                profile[bsps] = self._comma_list(profile['name'], bsps)
            self.profiles[profile['name']] = profile
        for a in set(archs):
            arch = {}
            arch['excludes'] = {}
            for exclude in self._comma_list(a, 'exclude', error = False):
                arch['excludes'][exclude] = ['all']
            for i in self._get_items(a, False):
                if i[0].startswith('exclude_'):
                    exclude = i[0][len('exclude_'):]
                    if exclude not in arch['excludes']:
                        arch['excludes'][exclude] = []
                    arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
            arch['bsps'] = self._comma_list(a, 'bsps', error = False)
            for b in arch['bsps']:
                arch[b] = {}
                arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
            self.archs[a] = arch
        builds = {}
        builds['default'] = self._get_item('builds', 'default')
        if build is None:
            build = builds['default']
        builds['config'] = { }
        for config in self._get_items('config'):
            builds['config'][config[0]] = config[1]
        builds['build'] = build
        builds_ = self._get_item_names('builds')
        builds['builds'] = {}
        for build in builds_:
            build_builds = self._comma_list('builds', build)
            has_config = False
            has_build = False
            for b in build_builds:
                if ':' in b:
                    if has_build:
                        raise error.general('config and build in build: %s' % (build))
                    has_config = True
                else:
                    if has_config:
                        raise error.general('config and build in build: %s' % (build))
                    has_build = True
            builds['builds'][build] = build_builds
        self.builds_ = builds

    def build(self):
        return self.builds_['build']

    def builds(self):
        if self.builds_['build'] in self.builds_['builds']:
            build = self.builds_['builds'][self.builds_['build']]
            if ':' in build[0]:
                return [self.builds_['build']]
            return build
        return None

    def build_options(self, build):
        return ' '.join(self._build_options(build))

    def excludes(self, arch):
        excludes = self.archs[arch]['excludes'].keys()
        for exclude in self.archs[arch]['excludes']:
            if 'all' not in self.archs[arch]['excludes'][exclude]:
                excludes.remove(exclude)
        return sorted(excludes)

    def archs(self):
        return sorted(self.archs.keys())

    def arch_present(self, arch):
        return arch in self.archs

    def arch_bsps(self, arch):
        return sorted(self.archs[arch]['bsps'])

    def bsp_present(self, arch, bsp):
        return bsp in self.archs[arch]['bsps']

    def bsp_excludes(self, arch, bsp):
        excludes = self.archs[arch]['excludes'].keys()
        for exclude in self.archs[arch]['excludes']:
            if bsp not in self.archs[arch]['excludes'][exclude]:
                excludes.remove(exclude)
        return sorted(excludes)

    def bspopts(self, arch, bsp):
        return self.archs[arch][bsp]['bspopts']

    def profile_present(self, profile):
        return profile in self.profiles

    def profile_archs(self, profile):
        return self.profiles[profile]['archs']

    def profile_arch_bsps(self, profile, arch):
        return self.profiles[profile]['bsps_%s' % (arch)]

    def report(self, profiles = True, builds = True, architectures = True):
        width = 70
        cols_1 = [width]
        cols_2 = [10, width - 10]
        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
        s1 = ' File'
        colon = ':'
        for l in textwrap.wrap(self.name, width = cols_2[1] - 3):
            s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
            colon = ' '
            s1 = ' ' * len(s1)
        s += textbox.line(cols_1, marker = '+', indent = 1)
        s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
        if profiles:
            profiles = sorted(self.profiles['profiles'])
            bsps = 0
            for profile in profiles:
                archs = sorted(self.profiles[profile]['archs'])
                for arch in archs:
                    bsps += len(self.profiles[profile]['bsps_%s' % (arch)])
            s += textbox.row(cols_1,
                             [' Profiles : %d/%d' % (len(archs), bsps)],
                             indent = 1)
            for profile in profiles:
                textbox.row(cols_2,
                            [profile, self.profiles[profile]['name']],
                            indent = 1)
            s += textbox.line(cols_1, marker = '+', indent = 1)
            for profile in profiles:
                s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
                profile = self.profiles[profile]
                archs = sorted(profile['archs'])
                for arch in archs:
                    s += textbox.line(cols_2, marker = '+', indent = 1)
                    s1 = ' ' + arch
                    for l in textwrap.wrap(', '.join(profile['bsps_%s' % (arch)]),
                                           width = cols_2[1] - 2):
                        s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
                        s1 = ' ' * len(s1)
                s += textbox.line(cols_2, marker = '+', indent = 1)
        if builds:
            s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
            s += textbox.row(cols_1,
                             [' Builds:  %s (default)' % (self.builds_['default'])],
                             indent = 1)
            builds = self.builds_['builds']
            bsize = 0
            for build in builds:
                if len(build) > bsize:
                    bsize = len(build)
            cols_b = [bsize + 2, width - bsize - 2]
            s += textbox.line(cols_b, marker = '+', indent = 1)
            for build in builds:
                s1 = ' ' + build
                for l in textwrap.wrap(', '.join(builds[build]),
                                       width = cols_b[1] - 2):
                    s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
                    s1 = ' ' * len(s1)
                s += textbox.line(cols_b, marker = '+', indent = 1)
            configs = self.builds_['config']
            s += textbox.row(cols_1,
                             [' Configure Options: %d' % (len(configs))],
                             indent = 1)
            csize = 0
            for config in configs:
                if len(config) > csize:
                    csize = len(config)
            cols_c = [csize + 3, width - csize - 3]
            s += textbox.line(cols_c, marker = '+', indent = 1)
            for config in configs:
                s1 = ' ' + config
                for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
                    s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
                    s1 = ' ' * len(s1)
                s += textbox.line(cols_c, marker = '+', indent = 1)
        if architectures:
            s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
            archs = sorted(self.archs.keys())
            bsps = 0
            asize = 0
            for arch in archs:
                if len(arch) > asize:
                    asize = len(arch)
                bsps += len(self.archs[arch]['bsps'])
            s += textbox.row(cols_1,
                             [' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
                             indent = 1)
            cols_a = [asize + 2, width - asize - 2]
            s += textbox.line(cols_a, marker = '+', indent = 1)
            for arch in archs:
                s += textbox.row(cols_a,
                                 [' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
                                 indent = 1)
            s += textbox.line(cols_a, marker = '+', indent = 1)
            for archn in archs:
                arch = self.archs[archn]
                if len(arch['bsps']) > 0:
                    bsize = 0
                    for bsp in arch['bsps']:
                        if len(bsp) > bsize:
                            bsize = len(bsp)
                    cols_b = [bsize + 3, width - bsize - 3]
                    s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
                    s += textbox.line(cols_b, marker = '+', indent = 1)
                    for bsp in arch['bsps']:
                        s1 = ' ' + bsp
                        bspopts = ' '.join(arch[bsp]['bspopts'])
                        if len(bspopts):
                            for l in textwrap.wrap('bopt: ' + bspopts,
                                                   width = cols_b[1] - 3):
                                s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
                                s1 = ' ' * len(s1)
                        excludes = []
                        for exclude in arch['excludes']:
                            if bsp in arch['excludes'][exclude]:
                                excludes += [exclude]
                        excludes = ', '.join(excludes)
                        if len(excludes):
                            for l in textwrap.wrap('ex: ' + excludes,
                                                   width = cols_b[1] - 3):
                                s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
                                s1 = ' ' * len(s1)
                        if len(bspopts) == 0 and len(excludes) == 0:
                            s += textbox.row(cols_b, [s1, ' '], indent = 1)
                    s += textbox.line(cols_b, marker = '+', indent = 1)
        return s

class build:

    def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
        self.config = config
        self.build_dir = build_dir
        self.rtems_version = version
        self.prefix = prefix
        self.tools = tools
        self.rtems = rtems
        self.options = options
        self.errors = { 'configure': 0,
                        'build':     0,
                        'tests':     0,
                        'fails':     []}
        self.counts = { 'h'        : 0,
                        'exes'     : 0,
                        'objs'     : 0,
                        'libs'     : 0 }
        self.warnings_errors = warnings_errors(rtems)
        self.results = results()
        if not path.exists(path.join(rtems, 'configure')) or \
           not path.exists(path.join(rtems, 'Makefile.in')) or \
           not path.exists(path.join(rtems, 'cpukit')):
            raise error.general('RTEMS source path does not look like RTEMS')

    def _error_str(self):
        return 'Status: configure:%d build:%d' % \
            (self.errors['configure'], self.errors['build'])

    def _path(self, arch, bsp):
        return path.join(self.build_dir, arch, bsp)

    def _archs(self, build_data):
        return sorted(build_data.keys())

    def _bsps(self, arch):
        return self.config.arch_bsps(arch)

    def _build(self):
        return self.config.build()

    def _builds(self, arch, bsp):
        builds = self.config.builds()
        if builds is None:
            return None
        for b in self.config.excludes(arch):
            builds.remove(b)
        for b in self.config.bsp_excludes(arch, bsp):
            builds.remove(b)
        return builds

    def _arch_bsp_dir_make(self, arch, bsp):
        if not path.exists(self._path(arch, bsp)):
            path.mkdir(self._path(arch, bsp))

    def _arch_bsp_dir_clean(self, arch, bsp):
        if path.exists(self._path(arch, bsp)):
            path.removeall(self._path(arch, bsp))

    def _config_command(self, commands, arch, bsp):
        if type(commands) is not list:
            commands = [commands]
        cmd = [path.join(self.rtems, 'configure')]
        commands += self.config.bspopts(arch, bsp)
        for c in commands:
            c = c.replace('@PREFIX@', self.prefix)
            c = c.replace('@RTEMS_VERSION@', self.rtems_version)
            c = c.replace('@ARCH@', arch)
            c = c.replace('@BSP@', bsp)
            cmd += [c]
        return ' '.join(cmd)

    def _build_set(self, builds):
        build_set = { }
        for build in builds:
            build_set[build] = self.config.build_options(build)
        return build_set

    def _build_dir(self, arch, bsp, build):
        return path.join(self._path(arch, bsp), build)

    def _count_files(self, arch, bsp, build):
        counts = { 'h'    : 0,
                   'exes' : 0,
                   'objs' : 0,
                   'libs' : 0 }
        for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
            for file in files:
                if file.endswith('.exe'):
                    counts['exes'] += 1
                elif file.endswith('.o'):
                    counts['objs'] += 1
                elif file.endswith('.a'):
                    counts['libs'] += 1
                elif file.endswith('.h'):
                    counts['h'] += 1
        for f in self.counts:
            if f in counts:
                self.counts[f] += counts[f]
        return counts

    def _have_failures(self, fails):
        return len(fails) != 0

    def _warnings_report(self):
        if self.options['warnings-report'] is not None:
            with open(self.options['warnings-report'], 'w') as f:
                f.write(title() + os.linesep)
                f.write(os.linesep)
                f.write('Date: %s%s' % (datetime.date.today().strftime('%c'), os.linesep))
                f.write(os.linesep)
                f.write(command_line() + os.linesep)
                f.write(self.warnings_errors.report())

    def _finished(self):
        log.notice('+  warnings:%d  exes:%d  objs:%d  libs:%d' % \
                   (self.warnings_errors.get_warning_count(), self.counts['exes'],
                    self.counts['objs'], self.counts['libs']))
        log.output()
        log.output('Warnings:')
        log.output(self.warnings_errors.report())
        log.output()
        log.notice('Failures:')
        log.notice(self.failures_report(self.errors['fails']))
        self._warnings_report()

    def failures_report(self, fails):
        if not self._have_failures(fails):
            return ' No failure(s)'
        absize = 0
        bsize = 0
        ssize = 0
        for f in fails:
            arch_bsp = '%s/%s' % (f[1], f[2])
            if len(arch_bsp) > absize:
                absize = len(arch_bsp)
            if len(f[3]) > bsize:
                bsize = len(f[3])
            if len(f[0]) > ssize:
                ssize = len(f[0])
        fc = 1
        s = ''
        for f in fails:
            fcl = '%3d' % (fc)
            arch_bsp = '%s/%s' % (f[1], f[2])
            state = f[0]
            s1 = '%s %-*s %-*s %-*s ' % (fcl, bsize, f[3], absize, arch_bsp, ssize, state)
            s += wrap((s1, f[4]), lineend = '\\')
            s1 = ' ' * (len(s1) + 2)
            for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
                s += wrap([s1, 'error: ' + e])
            fc += 1
        return s

    def build_arch_bsp(self, arch, bsp):
        if not self.config.bsp_present(arch, bsp):
            raise error.general('BSP not found: %s/%s' % (arch, bsp))
        log.output('-' * 70)
        log.notice('] BSP: %s/%s' % (arch, bsp))
        log.notice('. Creating: %s' % (self._path(arch, bsp)))
        self._arch_bsp_dir_clean(arch, bsp)
        self._arch_bsp_dir_make(arch, bsp)
        builds = self._builds(arch, bsp)
        if builds is None:
            raise error.general('build not found: %s' % (self._build()))
        build_set = self._build_set(builds)
        bsp_start = datetime.datetime.now()
        env_path = os.environ['PATH']
        os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
                             os.pathsep + os.environ['PATH']
        fails = []
        for bs in sorted(build_set.keys()):
            self.warnings_errors.set_build(arch, bsp, bs)
            start = datetime.datetime.now()
            log.output('- ' * 35)
            log.notice('. Configuring: %s' % (bs))
            try:
                warnings = self.warnings_errors.get_warning_count()
                result = '+ Pass'
                bpath = self._build_dir(arch, bsp, bs)
                good = True
                error_messages = None
                path.mkdir(bpath)
                config_cmd = self._config_command(build_set[bs], arch, bsp)
                cmd = config_cmd
                e = execute.capture_execution(log = self.warnings_errors)
                log.output(wrap(('run: ', cmd), lineend = '\\'))
                if self.options['dry-run']:
                    exit_code = 0
                else:
                    exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
                if exit_code != 0:
                    result = '- FAIL'
                    failure = ('configure', arch, bsp, bs, config_cmd)
                    fails += [failure]
                    self.errors['configure'] += 1
                    self.errors['fails'] += [failure]
                    log.notice('- Configure failed: %s' % (bs))
                    log.output('cmd failed: %s' % (cmd))
                    good = False
                else:
                    log.notice('. Building: %s' % (bs))
                    cmd = 'make'
                    if 'jobs' in self.options:
                        cmd += ' -j %s' % (self.options['jobs'])
                    log.output('run: ' + cmd)
                    if self.options['dry-run']:
                        exit_code = 0
                    else:
                        exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
                    if exit_code != 0:
                        error_messages = self.warnings_errors.get_error_messages()
                        result = '- FAIL'
                        failure = ('build', arch, bsp, bs, config_cmd, error_messages)
                        fails += [failure]
                        self.errors['build'] += 1
                        self.errors['fails'] += [failure]
                        log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
                        log.output('cmd failed: %s' % (cmd))
                        good = False
                    files = self._count_files(arch, bsp, bs)
                    log.notice('%s: %s: warnings:%d  exes:%d  objs:%s  libs:%d' % \
                               (result, bs,
                                self.warnings_errors.get_warning_count() - warnings,
                                files['exes'], files['objs'], files['libs']))
                log.notice('  %s' % (self._error_str()))
                self.results.add(good, arch, bsp, config_cmd,
                                 self.warnings_errors.get_warning_count() - warnings,
                                 error_messages)
                if not good and self.options['stop-on-error']:
                    raise error.general('Configuring %s failed' % (bs))
            finally:
                end = datetime.datetime.now()
                if not self.options['no-clean']:
                    log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
                    path.removeall(self._build_dir(arch, bsp, bs))
            log.notice('^ Time %s' % (str(end - start)))
            self.warnings_errors.clear_build()
        bsp_end = datetime.datetime.now()
        log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
        log.output('Failure Report:')
        log.output(self.failures_report(fails))
        os.environ['PATH'] = env_path

    def build_bsp(self, arch, bsp):
        self.build_arch_bsp(arch, bsp)
        self._finished()

    def build_arch(self, arch):
        start = datetime.datetime.now()
        log.output('=' * 70)
        log.notice(']] Architecture: %s' % (arch))
        if not self.config.arch_present(arch):
            raise error.general('Architecture not found: %s' % (arch))
        for bsp in self._bsps(arch):
            self.build_arch_bsp(arch, bsp)
        end = datetime.datetime.now()
        log.notice('^ Architecture Time %s' % (str(end - start)))
        self._finished()

    def build(self):
        for arch in self.config.archs():
            self.build_arch(arch)
        log.notice('^ Profile Time %s' % (str(end - start)))
        self._finished()

    def build_profile(self, profile):
        if not self.config.profile_present(profile):
            raise error.general('Profile not found: %s' % (profile))
        start = datetime.datetime.now()
        log.notice(']] Profile: %s' % (profile))
        for arch in self.config.profile_archs(profile):
            for bsp in self.config.profile_arch_bsps(profile, arch):
                self.build_arch_bsp(arch, bsp)
        end = datetime.datetime.now()
        log.notice('^ Profile Time %s' % (str(end - start)))
        self._finished()

def run_args(args):
    b = None
    ec = 0
    try:
        #
        # On Windows MSYS2 prepends a path to itself to the environment
        # path. This means the RTEMS specific automake is not found and which
        # breaks the bootstrap. We need to remove the prepended path. Also
        # remove any ACLOCAL paths from the environment.
        #
        if os.name == 'nt':
            cspath = os.environ['PATH'].split(os.pathsep)
            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
                os.environ['PATH'] = os.pathsep.join(cspath[1:])

        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
        prefix = '/opt/rtems/%s' % (rtems_version())
        tools = prefix
        build_dir = 'bsp-builds'
        logf = 'bsp-build-%s.txt' % \
               (datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
        config_file = path.join(top, 'share', 'rtems', 'tester',
                                'rtems', 'rtems-bsps.ini')
        if not path.exists(config_file):
            config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')

        argsp = argparse.ArgumentParser()
        argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
                           type = str)
        argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
                           type = str)
        argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
                           type = str)
        argsp.add_argument('--config-report', help = 'Report the configuration.',
                           action = 'store_true')
        argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
                           type = str, default = None)
        argsp.add_argument('--build-path', help = 'Path to build in.',
                           type = str)
        argsp.add_argument('--log', help = 'Log file.', type = str)
        argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
                           action = 'store_true')
        argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
                           action = 'store_true')
        argsp.add_argument('--profiles', help = 'Build the listed profiles.',
                           type = str, default = 'tier-1')
        argsp.add_argument('--build', help = 'Build name to build.',
                           type = str, default='all')
        argsp.add_argument('--arch', help = 'Build the specific architecture.',
                           type = str)
        argsp.add_argument('--bsp', help = 'Build the specific BSP.',
                           type = str)
        argsp.add_argument('--jobs', help = 'Number of jobs to run.',
                           type = int, default = host.cpus())
        argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
                           action = 'store_true')

        opts = argsp.parse_args(args[1:])
        if opts.log is not None:
            logf = opts.log
        log.default = log.log([logf])
        log.notice(title())
        log.output(command_line())
        if opts.rtems is None:
            raise error.general('No RTEMS source provided on the command line')
        if opts.prefix is not None:
            prefix = path.shell(opts.prefix)
        if opts.rtems_tools is not None:
            tools = path.shell(opts.rtems_tools)
        if opts.build_path is not None:
            build_dir = path.shell(opts.build_path)
        if opts.bsp is not None and opts.arch is None:
            raise error.general('BSP provided but no architecture')

        config = configuration()
        config.load(config_file, opts.build)

        if opts.config_report:
            log.notice('Configuration Report:')
            log.notice(config.report())
            sys.exit(0)

        options = { 'stop-on-error'   : opts.stop_on_error,
                    'no-clean'        : opts.no_clean,
                    'dry-run'         : opts.dry_run,
                    'jobs'            : opts.jobs,
                    'warnings-report' : opts.warnings_report }

        b = build(config, rtems_version(), prefix, tools,
                  path.shell(opts.rtems), build_dir, options)
        if opts.arch is not None:
            if opts.bsp is not None:
                b.build_bsp(opts.arch, opts.bsp)
            else:
                b.build_arch(opts.arch)
        else:
            for profile in opts.profiles.split(','):
                b.build_profile(profile.strip())

    except error.general as gerr:
        print(gerr)
        print('BSP Build FAILED', file = sys.stderr)
        ec = 1
    except error.internal as ierr:
        print(ierr)
        print('BSP Build FAILED', file = sys.stderr)
        ec = 1
    except error.exit as eerr:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        ec = 1
    if b is not None:
        b.results.report()
    sys.exit(ec)

def run():
    run_args(sys.argv)

if __name__ == "__main__":
    run()