bsb-builder: Add email support, and config report types.

3 files changed, 229 insertions, 61 deletions

diff --git a/rtemstoolkit/mailer.py b/rtemstoolkit/mailer.py
index 81a5c70..b5a0a1e 100644
--- a/rtemstoolkit/mailer.py
+++ b/rtemstoolkit/mailer.py
@@ -51,16 +51,42 @@ except (ValueError, SystemError):
     import options
     import path
 
+_options = {
+    '--mail'     : 'Send email report or results.',
+    '--smtp-host': 'SMTP host to send via.',
+    '--mail-to'  : 'Email address to send the email too.',
+    '--mail-from': 'Email address the report is from.'
+}
+
 def append_options(opts):
-    opts['--mail'] = 'Send email report or results.'
-    opts['--smtp-host'] = 'SMTP host to send via.'
-    opts['--mail-to'] = 'Email address to send the email too.'
-    opts['--mail-from'] = 'Email address the report is from.'
+    for o in _options:
+        opts[o] = _options[o]
+
+def add_arguments(argsp):
+    argsp.add_argument('--mail', help = _options['--mail'], action = 'store_true')
+    for o in ['--smtp-host', '--mail-to', '--mail-from']:
+        argsp.add_argument(o, help = _options[o], type = str)
 
 class mail:
     def __init__(self, opts):
         self.opts = opts
 
+    def _args_are_macros(self):
+        return type(self.opts) is 'command_line'
+
+    def _get_arg(self, arg):
+        if self._args_are_macros():
+            value = self.opts.find_arg(arg)[1]
+        else:
+            if arg.startswith('--'):
+                arg = arg[2:]
+            arg = arg.replace('-', '_')
+            if arg in vars(self.opts):
+                value = vars(self.opts)[arg]
+            else:
+                value = None
+        return value
+
     def from_address(self):
 
         def _clean(l):
@@ -72,9 +98,9 @@ class mail:
                 l = l[:l.index('\n')]
             return l.strip()
 
-        addr = self.opts.find_arg('--mail-from')
+        addr = self._get_arg('--mail-from')
         if addr is not None:
-            return addr[1]
+            return addr
         mailrc = None
         if 'MAILRC' in os.environ:
             mailrc = os.environ['MAILRC']
@@ -95,14 +121,18 @@ class mail:
                         addr = fa[fa.index('=') + 1:].replace('"', ' ').strip()
             if addr is not None:
                 return addr
-        addr = self.opts.defaults.get_value('%{_sbgit_mail}')
+        if self._args_are_macros():
+            addr = self.opts.defaults.get_value('%{_sbgit_mail}')
+        else:
+            raise error.general('no valid from address for mail')
         return addr
 
     def smtp_host(self):
-        host = self.opts.find_arg('--smtp-host')
+        host = self._get_arg('--smtp-host')
         if host is not None:
             return host[1]
-        host = self.opts.defaults.get_value('%{_mail_smtp_host}')
+        if self._args_are_macros():
+            host = self.opts.defaults.get_value('%{_mail_smtp_host}')
         if host is not None:
             return host
         return 'localhost'
diff --git a/tester/rt/check.py b/tester/rt/check.py
index af1d3e9..7e81d30 100755
--- a/tester/rt/check.py
+++ b/tester/rt/check.py
@@ -49,6 +49,7 @@ from rtemstoolkit import execute
 from rtemstoolkit import error
 from rtemstoolkit import host
 from rtemstoolkit import log
+from rtemstoolkit import mailer
 from rtemstoolkit import path
 from rtemstoolkit import textbox
 from rtemstoolkit import version
@@ -267,7 +268,7 @@ class warnings_errors:
             data[category]['common'] = common
         return data
 
-    def _report_category(self, label, warnings, group_counts):
+    def _report_category(self, label, warnings, group_counts, summary):
         width = 70
         cols_1 = [width]
         cols_2 = [8, width - 8]
@@ -311,15 +312,16 @@ class warnings_errors:
                             d = gs[row:+4]
                         s += textbox.row(cols_4, d, indent = 1)
                 s += textbox.line(cols_2_4, marker = '+', indent = 1)
-                vw = sorted([(w, warnings[build][w]) for w in build_warnings],
-                            key = operator.itemgetter(1),
-                            reverse = True)
-                for w in vw:
-                    c1 = '%6d' % w[1]
-                    for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
-                        s += textbox.row(cols_2, [c1, l], indent = 1)
-                        c1 = ' ' * 6
-                s += textbox.line(cols_2, marker = '+', indent = 1)
+                if not summary:
+                    vw = sorted([(w, warnings[build][w]) for w in build_warnings],
+                                key = operator.itemgetter(1),
+                                reverse = True)
+                    for w in vw:
+                        c1 = '%6d' % w[1]
+                        for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
+                            s += textbox.row(cols_2, [c1, l], indent = 1)
+                            c1 = ' ' * 6
+                    s += textbox.line(cols_2, marker = '+', indent = 1)
         return s
 
     def _report_warning_map(self):
@@ -341,9 +343,9 @@ class warnings_errors:
             s += textbox.line(cols_1, marker = '+', indent = 1)
         return s
 
-    def warnings_report(self):
+    def warnings_report(self, summary):
         self.lock.acquire()
-        s = ' No warnings'
+        s = ' No warnings' + os.linesep
         try:
             total = 0
             for build in self.warnings:
@@ -351,16 +353,20 @@ class warnings_errors:
             if total != 0:
                 data = self._analyze(self.warnings, self.groups['exclude'])
                 s = self._report_category('By Architecture (total : %d)' % (total),
-                                          data['arch'], data['groups']['arch'])
+                                          data['arch'], data['groups']['arch'],
+                                          summary)
                 s += os.linesep
                 s += self._report_category('By BSP (total : %d)' % (total),
-                                           data['arch_bsp'], data['groups']['arch_bsp'])
+                                           data['arch_bsp'], data['groups']['arch_bsp'],
+                                           summary)
                 s += os.linesep
                 s += self._report_category('By Build (total : %d)' % (total),
-                                           data['build'], data['groups']['build'])
-                s += os.linesep
-                s += self._report_warning_map()
+                                           data['build'], data['groups']['build'],
+                                           summary)
                 s += os.linesep
+                if not summary:
+                    s += self._report_warning_map()
+                    s += os.linesep
         finally:
             self.lock.release()
         return s
@@ -631,11 +637,11 @@ class results:
                 if build_fails > 0:
                     s += bs + os.linesep
         if count == 0:
-            s = ' No failures'
+            s = ' No failures' + os.linesep
         return s
 
-    def warnings_report(self):
-        return self.warnings_errors.warnings_report()
+    def warnings_report(self, summary = False):
+        return self.warnings_errors.warnings_report(summary)
 
     def report(self):
         self.lock.acquire()
@@ -923,7 +929,7 @@ class configuration_:
         for p in self.profiles['profiles']:
             profile = {}
             profile['name'] = p
-            profile['archs'] = self.config.comma_list(profile['name'], 'archs')
+            profile['archs'] = self.config.comma_list(profile['name'], 'archs', err = False)
             archs += profile['archs']
             for arch in profile['archs']:
                 bsps = 'bsps_%s' % (arch)
@@ -1281,6 +1287,17 @@ class builder:
                 f.write(command_line() + os.linesep)
                 f.write(self.results.warnings_errors.report())
 
+    def _failures_report(self):
+        if self.options['failures-report'] is not None:
+            with open(self.options['failures-report'], 'w') as f:
+                f.write(title() + os.linesep)
+                f.write(os.linesep)
+                f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
+                                        os.linesep))
+                f.write(os.linesep)
+                f.write(command_line() + os.linesep)
+                f.write(self.results.failures_report())
+
     def _finished(self):
         log.notice('Total: Warnings:%d  exes:%d  objs:%d  libs:%d' % \
                    (self.results.get_warning_count(), self.counts['exes'],
@@ -1292,19 +1309,23 @@ class builder:
         log.notice('Failures:')
         log.notice(self.results.failures_report())
         self._warnings_report()
+        self._failures_report()
 
     def run_jobs(self, jobs):
         if path.exists(self.build_dir):
             log.notice('Cleaning: %s' % (self.build_dir))
             path.removeall(self.build_dir)
-        start = datetime.datetime.now()
+        self.start = datetime.datetime.now()
+        self.end = datetime.datetime.now()
+        self.duration = self.end - self.start
+        self.average = self.duration
         env_path = os.environ['PATH']
         os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
                              os.pathsep + os.environ['PATH']
         job_count, build_job_count = jobs_option_parse(self.options['jobs'])
         builds = self._create_build_jobs(jobs, build_job_count)
         active_jobs = []
-        jobs_completed = 0
+        self.jobs_completed = 0
         try:
             while len(builds) > 0 or len(active_jobs) > 0:
                 new_jobs = job_count - len(active_jobs)
@@ -1323,7 +1344,7 @@ class builder:
                     job.log_output()
                     job.clean()
                     active_jobs.remove(job)
-                    jobs_completed += 1
+                    self.jobs_completed += 1
                 time.sleep(0.250)
         except:
             for job in active_jobs:
@@ -1332,15 +1353,15 @@ class builder:
                 except:
                     pass
             raise
-        end = datetime.datetime.now()
+        self.end = datetime.datetime.now()
         os.environ['PATH'] = env_path
-        duration = end - start
-        if jobs_completed == 0:
-            jobs_completed = 1
+        self.duration = self.end - self.start
+        if self.jobs_completed == 0:
+            self.jobs_completed = 1
         self._finished()
-        log.notice('Average BSP Build Time: %s' % \
-                   (str(duration / jobs_completed)))
-        log.notice('Total Time %s' % (str(duration)))
+        self.average = self.duration / self.jobs_completed
+        log.notice('Average BSP Build Time: %s' % (str(self.average)))
+        log.notice('Total Time %s' % (str(self.duration)))
 
     def arch_bsp_jobs(self, arch, bsps):
         jobs = []
@@ -1401,6 +1422,7 @@ def run_args(args):
             if 'msys' in cspath[0] and cspath[0].endswith('bin'):
                 os.environ['PATH'] = os.pathsep.join(cspath[1:])
 
+        start = datetime.datetime.now()
         top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
         prefix = '/opt/rtems/%s' % (rtems_version())
         tools = prefix
@@ -1423,9 +1445,12 @@ def run_args(args):
                            type = str)
         argsp.add_argument('--log', help = 'Log file.', type = str)
         argsp.add_argument('--config-report', help = 'Report the configuration.',
-                           action = 'store_true')
+                           type = str, default = None,
+                           choices = ['all', 'profiles', 'builds', 'archs'])
         argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
                            type = str, default = None)
+        argsp.add_argument('--failures-report', help = 'Report the failures to a file.',
+                           type = str, default = None)
         argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
                            action = 'store_true')
         argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
@@ -1442,20 +1467,48 @@ def run_args(args):
                            type = str, default = '1/%d' % (host.cpus()))
         argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
                            action = 'store_true')
+        mailer.add_arguments(argsp)
 
         opts = argsp.parse_args(args[1:])
+        mail = None
+        if opts.mail:
+            mail = mailer.mail(opts)
+            # Request these now to generate any errors.
+            from_addr = mail.from_address()
+            smtp_host = mail.smtp_host()
+            if 'mail_to' in opts and opts.mail_to is not None:
+                to_addr = opts.mail_to
+            else:
+                to_addr = 'build@rtems.org'
         if opts.log is not None:
             logf = opts.log
         log.default = log.log([logf])
         log.notice(title())
         log.output(command_line())
+        if mail:
+            log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr,
+                                                        to_addr,
+                                                        smtp_host))
 
         config = configuration_()
         config.load(config_file, opts.build)
 
         if opts.config_report:
-            log.notice('Configuration Report:')
-            log.notice(config.report())
+            log.notice('Configuration Report: %s' % (opts.config_report))
+            c_profiles = False
+            c_builds = False
+            c_archs = False
+            if opts.config_report == 'all':
+                c_profiles = True
+                c_builds = True
+                c_archs = True
+            elif opts.config_report == 'profiles':
+                c_profiles = True
+            elif opts.config_report == 'builds':
+                c_builds = True
+            elif opts.config_report == 'archs':
+                c_archs = True
+            log.notice(config.report(c_profiles, c_builds, c_archs))
             sys.exit(0)
 
         if opts.rtems is None:
@@ -1471,7 +1524,8 @@ def run_args(args):
                     'no-clean'        : opts.no_clean,
                     'dry-run'         : opts.dry_run,
                     'jobs'            : opts.jobs,
-                    'warnings-report' : opts.warnings_report }
+                    'warnings-report' : opts.warnings_report,
+                    'failures-report' : opts.failures_report }
 
         b = builder(config, rtems_version(), prefix, tools,
                     path.shell(opts.rtems), build_dir, options)
@@ -1485,12 +1539,49 @@ def run_args(args):
         #
         if bsps is not None:
             if archs is not None:
-                raise error.general('--arch supplied with --bsp; use --bsp=arch/bsp,arch/bsp,..')
+                raise error.general('--arch supplied with --bsp;' \
+                                    ' use --bsp=arch/bsp,arch/bsp,..')
+            what = 'BSPs: %s' % (' '.join(bsps))
             b.build_bsps(bsps)
         elif archs is not None:
+            what = 'Archs: %s' % (' '.join(archs))
             b.build_archs(archs)
         else:
+            what = 'Profile(s): %s' % (' '.join(profiles))
             b.build_profiles(profiles)
+        end = datetime.datetime.now()
+
+        #
+        # Email the results of the build.
+        #
+        if mail is not None:
+            subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0],
+                                                      what)
+            t = title()
+            body = t + os.linesep
+            body += '=' * len(t) + os.linesep
+            body += os.linesep
+            body += 'Host: %s' % (os.uname()[3]) + os.linesep
+            body += os.linesep
+            body += command_line()
+            body += os.linesep
+            body += 'Total Time            : %s for %d completed job(s)' % \
+                    (str(b.duration), b.jobs_completed)
+            body += os.linesep
+            body += 'Average BSP Build Time: %s' % (str(b.average))
+            body += os.linesep + os.linesep
+            body += 'Builds' + os.linesep
+            body += '======' + os.linesep
+            body += os.linesep.join([' ' + cb for cb in config.builds()])
+            body += os.linesep + os.linesep
+            body += 'Failures Report' + os.linesep
+            body += '===============' + os.linesep
+            body += b.results.failures_report()
+            body += os.linesep
+            body += 'Warnings Report' + os.linesep
+            body += '===============' + os.linesep
+            body += b.results.warnings_report(summary = True)
+            mail.send(to_addr, subject, body)
 
     except error.general as gerr:
         print(gerr)
diff --git a/tester/rtems/rtems-bsps-tiers.ini b/tester/rtems/rtems-bsps-tiers.ini
index c6bba14..cc611cb 100644
--- a/tester/rtems/rtems-bsps-tiers.ini
+++ b/tester/rtems/rtems-bsps-tiers.ini
@@ -21,36 +21,31 @@
 # Tier 1: no build errors and no unexpected tests failures on hardware.
 #
 [tier-1]
-archs = arm, i386, sparc
-bsps_arm = altcycv_devkit,
-    altcycv_devkit_smp,
-    xilinx_zynq_zc702, xilinx_zynq_zc706, xilinx_zynq_zedboard
+archs = arm, i386
+bsps_arm = beagleboneblack, xilinx_zynq_zedboard
 bsps_i386 = pc686
-bsps_sparc = leon2, leon3
 
 #
 # Tier 2: no build errors and no unexpected tests failures on hardware and
 #         simulators.
 #
 [tier-2]
-archs = arm, sparc
-bsps_arm = lm3s6965_qemu,
-    realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp,
-    xilinx_zynq_a9_qemu
-bsps_sparc = erc32
+
 #
 # Tier 3: no build errors, no tests run.
 #
 [tier-3]
-archs = arm, moxie
-bsps_arm = arm1136jfs,
-    arm1136js, arm7tdmi, arm920, armcortexa9, atsamv,
-    beagleboardorig, beagleboardxm, beagleboneblack, beaglebonewhite,
+archs = arm, bfin, i386, lm32, m32c, m68k, mips, moxie,
+        nios2, or1k, powerpc, sh, sparc, sparc64, v850
+bsps_arm = altcycv_devkit, altcycv_devkit_smp,
+    arm1136jfs, arm1136js, arm7tdmi, arm920, armcortexa9, atsamv,
+    beagleboardorig, beagleboardxm, beaglebonewhite,
     csb336, csb337, csb637,
     edb7312,
     kit637_v6,
     gumstix,
-    lm3s3749, lm3s6965, lm4f120,
+    imx7,
+    lm3s3749, lm3s6965, lm3s6965_qemu, lm4f120,
     lpc1768_mbed, lpc1768_mbed_ahb_ram, lpc1768_mbed_ahb_ram_eth,
     lpc17xx_ea_ram, lpc17xx_ea_rom_int, lpc17xx_plx800_ram,
     lpc17xx_plx800_rom_int, lpc2362, lpc23xx_tli800, lpc24xx_ea,
@@ -59,13 +54,65 @@ bsps_arm = arm1136jfs,
     lpc40xx_ea_rom_int, lpc32xx_mzx, lpc32xx_mzx_stage_1,
     lpc32xx_mzx_stage_2, lpc32xx_phycore,
     raspberrypi, raspberrypi2,
+    realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp,
     rtl22xx, rtl22xx_t,
     smdk2410,
     stm32f105rc, stm32f4,
     tms570ls3137_hdk, tms570ls3137_hdk_intram,
     tms570ls3137_hdk_sdram,
-    tms570ls3137_hdk_with_loader
+    tms570ls3137_hdk_with_loader,
+    xilinx_zynq_a9_qemu, xilinx_zynq_zc702, xilinx_zynq_zc706
+bsps_bfin = TLL6527M, bf537Stamp, eZKit533
+bsps_i386 = pc386, pc486, pc586-sse, pc586, pcp4
+bsps_lm32 = lm32_evr, lm32_evr_gdbsim, milkymist
+bsps_m32c = m32csim
+bsps_m68k = av5282,
+    csb360,
+    gen68340, gen68360, gen68360_040,
+    pgh360,
+    COBRA5475,
+    m5484FireEngine,
+    mcf5206elite,
+    mcf52235, mcf5225x,
+    mcf5235,
+    mcf5329,
+    mrm332,
+    mvme147, mvme147s, mvme162, mvme162lx, mvme167,
+    uC5282
+bsps_mips =  csb350, hurricane, jmr3904, malta, rbtx4925, rbtx4938
 bsps_moxie = moxiesim
+bsps_nios2 = nios2_iss
+bsps_or1k = generic_or1k
+bsps_powerpc = beatnik,
+    br_uid, brs5l, brs6l,
+    dp2,
+    gwlcfm,
+    haleakala,
+    hsc_cm01,
+    icecube,
+    mcp750,
+    mpc5566evb, mpc5566evb_spe, phycore_mpc5554,
+    mpc5643l_dpu, mpc5643l_evb, mpc5668g,
+    mpc5674f_ecu508_app, mpc5674f_ecu508_boot, mpc5674f_rsm6, mpc5674fevb, mpc5674fevb_spe,
+    mpc8260ads,
+    mpc8309som,
+    mpc8313erdb,
+    mpc8349eamds,
+    mtx603e,
+    mvme2100, mvme2307, mvme3100, mvme5500,
+    pghplus,
+    pm520_cr825, pm520_ze30,
+    psim,
+    qemuppc, qemuprep, qemuprep-altivec,
+    qoriq_core_0, qoriq_core_1, qoriq_e500, qoriq_e6500_32, qoriq_e6500_64
+    ss555,
+    t32mppc,
+    tqm8xx_stk8xx,
+    virtex, virtex4, virtex5
+bsps_sh = gensh1, gensh2, gensh4, simsh1, simsh2, simsh2e, simsh4
+bsps_sparc = erc32, leon2, leon3, ngmp
+bsps_sparc64 = niagara, usiii
+bsps_v850 =  v850e1sim, v850e2sim, v850e2v3sim, v850esim, v850essim, v850sim
 
 #
 # Tier 4: nothing expected.