diff options
author | Chris Johns <chrisj@rtems.org> | 2017-10-22 18:09:39 +1100 |
---|---|---|
committer | Chris Johns <chrisj@rtems.org> | 2017-10-22 18:56:26 +1100 |
commit | 7051ba5b1958b3926cb69b98d0d9f1db384f754f (patch) | |
tree | 5c35741d19a8d19fcf42c87697a88b611b227f4d | |
parent | tester: Update BeagleBone Black, PC and Zedboard configurations. (diff) | |
download | rtems-tools-7051ba5b1958b3926cb69b98d0d9f1db384f754f.tar.bz2 |
bsb-builder: Add email support, and config report types.
-rw-r--r-- | rtemstoolkit/mailer.py | 48 | ||||
-rwxr-xr-x | tester/rt/check.py | 163 | ||||
-rw-r--r-- | tester/rtems/rtems-bsps-tiers.ini | 79 |
3 files changed, 229 insertions, 61 deletions
diff --git a/rtemstoolkit/mailer.py b/rtemstoolkit/mailer.py index 81a5c70..b5a0a1e 100644 --- a/rtemstoolkit/mailer.py +++ b/rtemstoolkit/mailer.py @@ -51,16 +51,42 @@ except (ValueError, SystemError): import options import path +_options = { + '--mail' : 'Send email report or results.', + '--smtp-host': 'SMTP host to send via.', + '--mail-to' : 'Email address to send the email too.', + '--mail-from': 'Email address the report is from.' +} + def append_options(opts): - opts['--mail'] = 'Send email report or results.' - opts['--smtp-host'] = 'SMTP host to send via.' - opts['--mail-to'] = 'Email address to send the email too.' - opts['--mail-from'] = 'Email address the report is from.' + for o in _options: + opts[o] = _options[o] + +def add_arguments(argsp): + argsp.add_argument('--mail', help = _options['--mail'], action = 'store_true') + for o in ['--smtp-host', '--mail-to', '--mail-from']: + argsp.add_argument(o, help = _options[o], type = str) class mail: def __init__(self, opts): self.opts = opts + def _args_are_macros(self): + return type(self.opts) is 'command_line' + + def _get_arg(self, arg): + if self._args_are_macros(): + value = self.opts.find_arg(arg)[1] + else: + if arg.startswith('--'): + arg = arg[2:] + arg = arg.replace('-', '_') + if arg in vars(self.opts): + value = vars(self.opts)[arg] + else: + value = None + return value + def from_address(self): def _clean(l): @@ -72,9 +98,9 @@ class mail: l = l[:l.index('\n')] return l.strip() - addr = self.opts.find_arg('--mail-from') + addr = self._get_arg('--mail-from') if addr is not None: - return addr[1] + return addr mailrc = None if 'MAILRC' in os.environ: mailrc = os.environ['MAILRC'] @@ -95,14 +121,18 @@ class mail: addr = fa[fa.index('=') + 1:].replace('"', ' ').strip() if addr is not None: return addr - addr = self.opts.defaults.get_value('%{_sbgit_mail}') + if self._args_are_macros(): + addr = self.opts.defaults.get_value('%{_sbgit_mail}') + else: + raise error.general('no valid from address for mail') return addr def smtp_host(self): - host = self.opts.find_arg('--smtp-host') + host = self._get_arg('--smtp-host') if host is not None: return host[1] - host = self.opts.defaults.get_value('%{_mail_smtp_host}') + if self._args_are_macros(): + host = self.opts.defaults.get_value('%{_mail_smtp_host}') if host is not None: return host return 'localhost' diff --git a/tester/rt/check.py b/tester/rt/check.py index af1d3e9..7e81d30 100755 --- a/tester/rt/check.py +++ b/tester/rt/check.py @@ -49,6 +49,7 @@ from rtemstoolkit import execute from rtemstoolkit import error from rtemstoolkit import host from rtemstoolkit import log +from rtemstoolkit import mailer from rtemstoolkit import path from rtemstoolkit import textbox from rtemstoolkit import version @@ -267,7 +268,7 @@ class warnings_errors: data[category]['common'] = common return data - def _report_category(self, label, warnings, group_counts): + def _report_category(self, label, warnings, group_counts, summary): width = 70 cols_1 = [width] cols_2 = [8, width - 8] @@ -311,15 +312,16 @@ class warnings_errors: d = gs[row:+4] s += textbox.row(cols_4, d, indent = 1) s += textbox.line(cols_2_4, marker = '+', indent = 1) - vw = sorted([(w, warnings[build][w]) for w in build_warnings], - key = operator.itemgetter(1), - reverse = True) - for w in vw: - c1 = '%6d' % w[1] - for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3): - s += textbox.row(cols_2, [c1, l], indent = 1) - c1 = ' ' * 6 - s += textbox.line(cols_2, marker = '+', indent = 1) + if not summary: + vw = sorted([(w, warnings[build][w]) for w in build_warnings], + key = operator.itemgetter(1), + reverse = True) + for w in vw: + c1 = '%6d' % w[1] + for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3): + s += textbox.row(cols_2, [c1, l], indent = 1) + c1 = ' ' * 6 + s += textbox.line(cols_2, marker = '+', indent = 1) return s def _report_warning_map(self): @@ -341,9 +343,9 @@ class warnings_errors: s += textbox.line(cols_1, marker = '+', indent = 1) return s - def warnings_report(self): + def warnings_report(self, summary): self.lock.acquire() - s = ' No warnings' + s = ' No warnings' + os.linesep try: total = 0 for build in self.warnings: @@ -351,16 +353,20 @@ class warnings_errors: if total != 0: data = self._analyze(self.warnings, self.groups['exclude']) s = self._report_category('By Architecture (total : %d)' % (total), - data['arch'], data['groups']['arch']) + data['arch'], data['groups']['arch'], + summary) s += os.linesep s += self._report_category('By BSP (total : %d)' % (total), - data['arch_bsp'], data['groups']['arch_bsp']) + data['arch_bsp'], data['groups']['arch_bsp'], + summary) s += os.linesep s += self._report_category('By Build (total : %d)' % (total), - data['build'], data['groups']['build']) - s += os.linesep - s += self._report_warning_map() + data['build'], data['groups']['build'], + summary) s += os.linesep + if not summary: + s += self._report_warning_map() + s += os.linesep finally: self.lock.release() return s @@ -631,11 +637,11 @@ class results: if build_fails > 0: s += bs + os.linesep if count == 0: - s = ' No failures' + s = ' No failures' + os.linesep return s - def warnings_report(self): - return self.warnings_errors.warnings_report() + def warnings_report(self, summary = False): + return self.warnings_errors.warnings_report(summary) def report(self): self.lock.acquire() @@ -923,7 +929,7 @@ class configuration_: for p in self.profiles['profiles']: profile = {} profile['name'] = p - profile['archs'] = self.config.comma_list(profile['name'], 'archs') + profile['archs'] = self.config.comma_list(profile['name'], 'archs', err = False) archs += profile['archs'] for arch in profile['archs']: bsps = 'bsps_%s' % (arch) @@ -1281,6 +1287,17 @@ class builder: f.write(command_line() + os.linesep) f.write(self.results.warnings_errors.report()) + def _failures_report(self): + if self.options['failures-report'] is not None: + with open(self.options['failures-report'], 'w') as f: + f.write(title() + os.linesep) + f.write(os.linesep) + f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'), + os.linesep)) + f.write(os.linesep) + f.write(command_line() + os.linesep) + f.write(self.results.failures_report()) + def _finished(self): log.notice('Total: Warnings:%d exes:%d objs:%d libs:%d' % \ (self.results.get_warning_count(), self.counts['exes'], @@ -1292,19 +1309,23 @@ class builder: log.notice('Failures:') log.notice(self.results.failures_report()) self._warnings_report() + self._failures_report() def run_jobs(self, jobs): if path.exists(self.build_dir): log.notice('Cleaning: %s' % (self.build_dir)) path.removeall(self.build_dir) - start = datetime.datetime.now() + self.start = datetime.datetime.now() + self.end = datetime.datetime.now() + self.duration = self.end - self.start + self.average = self.duration env_path = os.environ['PATH'] os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \ os.pathsep + os.environ['PATH'] job_count, build_job_count = jobs_option_parse(self.options['jobs']) builds = self._create_build_jobs(jobs, build_job_count) active_jobs = [] - jobs_completed = 0 + self.jobs_completed = 0 try: while len(builds) > 0 or len(active_jobs) > 0: new_jobs = job_count - len(active_jobs) @@ -1323,7 +1344,7 @@ class builder: job.log_output() job.clean() active_jobs.remove(job) - jobs_completed += 1 + self.jobs_completed += 1 time.sleep(0.250) except: for job in active_jobs: @@ -1332,15 +1353,15 @@ class builder: except: pass raise - end = datetime.datetime.now() + self.end = datetime.datetime.now() os.environ['PATH'] = env_path - duration = end - start - if jobs_completed == 0: - jobs_completed = 1 + self.duration = self.end - self.start + if self.jobs_completed == 0: + self.jobs_completed = 1 self._finished() - log.notice('Average BSP Build Time: %s' % \ - (str(duration / jobs_completed))) - log.notice('Total Time %s' % (str(duration))) + self.average = self.duration / self.jobs_completed + log.notice('Average BSP Build Time: %s' % (str(self.average))) + log.notice('Total Time %s' % (str(self.duration))) def arch_bsp_jobs(self, arch, bsps): jobs = [] @@ -1401,6 +1422,7 @@ def run_args(args): if 'msys' in cspath[0] and cspath[0].endswith('bin'): os.environ['PATH'] = os.pathsep.join(cspath[1:]) + start = datetime.datetime.now() top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) prefix = '/opt/rtems/%s' % (rtems_version()) tools = prefix @@ -1423,9 +1445,12 @@ def run_args(args): type = str) argsp.add_argument('--log', help = 'Log file.', type = str) argsp.add_argument('--config-report', help = 'Report the configuration.', - action = 'store_true') + type = str, default = None, + choices = ['all', 'profiles', 'builds', 'archs']) argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.', type = str, default = None) + argsp.add_argument('--failures-report', help = 'Report the failures to a file.', + type = str, default = None) argsp.add_argument('--stop-on-error', help = 'Stop on an error.', action = 'store_true') argsp.add_argument('--no-clean', help = 'Do not clean the build output.', @@ -1442,20 +1467,48 @@ def run_args(args): type = str, default = '1/%d' % (host.cpus())) argsp.add_argument('--dry-run', help = 'Do not run the actual builds.', action = 'store_true') + mailer.add_arguments(argsp) opts = argsp.parse_args(args[1:]) + mail = None + if opts.mail: + mail = mailer.mail(opts) + # Request these now to generate any errors. + from_addr = mail.from_address() + smtp_host = mail.smtp_host() + if 'mail_to' in opts and opts.mail_to is not None: + to_addr = opts.mail_to + else: + to_addr = 'build@rtems.org' if opts.log is not None: logf = opts.log log.default = log.log([logf]) log.notice(title()) log.output(command_line()) + if mail: + log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr, + to_addr, + smtp_host)) config = configuration_() config.load(config_file, opts.build) if opts.config_report: - log.notice('Configuration Report:') - log.notice(config.report()) + log.notice('Configuration Report: %s' % (opts.config_report)) + c_profiles = False + c_builds = False + c_archs = False + if opts.config_report == 'all': + c_profiles = True + c_builds = True + c_archs = True + elif opts.config_report == 'profiles': + c_profiles = True + elif opts.config_report == 'builds': + c_builds = True + elif opts.config_report == 'archs': + c_archs = True + log.notice(config.report(c_profiles, c_builds, c_archs)) sys.exit(0) if opts.rtems is None: @@ -1471,7 +1524,8 @@ def run_args(args): 'no-clean' : opts.no_clean, 'dry-run' : opts.dry_run, 'jobs' : opts.jobs, - 'warnings-report' : opts.warnings_report } + 'warnings-report' : opts.warnings_report, + 'failures-report' : opts.failures_report } b = builder(config, rtems_version(), prefix, tools, path.shell(opts.rtems), build_dir, options) @@ -1485,12 +1539,49 @@ def run_args(args): # if bsps is not None: if archs is not None: - raise error.general('--arch supplied with --bsp; use --bsp=arch/bsp,arch/bsp,..') + raise error.general('--arch supplied with --bsp;' \ + ' use --bsp=arch/bsp,arch/bsp,..') + what = 'BSPs: %s' % (' '.join(bsps)) b.build_bsps(bsps) elif archs is not None: + what = 'Archs: %s' % (' '.join(archs)) b.build_archs(archs) else: + what = 'Profile(s): %s' % (' '.join(profiles)) b.build_profiles(profiles) + end = datetime.datetime.now() + + # + # Email the results of the build. + # + if mail is not None: + subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0], + what) + t = title() + body = t + os.linesep + body += '=' * len(t) + os.linesep + body += os.linesep + body += 'Host: %s' % (os.uname()[3]) + os.linesep + body += os.linesep + body += command_line() + body += os.linesep + body += 'Total Time : %s for %d completed job(s)' % \ + (str(b.duration), b.jobs_completed) + body += os.linesep + body += 'Average BSP Build Time: %s' % (str(b.average)) + body += os.linesep + os.linesep + body += 'Builds' + os.linesep + body += '======' + os.linesep + body += os.linesep.join([' ' + cb for cb in config.builds()]) + body += os.linesep + os.linesep + body += 'Failures Report' + os.linesep + body += '===============' + os.linesep + body += b.results.failures_report() + body += os.linesep + body += 'Warnings Report' + os.linesep + body += '===============' + os.linesep + body += b.results.warnings_report(summary = True) + mail.send(to_addr, subject, body) except error.general as gerr: print(gerr) diff --git a/tester/rtems/rtems-bsps-tiers.ini b/tester/rtems/rtems-bsps-tiers.ini index c6bba14..cc611cb 100644 --- a/tester/rtems/rtems-bsps-tiers.ini +++ b/tester/rtems/rtems-bsps-tiers.ini @@ -21,36 +21,31 @@ # Tier 1: no build errors and no unexpected tests failures on hardware. # [tier-1] -archs = arm, i386, sparc -bsps_arm = altcycv_devkit, - altcycv_devkit_smp, - xilinx_zynq_zc702, xilinx_zynq_zc706, xilinx_zynq_zedboard +archs = arm, i386 +bsps_arm = beagleboneblack, xilinx_zynq_zedboard bsps_i386 = pc686 -bsps_sparc = leon2, leon3 # # Tier 2: no build errors and no unexpected tests failures on hardware and # simulators. # [tier-2] -archs = arm, sparc -bsps_arm = lm3s6965_qemu, - realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp, - xilinx_zynq_a9_qemu -bsps_sparc = erc32 + # # Tier 3: no build errors, no tests run. # [tier-3] -archs = arm, moxie -bsps_arm = arm1136jfs, - arm1136js, arm7tdmi, arm920, armcortexa9, atsamv, - beagleboardorig, beagleboardxm, beagleboneblack, beaglebonewhite, +archs = arm, bfin, i386, lm32, m32c, m68k, mips, moxie, + nios2, or1k, powerpc, sh, sparc, sparc64, v850 +bsps_arm = altcycv_devkit, altcycv_devkit_smp, + arm1136jfs, arm1136js, arm7tdmi, arm920, armcortexa9, atsamv, + beagleboardorig, beagleboardxm, beaglebonewhite, csb336, csb337, csb637, edb7312, kit637_v6, gumstix, - lm3s3749, lm3s6965, lm4f120, + imx7, + lm3s3749, lm3s6965, lm3s6965_qemu, lm4f120, lpc1768_mbed, lpc1768_mbed_ahb_ram, lpc1768_mbed_ahb_ram_eth, lpc17xx_ea_ram, lpc17xx_ea_rom_int, lpc17xx_plx800_ram, lpc17xx_plx800_rom_int, lpc2362, lpc23xx_tli800, lpc24xx_ea, @@ -59,13 +54,65 @@ bsps_arm = arm1136jfs, lpc40xx_ea_rom_int, lpc32xx_mzx, lpc32xx_mzx_stage_1, lpc32xx_mzx_stage_2, lpc32xx_phycore, raspberrypi, raspberrypi2, + realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp, rtl22xx, rtl22xx_t, smdk2410, stm32f105rc, stm32f4, tms570ls3137_hdk, tms570ls3137_hdk_intram, tms570ls3137_hdk_sdram, - tms570ls3137_hdk_with_loader + tms570ls3137_hdk_with_loader, + xilinx_zynq_a9_qemu, xilinx_zynq_zc702, xilinx_zynq_zc706 +bsps_bfin = TLL6527M, bf537Stamp, eZKit533 +bsps_i386 = pc386, pc486, pc586-sse, pc586, pcp4 +bsps_lm32 = lm32_evr, lm32_evr_gdbsim, milkymist +bsps_m32c = m32csim +bsps_m68k = av5282, + csb360, + gen68340, gen68360, gen68360_040, + pgh360, + COBRA5475, + m5484FireEngine, + mcf5206elite, + mcf52235, mcf5225x, + mcf5235, + mcf5329, + mrm332, + mvme147, mvme147s, mvme162, mvme162lx, mvme167, + uC5282 +bsps_mips = csb350, hurricane, jmr3904, malta, rbtx4925, rbtx4938 bsps_moxie = moxiesim +bsps_nios2 = nios2_iss +bsps_or1k = generic_or1k +bsps_powerpc = beatnik, + br_uid, brs5l, brs6l, + dp2, + gwlcfm, + haleakala, + hsc_cm01, + icecube, + mcp750, + mpc5566evb, mpc5566evb_spe, phycore_mpc5554, + mpc5643l_dpu, mpc5643l_evb, mpc5668g, + mpc5674f_ecu508_app, mpc5674f_ecu508_boot, mpc5674f_rsm6, mpc5674fevb, mpc5674fevb_spe, + mpc8260ads, + mpc8309som, + mpc8313erdb, + mpc8349eamds, + mtx603e, + mvme2100, mvme2307, mvme3100, mvme5500, + pghplus, + pm520_cr825, pm520_ze30, + psim, + qemuppc, qemuprep, qemuprep-altivec, + qoriq_core_0, qoriq_core_1, qoriq_e500, qoriq_e6500_32, qoriq_e6500_64 + ss555, + t32mppc, + tqm8xx_stk8xx, + virtex, virtex4, virtex5 +bsps_sh = gensh1, gensh2, gensh4, simsh1, simsh2, simsh2e, simsh4 +bsps_sparc = erc32, leon2, leon3, ngmp +bsps_sparc64 = niagara, usiii +bsps_v850 = v850e1sim, v850e2sim, v850e2v3sim, v850esim, v850essim, v850sim # # Tier 4: nothing expected. |