From 30218f5ca5b6b060258791a4f4c915e16f1edbe9 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Sun, 5 Nov 2017 18:48:20 +1100 Subject: tester: Add reporting the RTEMS version, build and tools. Update #3217. --- tester/rt/config.py | 19 ++++++---- tester/rt/report.py | 102 ++++++++++++++++++++++++++++++++++++++++++---------- tester/rt/run.py | 4 ++- tester/rt/test.py | 18 +++++++--- 4 files changed, 112 insertions(+), 31 deletions(-) diff --git a/tester/rt/config.py b/tester/rt/config.py index eb11740..1c2670a 100644 --- a/tester/rt/config.py +++ b/tester/rt/config.py @@ -67,6 +67,7 @@ class file(config.file): self.realtime_trace = self.exe_trace('output') self.console_trace = self.exe_trace('console') self.console_prefix = console_prefix + self.show_header = not self.defined('test_disable_header') self.process = None self.console = None self.output = None @@ -301,8 +302,8 @@ class file(config.file): bsp_arch = self.expand('%{arch}') bsp = self.expand('%{bsp}') fexe = self._target_exe_filter(exe) - if self.report is not None: - self.report.start(index, total, exe, fexe, bsp_arch, bsp) + self.report.start(index, total, exe, fexe, + bsp_arch, bsp, self.show_header) if self.index == 1: self._target_command('on', bsp_arch, bsp, exe, fexe) self._target_command('pretest', bsp_arch, bsp, exe, fexe) @@ -321,10 +322,14 @@ class file(config.file): self._target_command('off', bsp_arch, bsp, exe, fexe) self._target_command('posttest', bsp_arch, bsp, exe, fexe) try: - status = '' - if self.report is not None: - status = self.report.end(exe, self.output) - self._capture_console('test result: %s' % (status)) + status = self.report.end(exe, self.output, self.console_prefix) + version = self.report.get_config('version', not_found = 'n/p') + build = self.report.get_config('build', not_found = 'n/p') + tools = self.report.get_config('tools', not_found = 'n/p') + self._capture_console('test result: %s' % (status)) + self._capture_console('test version: %s' % (version)) + self._capture_console('test build: %s' % (build)) + self._capture_console('test tools: %s' % (tools)) if status == 'timeout': if self.index != self.total: self._target_command('reset', bsp_arch, bsp, exe, fexe) @@ -336,7 +341,7 @@ class file(config.file): def _realtime_trace(self, text): for l in text: - print(l) + print(''.join(l)) def run(self): self.target_start_regx = self._target_regex('target_start_regex') diff --git a/tester/rt/report.py b/tester/rt/report.py index 7c906de..4bf6ef4 100644 --- a/tester/rt/report.py +++ b/tester/rt/report.py @@ -55,14 +55,18 @@ class report(object): self.total = total self.total_len = len(str(total)) self.passed = 0 - self.user_input = 0 self.failed = 0 + self.user_input = 0 self.expected_fail = 0 self.indeterminate = 0 self.benchmark = 0 self.timeouts = 0 self.invalids = 0 + self.wrong_version = 0 + self.wrong_build = 0 + self.wrong_tools = 0 self.results = {} + self.config = {} self.name_max_len = 0 def __str__(self): @@ -74,10 +78,14 @@ class report(object): msg += 'Benchmark: %*d%s' % (self.total_len, self.self.benchmark, os.linesep) msg += 'Timeout: %*d%s' % (self.total_len, self.timeouts, os.linesep) msg += 'Invalid: %*d%s' % (self.total_len, self.invalids, os.linesep) + msg += 'Wrong Version %*d%s' % (self.total_len, self.wrong_version, os.linesep) + msg += 'Wrong Build %*d%s' % (self.total_len, self.wrong_build, os.linesep) + msg += 'Wrong Tools %*d%s' % (self.total_len, self.wrong_tools, os.linesep) return msg - def start(self, index, total, name, executable, bsp_arch, bsp): - header = '[%*d/%*d] p:%-*d f:%-*d u:%-*d e:%-*d I:%-*d B:%-*d t:%-*d i:%-*d | %s/%s: %s' % \ + def start(self, index, total, name, executable, bsp_arch, bsp, show_header): + header = '[%*d/%*d] p:%-*d f:%-*d u:%-*d e:%-*d I:%-*d B:%-*d ' \ + 't:%-*d i:%-*d W:%-*d | %s/%s: %s' % \ (len(str(total)), index, len(str(total)), total, len(str(total)), self.passed, @@ -88,6 +96,7 @@ class report(object): len(str(total)), self.benchmark, len(str(total)), self.timeouts, len(str(total)), self.invalids, + len(str(total)), self.wrong_version + self.wrong_build + self.wrong_tools, bsp_arch, bsp, path.basename(name)) @@ -106,25 +115,36 @@ class report(object): 'header': header } self.lock.release() - log.notice(header, stdout_only = True) + if show_header: + log.notice(header, stdout_only = True) - def end(self, name, output): + def end(self, name, output, output_prefix): start = False end = False state = None + version = None + build = None + tools = None timeout = False prefixed_output = [] for line in output: - if line[0] == ']': + if line[0] == output_prefix: if line[1].startswith('*** '): - if line[1][4:].startswith('BEGIN OF '): + banner = line[1][4:] + if banner.startswith('BEGIN OF '): start = True - if line[1][4:].startswith('END OF '): + elif line[1][4:].startswith('END OF '): end = True - if line[1][4:].startswith('TEST STATE:'): - state = line[1][15:].strip() - if line[1][4:].startswith('TIMEOUT TIMEOUT'): + elif banner.startswith('TIMEOUT TIMEOUT'): timeout = True + elif banner.startswith('TEST VERSION:'): + version = banner[13:].strip() + elif banner.startswith('TEST STATE:'): + state = banner[11:].strip() + elif banner.startswith('TEST BUILD:'): + build = ','.join(banner[11:].strip().split(' ')) + elif banner.startswith('TEST TOOLS:'): + tools = banner[11:].strip() prefixed_output += [line[0] + ' ' + line[1]] self.lock.acquire() if name not in self.results: @@ -134,9 +154,27 @@ class report(object): self.lock.release() raise error.general('test already finished: %s' % (name)) self.results[name]['end'] = datetime.datetime.now() - if state is None: + if version: + if 'version' not in self.config: + self.config['version'] = version + else: + if version != self.config['version']: + state = 'WRONG-VERSION' + if build: + if 'build' not in self.config: + self.config['build'] = build + else: + if build != self.config['build']: + state = 'WRONG-BUILD' + if tools: + if 'tools' not in self.config: + self.config['tools'] = tools + else: + if tools != self.config['tools']: + state = 'WRONG-TOOLS' + if state is None or state == 'EXPECTED-PASS': if start and end: - if state is None: + if state is None or state == 'EXPECTED-PASS': status = 'passed' self.passed += 1 elif timeout: @@ -175,6 +213,15 @@ class report(object): elif state == 'BENCHMARK': status = 'benchmark' self.benchmark += 1 + elif state == 'WRONG-VERSION': + status = 'wrong-version' + self.wrong_version += 1 + elif state == 'WRONG-BUILD': + status = 'wrong-build' + self.wrong_build += 1 + elif state == 'WRONG-TOOLS': + status = 'wrong-tools' + self.wrong_tools += 1 else: raise error.general('invalid test state: %s: %s' % (name, state)) self.results[name]['result'] = status @@ -185,6 +232,8 @@ class report(object): return status def log(self, name, mode): + status_fails = ['failed', 'timeout', 'invalid', + 'wrong-version', 'wrong-build', 'wrong-tools'] if mode != 'none': self.lock.acquire() if name not in self.results: @@ -193,7 +242,7 @@ class report(object): exe = path.basename(self.results[name]['exe']) result = self.results[name]['result'] time = self.results[name]['end'] - self.results[name]['start'] - failed = result in ['failed', 'timeout', 'invalid'] + failed = result in status_fails result = 'Result: %-10s Time: %s %s' % (result, str(time), exe) if mode != 'none': header = self.results[name]['header'] @@ -208,12 +257,16 @@ class report(object): log.output(result) log.output(output) + def get_config(self, config, not_found = None): + if config in self.config: + return self.config[config] + return not_found + def score_card(self, mode = 'full'): if mode == 'short': - return 'Passed:%d Failed:%d Timeout:%d Invalid:%d' % (self.passed, - self.failed, - self.timeouts, - self.invalids) + wrongs = self.wrong_version + self.wrong_build + self.wrong_tools + return 'Passed:%d Failed:%d Timeout:%d Invalid:%d Wrong:%d' % \ + (self.passed, self.failed, self.timeouts, self.invalids, wrongs) elif mode == 'full': l = [] l += ['Passed: %*d' % (self.total_len, self.passed)] @@ -224,6 +277,9 @@ class report(object): l += ['Benchmark: %*d' % (self.total_len, self.benchmark)] l += ['Timeout: %*d' % (self.total_len, self.timeouts)] l += ['Invalid: %*d' % (self.total_len, self.invalids)] + l += ['Wrong Version: %*d' % (self.total_len, self.wrong_version)] + l += ['Wrong Build: %*d' % (self.total_len, self.wrong_build)] + l += ['Wrong Tools: %*d' % (self.total_len, self.wrong_tools)] l += ['---------------%s' % ('-' * self.total_len)] l += ['Total: %*d' % (self.total_len, self.total)] return os.linesep.join(l) @@ -258,10 +314,18 @@ class report(object): if self.invalids: l += ['Invalid:'] l += show_state(self.results, 'invalid', self.name_max_len) + if self.wrong_version: + l += ['Wrong Version:'] + l += show_state(self.results, 'wrong-version', self.name_max_len) + if self.wrong_build: + l += ['Wrong Build:'] + l += show_state(self.results, 'wrong-build', self.name_max_len) + if self.wrong_tools: + l += ['Wrong Tools:'] + l += show_state(self.results, 'wrong-tools', self.name_max_len) return os.linesep.join(l) def summary(self): log.output() log.notice(self.score_card()) - log.output() log.output(self.failures()) diff --git a/tester/rt/run.py b/tester/rt/run.py index bcd0697..cf8c4e5 100644 --- a/tester/rt/run.py +++ b/tester/rt/run.py @@ -143,9 +143,11 @@ def run(command_path = None): executables = find_executables(opts.params()) if len(executables) != 1: raise error.general('one executable required, found %d' % (len(executables))) + opts.defaults['test_disable_header'] = '1' + reports = report.report(1) start_time = datetime.datetime.now() opts.defaults['exe_trace'] = debug_trace - tst = test(1, 1, None, executables[0], rtems_tools, bsp, bsp_config, opts) + tst = test(1, 1, reports, executables[0], rtems_tools, bsp, bsp_config, opts) tst.run() end_time = datetime.datetime.now() total_time = 'Run time : %s' % (str(end_time - start_time)) diff --git a/tester/rt/test.py b/tester/rt/test.py index 32ff56f..1a91763 100644 --- a/tester/rt/test.py +++ b/tester/rt/test.py @@ -351,15 +351,25 @@ def run(command_path = None): if mail is not None and output is not None: m_arch = opts.defaults.expand('%{arch}') m_bsp = opts.defaults.expand('%{bsp}') - subject = '[rtems-test] %s/%s: %s' % (m_arch, - m_bsp, - reports.score_card('short')) + build = ' %s:' % (reports.get_config('build', not_found = '')) + subject = '[rtems-test] %s/%s:%s %s' % (m_arch, + m_bsp, + build, + reports.score_card('short')) + np = 'Not present in test' + ver = reports.get_config('version', not_found = np) + build = reports.get_config('build', not_found = np) + tools = reports.get_config('tools', not_found = np) body = [total_time, average_time, '', 'Host', '====', host.label(mode = 'all'), + '', 'Configuration', '=============', + 'Version: %s' % (ver), + 'Build : %s' % (build), + 'Tools : %s' % (tools), '', 'Summary', '=======', '', reports.score_card(), '', reports.failures(), - '', 'Log', '===', ''] + output.get() + 'Log', '===', ''] + output.get() mail.send(to_addr, subject, os.linesep.join(body)) except error.general as gerr: -- cgit v1.2.3