testoutputparser: New

2 files changed, 1411 insertions, 0 deletions

diff --git a/rtemsspec/testoutputparser.py b/rtemsspec/testoutputparser.py
new file mode 100644
index 00000000..c2ccac21
--- /dev/null
+++ b/rtemsspec/testoutputparser.py
@@ -0,0 +1,580 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" This module provides a test output parser. """
+
+# Copyright (C) 2022 embedded brains GmbH (http://www.embedded-brains.de)
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import base64
+import hashlib
+import re
+from typing import Any, Dict, Iterable
+
+_TEST_BEGIN = re.compile(r"\*\*\* BEGIN OF TEST ([^*]+) \*\*\*")
+_TEST_VERSION = re.compile(r"\*\*\* TEST VERSION: (.+)")
+_TEST_STATE = re.compile(r"\*\*\* TEST STATE: (.+)")
+_TEST_BUILD = re.compile(r"\*\*\* TEST BUILD: ?(.*)")
+_TEST_TOOLS = re.compile(r"\*\*\* TEST TOOLS: (.+)")
+_TEST_END = re.compile(r"\*\*\* END OF TEST ([^*]+) \*\*\*")
+
+_TS_SUITE_BEGIN = re.compile(r"A:(.+)")
+_TS_SUITE_END = re.compile(r"Z:([^:]+):C:([^:]+):N:([^:]+):F:([^:]+):D:(.+)")
+_TS_CASE_BEGIN = re.compile(r"B:(.+)")
+_TS_CASE_END = re.compile(r"E:([^:]+):N:([^:]+):F:([^:]+):D:(.+)")
+_TS_PLATFORM = re.compile(r"S:Platform:(.+)")
+_TS_COMPILER = re.compile(r"S:Compiler:(.+)")
+_TS_VERSION = re.compile(r"S:Version:(.+)")
+_TS_BSP = re.compile(r"S:BSP:(.+)")
+_TS_BUILD_LABEL = re.compile(r"S:BuildLabel:(.+)")
+_TS_TARGET_HASH = re.compile(r"S:TargetHash:SHA256:(.*)")
+_TS_RTEMS_DEBUG = re.compile(r"S:RTEMS_DEBUG:([01])$")
+_TS_RTEMS_MULTIPROCESSING = re.compile(r"S:RTEMS_MULTIPROCESSING:([01])$")
+_TS_RTEMS_POSIX_API = re.compile(r"S:RTEMS_POSIX_API:([01])$")
+_TS_RTEMS_PROFILING = re.compile(r"S:RTEMS_PROFILING:([01])$")
+_TS_RTEMS_SMP = re.compile(r"S:RTEMS_SMP:([01])$")
+_TS_REPORT_HASH = re.compile(r"Y:ReportHash:SHA256:(.+)")
+
+_M_BEGIN = re.compile(r"M:B:(.+)")
+_M_END = re.compile(r"M:E:([^:]+):D:(.+)")
+_M_V = re.compile(r"M:V:(.+)")
+_M_N = re.compile(r"M:N:(.+)")
+_M_S = re.compile(r"M:S:([^:]+):(.+)")
+_M_MI = re.compile(r"M:MI:(.+)")
+_M_P1 = re.compile(r"M:P1:(.+)")
+_M_Q1 = re.compile(r"M:Q1:(.+)")
+_M_Q2 = re.compile(r"M:Q2:(.+)")
+_M_Q3 = re.compile(r"M:Q3:(.+)")
+_M_P99 = re.compile(r"M:P99:(.+)")
+_M_MX = re.compile(r"M:MX:(.+)")
+_M_MAD = re.compile(r"M:MAD:(.+)")
+_M_D = re.compile(r"M:D:(.+)")
+
+_GCOV_BEGIN = re.compile(r"\*\*\* BEGIN OF GCOV INFO BASE64 \*\*\*")
+_GCOV_END = re.compile(r"\*\*\* END OF GCOV INFO BASE64 \*\*\*")
+
+_RECORDS_BEGIN = re.compile(r"\*\*\* BEGIN OF RECORDS BASE64 \*\*\*")
+_RECORDS_END = re.compile(r"\*\*\* END OF RECORDS BASE64 \*\*\*")
+_RECORDS_ZLIB_BEGIN = re.compile(r"\*\*\* BEGIN OF RECORDS BASE64 ZLIB \*\*\*")
+_RECORDS_ZLIB_END = re.compile(r"\*\*\* END OF RECORDS BASE64 ZLIB \*\*\*")
+
+
+def _are_samples_valid(measurement) -> bool:
+    if len(measurement["samples"]) != measurement["sample-count"]:
+        return False
+    if not measurement["samples"]:
+        return True
+    if measurement["min"] != measurement["samples"][0]:
+        return False
+    return measurement["max"] == measurement["samples"][-1]
+
+
+class TestOutputParser:
+    """ Provides a line by line parser of test output. """
+
+    # pylint: disable=too-few-public-methods
+    def __init__(self, data) -> None:
+        self.data = data
+        self.consume = self._test_begin
+        self.hash_line = self._hash_none
+        assert "info" not in data
+        self.data["info"] = {}
+        self.data["data-ranges"] = []
+        assert "test-suite" not in data
+        self.level = 0
+        self._hash_state = hashlib.sha256()
+        self._measurement: Dict[str, Any] = {}
+        self._test_case: Dict[str, Any] = {}
+
+    def _error(self, index: int) -> bool:
+        assert "line-parser-error" not in self.data
+        self.data["line-parser-error"] = index
+        self.consume = self._extra
+        return False
+
+    def _hash_none(self, line: str) -> None:
+        pass
+
+    def _hash_sha256(self, line: str) -> None:
+        self._hash_state.update(f"{line}\n".encode("ascii"))
+
+    def _test_begin(self, index: int, line: str) -> bool:
+        mobj = _TEST_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self.data["info"]["line-begin-of-test"] = index
+            self.data["info"]["name"] = mobj.group(1)
+            self.consume = self._test_version
+            return True
+        return self._extra(index, line)
+
+    def _test_version(self, index: int, line: str) -> bool:
+        mobj = _TEST_VERSION.match(line)
+        if mobj:
+            self.data["info"]["version"] = mobj.group(1)
+            self.data["info"]["line-version"] = index
+            self.consume = self._test_state
+            return True
+        self.consume = self._test_body
+        return self._test_body(index, line)
+
+    def _test_state(self, index: int, line: str) -> bool:
+        mobj = _TEST_STATE.match(line)
+        if mobj:
+            self.data["info"]["state"] = mobj.group(1)
+            self.data["info"]["line-state"] = index
+            self.consume = self._test_build
+            return True
+        return self._error(index)
+
+    def _test_build(self, index: int, line: str) -> bool:
+        mobj = _TEST_BUILD.match(line)
+        if mobj:
+            build = mobj.group(1)
+            if build:
+                self.data["info"]["build"] = build.split(", ")
+            else:
+                self.data["info"]["build"] = []
+            self.data["info"]["line-build"] = index
+            self.consume = self._test_tools
+            return True
+        return self._error(index)
+
+    def _test_tools(self, index: int, line: str) -> bool:
+        mobj = _TEST_TOOLS.match(line)
+        if mobj:
+            self.data["info"]["tools"] = mobj.group(1)
+            self.data["info"]["line-tools"] = index
+            self.consume = self._test_body
+            return True
+        return self._error(index)
+
+    def _test_body(self, index: int, line: str) -> bool:
+        if self._test_suite_begin(index, line):
+            return True
+        mobj = _TEST_END.match(line)
+        if mobj:
+            self.level -= 1
+            if self.data["info"]["name"] == mobj.group(1):
+                self.data["info"]["line-end-of-test"] = index
+                self.consume = self._extra
+                return True
+            return self._error(index)
+        return self._extra(index, line)
+
+    def _test_suite_begin(self, index: int, line: str) -> bool:
+        mobj = _TS_SUITE_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self.data["test-suite"] = {
+                "duration": "?",
+                "failed-steps-count": "?",
+                "line-begin": index,
+                "line-duration": "?",
+                "line-end": "?",
+                "line-failed-steps-count": "?",
+                "line-report-hash": "?",
+                "line-step-count": "?",
+                "name": mobj.group(1),
+                "report-hash": "?",
+                "report-hash-calculated": "?",
+                "step-count": "?",
+                "test-cases": []
+            }
+            self.consume = self._test_suite_platform
+            self.hash_line = self._hash_sha256
+            return True
+        return self._extra(index, line)
+
+    def _test_suite_platform(self, index: int, line: str) -> bool:
+        mobj = _TS_PLATFORM.match(line)
+        if mobj:
+            self.data["test-suite"]["platform"] = mobj.group(1)
+            self.data["test-suite"]["line-platform"] = index
+            self.consume = self._test_suite_compiler
+            return True
+        return self._error(index)
+
+    def _test_suite_compiler(self, index: int, line: str) -> bool:
+        mobj = _TS_COMPILER.match(line)
+        if mobj:
+            self.data["test-suite"]["compiler"] = mobj.group(1)
+            self.data["test-suite"]["line-compiler"] = index
+            self.consume = self._test_suite_version
+            return True
+        return self._error(index)
+
+    def _test_suite_version(self, index: int, line: str) -> bool:
+        mobj = _TS_VERSION.match(line)
+        if mobj:
+            self.data["test-suite"]["version"] = mobj.group(1)
+            self.data["test-suite"]["line-version"] = index
+            self.consume = self._test_suite_bsp
+            return True
+        return self._error(index)
+
+    def _test_suite_bsp(self, index: int, line: str) -> bool:
+        mobj = _TS_BSP.match(line)
+        if mobj:
+            self.data["test-suite"]["bsp"] = mobj.group(1)
+            self.data["test-suite"]["line-bsp"] = index
+            self.consume = self._test_suite_build_label
+            return True
+        return self._error(index)
+
+    def _test_suite_build_label(self, index: int, line: str) -> bool:
+        mobj = _TS_BUILD_LABEL.match(line)
+        if mobj:
+            self.data["test-suite"]["build-label"] = mobj.group(1)
+            self.data["test-suite"]["line-build-label"] = index
+            self.consume = self._test_suite_target_hash
+            return True
+        return self._error(index)
+
+    def _test_suite_target_hash(self, index: int, line: str) -> bool:
+        mobj = _TS_TARGET_HASH.match(line)
+        if mobj:
+            self.data["test-suite"]["target-hash"] = mobj.group(1)
+            self.data["test-suite"]["line-target-hash"] = index
+            self.consume = self._test_suite_rtems_debug
+            return True
+        return self._error(index)
+
+    def _test_suite_rtems_debug(self, index: int, line: str) -> bool:
+        mobj = _TS_RTEMS_DEBUG.match(line)
+        if mobj:
+            self.data["test-suite"]["rtems-debug"] = bool(int(mobj.group(1)))
+            self.data["test-suite"]["line-rtems-debug"] = index
+            self.consume = self._test_suite_rtems_multiprocessing
+            return True
+        return self._error(index)
+
+    def _test_suite_rtems_multiprocessing(self, index: int, line: str) -> bool:
+        mobj = _TS_RTEMS_MULTIPROCESSING.match(line)
+        if mobj:
+            self.data["test-suite"]["rtems-multiprocessing"] = bool(
+                int(mobj.group(1)))
+            self.data["test-suite"]["line-rtems-multiprocessing"] = index
+            self.consume = self._test_suite_rtems_posix_api
+            return True
+        return self._error(index)
+
+    def _test_suite_rtems_posix_api(self, index: int, line: str) -> bool:
+        mobj = _TS_RTEMS_POSIX_API.match(line)
+        if mobj:
+            self.data["test-suite"]["rtems-posix-api"] = bool(
+                int(mobj.group(1)))
+            self.data["test-suite"]["line-rtems-posix-api"] = index
+            self.consume = self._test_suite_rtems_profiling
+            return True
+        return self._error(index)
+
+    def _test_suite_rtems_profiling(self, index: int, line: str) -> bool:
+        mobj = _TS_RTEMS_PROFILING.match(line)
+        if mobj:
+            self.data["test-suite"]["rtems-profiling"] = bool(
+                int(mobj.group(1)))
+            self.data["test-suite"]["line-rtems-profiling"] = index
+            self.consume = self._test_suite_rtems_smp
+            return True
+        return self._error(index)
+
+    def _test_suite_rtems_smp(self, index: int, line: str) -> bool:
+        mobj = _TS_RTEMS_SMP.match(line)
+        if mobj:
+            self.data["test-suite"]["rtems-smp"] = bool(int(mobj.group(1)))
+            self.data["test-suite"]["line-rtems-smp"] = index
+            self.consume = self._test_suite_body
+            return True
+        return self._error(index)
+
+    def _test_suite_body(self, index: int, line: str) -> bool:
+        if self._test_case_begin(index, line):
+            return True
+        mobj = _TS_SUITE_END.match(line)
+        if mobj:
+            self.level -= 1
+            data = self.data["test-suite"]
+            count = int(mobj.group(2))
+            if data["name"] == mobj.group(1) and len(
+                    data["test-cases"]) == count:
+                data["line-end"] = index
+                data["line-step-count"] = index
+                data["line-failed-steps-count"] = index
+                data["line-duration"] = index
+                data["step-count"] = int(mobj.group(3))
+                data["failed-steps-count"] = int(mobj.group(4))
+                data["duration"] = float(mobj.group(5))
+                self.consume = self._report_hash
+                return True
+            return self._error(index)
+        return self._extra(index, line)
+
+    def _test_case_begin(self, index: int, line: str) -> bool:
+        mobj = _TS_CASE_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self._test_case = {
+                "line-begin": index,
+                "name": mobj.group(1),
+                "runtime-measurements": []
+            }
+            self.consume = self._test_case_body
+            return True
+        return self._extra(index, line)
+
+    def _test_case_body(self, index: int, line: str) -> bool:
+        if self._measurement_begin(index, line):
+            return True
+        mobj = _TS_CASE_END.match(line)
+        if mobj:
+            self.level -= 1
+            if self._test_case["name"] == mobj.group(1):
+                self._test_case["line-end"] = index
+                self._test_case["line-step-count"] = index
+                self._test_case["line-failed-steps-count"] = index
+                self._test_case["line-duration"] = index
+                self._test_case["step-count"] = int(mobj.group(2))
+                self._test_case["failed-steps-count"] = int(mobj.group(3))
+                self._test_case["duration"] = float(mobj.group(4))
+                self.data["test-suite"]["test-cases"].append(self._test_case)
+                self.consume = self._test_suite_body
+                return True
+            return self._error(index)
+        return self._extra(index, line)
+
+    def _measurement_begin(self, index: int, line: str) -> bool:
+        mobj = _M_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self._measurement = {
+                "line-begin": index,
+                "name": mobj.group(1),
+                "samples": []
+            }
+            self.consume = self._measurement_variant
+            return True
+        return self._extra(index, line)
+
+    def _measurement_variant(self, index: int, line: str) -> bool:
+        mobj = _M_V.match(line)
+        if mobj:
+            self._measurement["variant"] = mobj.group(1)
+            self.consume = self._measurement_count
+            return True
+        return self._error(index)
+
+    def _measurement_count(self, index: int, line: str) -> bool:
+        mobj = _M_N.match(line)
+        if mobj:
+            self._measurement["sample-count"] = int(mobj.group(1))
+            self.consume = self._measurement_samples
+            return True
+        return self._error(index)
+
+    def _measurement_samples(self, index: int, line: str) -> bool:
+        if self._measurement_min(index, line):
+            return True
+        mobj = _M_S.match(line)
+        if mobj:
+            self._measurement["samples"].extend(  # type: ignore
+                int(mobj.group(1)) * [float(mobj.group(2))])
+            return True
+        return self._error(index)
+
+    def _measurement_min(self, index: int, line: str) -> bool:
+        mobj = _M_MI.match(line)
+        if mobj:
+            self._measurement["min"] = float(mobj.group(1))
+            self.consume = self._measurement_p1
+            return True
+        return self._extra(index, line)
+
+    def _measurement_p1(self, index: int, line: str) -> bool:
+        mobj = _M_P1.match(line)
+        if mobj:
+            self._measurement["p1"] = float(mobj.group(1))
+            self.consume = self._measurement_q1
+            return True
+        return self._error(index)
+
+    def _measurement_q1(self, index: int, line: str) -> bool:
+        mobj = _M_Q1.match(line)
+        if mobj:
+            self._measurement["q1"] = float(mobj.group(1))
+            self.consume = self._measurement_q2
+            return True
+        return self._error(index)
+
+    def _measurement_q2(self, index: int, line: str) -> bool:
+        mobj = _M_Q2.match(line)
+        if mobj:
+            self._measurement["q2"] = float(mobj.group(1))
+            self.consume = self._measurement_q3
+            return True
+        return self._error(index)
+
+    def _measurement_q3(self, index: int, line: str) -> bool:
+        mobj = _M_Q3.match(line)
+        if mobj:
+            self._measurement["q3"] = float(mobj.group(1))
+            self.consume = self._measurement_p99
+            return True
+        return self._error(index)
+
+    def _measurement_p99(self, index: int, line: str) -> bool:
+        mobj = _M_P99.match(line)
+        if mobj:
+            self._measurement["p99"] = float(mobj.group(1))
+            self.consume = self._measurement_max
+            return True
+        return self._error(index)
+
+    def _measurement_max(self, index: int, line: str) -> bool:
+        mobj = _M_MX.match(line)
+        if mobj:
+            self._measurement["max"] = float(mobj.group(1))
+            self.consume = self._measurement_mad
+            return True
+        return self._error(index)
+
+    def _measurement_mad(self, index: int, line: str) -> bool:
+        mobj = _M_MAD.match(line)
+        if mobj:
+            self._measurement["mad"] = float(mobj.group(1))
+            self.consume = self._measurement_duration
+            return True
+        return self._error(index)
+
+    def _measurement_duration(self, index: int, line: str) -> bool:
+        mobj = _M_D.match(line)
+        if mobj:
+            self._measurement["duration-sum"] = float(mobj.group(1))
+            self.consume = self._measurement_end
+            return True
+        return self._error(index)
+
+    def _measurement_end(self, index: int, line: str) -> bool:
+        mobj = _M_END.match(line)
+        if mobj:
+            self.level -= 1
+            if self._measurement["name"] == mobj.group(
+                    1) and _are_samples_valid(self._measurement):
+                self._measurement["line-end"] = index
+                self._measurement["duration-total"] = float(mobj.group(2))
+                self._test_case["runtime-measurements"].append(  # type: ignore
+                    self._measurement)
+                self.consume = self._test_case_body
+                return True
+        return self._error(index)
+
+    def _report_hash(self, index: int, line: str) -> bool:
+        mobj = _TS_REPORT_HASH.match(line)
+        if mobj:
+            digest = base64.urlsafe_b64encode(
+                self._hash_state.digest()).decode("ascii")
+            self._hash_state = hashlib.sha256()
+            self.data["test-suite"]["report-hash-calculated"] = digest
+            self.data["test-suite"]["report-hash"] = mobj.group(1)
+            self.data["test-suite"]["line-report-hash"] = index
+            self.consume = self._test_body
+            self.hash_line = self._hash_none
+            return True
+        return self._extra(index, line)
+
+    def _gcov_begin(self, index: int, line: str) -> bool:
+        mobj = _GCOV_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self.data["line-gcov-info-base64-begin"] = index
+            self.consume = self._gcov_end
+            return True
+        return False
+
+    def _gcov_end(self, index: int, line: str) -> bool:
+        mobj = _GCOV_END.match(line)
+        if mobj:
+            self.level -= 1
+            self.data["line-gcov-info-base64-end"] = index
+            self.data["data-ranges"].append(
+                (self.data["line-gcov-info-base64-begin"] + 1, index))
+            self.consume = self._extra
+            return True
+        return False
+
+    def _records_begin(self, index: int, line: str) -> bool:
+        mobj = _RECORDS_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self.data["line-records-base64-begin"] = index
+            self.consume = self._records_end
+            return True
+        return False
+
+    def _records_end(self, index: int, line: str) -> bool:
+        mobj = _RECORDS_END.match(line)
+        if mobj:
+            self.level -= 1
+            self.data["line-records-base64-end"] = index
+            self.data["data-ranges"].append(
+                (self.data["line-records-base64-begin"] + 1, index))
+            self.consume = self._extra
+            return True
+        return False
+
+    def _records_zlib_begin(self, index: int, line: str) -> bool:
+        mobj = _RECORDS_ZLIB_BEGIN.match(line)
+        if mobj:
+            self.level += 1
+            self.data["line-records-base64-zlib-begin"] = index
+            self.consume = self._records_zlib_end
+            return True
+        return False
+
+    def _records_zlib_end(self, index: int, line: str) -> bool:
+        mobj = _RECORDS_ZLIB_END.match(line)
+        if mobj:
+            self.level -= 1
+            self.data["line-records-base64-zlib-end"] = index
+            self.data["data-ranges"].append(
+                (self.data["line-records-base64-zlib-begin"] + 1, index))
+            self.consume = self._extra
+            return True
+        return False
+
+    def _extra(self, index: int, line: str) -> bool:
+        if self._gcov_begin(index, line):
+            return True
+        if self._records_begin(index, line):
+            return True
+        if self._records_zlib_begin(index, line):
+            return True
+        return False
+
+
+def augment_report(report: Dict[str, Any], output: Iterable[str]) -> None:
+    """ Augments the report with the results of the parsed output. """
+    test_parser = TestOutputParser(report)
+    for index, line in enumerate(output):
+        if not line:
+            continue
+        test_parser.consume(index, line)
+        test_parser.hash_line(line)
diff --git a/rtemsspec/tests/test_testoutputparser.py b/rtemsspec/tests/test_testoutputparser.py
new file mode 100644
index 00000000..513cd76e
--- /dev/null
+++ b/rtemsspec/tests/test_testoutputparser.py
@@ -0,0 +1,831 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" Unit tests for the rtemsspec.testoutputparser module. """
+
+# Copyright (C) 2023 embedded brains GmbH & Co. KG
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import pytest
+
+from rtemsspec.testoutputparser import augment_report
+
+_OUTPUT = [
+    "*** BEGIN OF TEST TestsuitesUnitNoClock0 ***",
+    "*** TEST VERSION: 6.0.0.52f06822b8921ad825cb593b792eab7640e26cde",
+    "*** TEST STATE: EXPECTED_PASS", "*** TEST BUILD:",
+    "*** TEST TOOLS: 10.4.0", "A:TestsuitesUnitNoClock0", "S:Platform:RTEMS",
+    "S:Compiler:10.4.0",
+    "S:Version:6.0.0.52f06822b8921ad825cb593b792eab7640e26cde",
+    "S:BSP:xilinx_zynq_a9_qemu", "S:BuildLabel:foobar",
+    "S:TargetHash:SHA256:oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+    "S:RTEMS_DEBUG:0", "S:RTEMS_MULTIPROCESSING:0", "S:RTEMS_POSIX_API:0",
+    "S:RTEMS_PROFILING:0", "S:RTEMS_SMP:0", "B:ScoreRbtreeUnitRbtree",
+    "E:ScoreRbtreeUnitRbtree:N:495132:F:0:D:0.868197",
+    "B:RtemsConfigUnitConfig", "E:RtemsConfigUnitConfig:N:1:F:0:D:0.000291",
+    "B:RtemsTaskValPerf", "M:B:RtemsTaskReqPerfConstruct", "M:V:FullCache",
+    "M:N:6", "M:S:6:0.000006460", "M:MI:0.000006460", "M:P1:0.000006460",
+    "M:Q1:0.000006460", "M:Q2:0.000006460", "M:Q3:0.000006460",
+    "M:P99:0.000006460", "M:MX:0.000006460", "M:MAD:0.000000000",
+    "M:D:0.000908880", "M:E:RtemsTaskReqPerfConstruct:D:0.013368190",
+    "E:RtemsTaskValPerf:N:1007:F:0:D:0.293161",
+    "Z:TestsuitesUnitNoClock0:C:3:N:495175:F:0:D:0.897917",
+    "Y:ReportHash:SHA256:ZNUhinVyKcmR1PY5VSQVJIxxvXK5LMnG9Zf9JU5nOoE=", "",
+    "*** END OF TEST TestsuitesUnitNoClock0 ***", "",
+    "*** BEGIN OF GCOV INFO BASE64 ***",
+    "bmZjZ1I0MEKUAAAAL29wdC9ydGVtcy9ydGVtcy02LXNhZmVzdC0xL2J1aWxkL2JzcC1xdWFsLW9u",
+    "AAAAOi+8CuS72SFYlu6BAAChAcD///8AAAAA", "*** END OF GCOV INFO BASE64 ***"
+    "", "*** BEGIN OF RECORDS BASE64 ***",
+    "bmZjZ1I0MEKUAAAAL29wdC9ydGVtcy9ydGVtcy02LXNhZmVzdC0xL2J1aWxkL2JzcC1xdWFsLW9u",
+    "AAAAOi+8CuS72SFYlu6BAAChAcD///8AAAAA", "*** END OF RECORDS BASE64 ***"
+    "", "*** BEGIN OF RECORDS BASE64 ZLIB ***",
+    "bmZjZ1I0MEKUAAAAL29wdC9ydGVtcy9ydGVtcy02LXNhZmVzdC0xL2J1aWxkL2JzcC1xdWFsLW9u",
+    "AAAAOi+8CuS72SFYlu6BAAChAcD///8AAAAA",
+    "*** END OF RECORDS BASE64 ZLIB ***"
+]
+
+_INCOMPLETE_TEST_SUITE = {
+    "duration": "?",
+    "failed-steps-count": "?",
+    "line-duration": "?",
+    "line-end": "?",
+    "line-failed-steps-count": "?",
+    "line-report-hash": "?",
+    "line-step-count": "?",
+    "report-hash": "?",
+    "report-hash-calculated": "?",
+    "step-count": "?"
+}
+
+
+def _check(old: str, new: str, line: int, error: int) -> dict:
+    report = {}
+    assert _OUTPUT[line] == old
+    _OUTPUT[line] = new
+    augment_report(report, _OUTPUT)
+    assert error < 0 or report["line-parser-error"] == error
+    _OUTPUT[line] = old
+    return report
+
+
+def _info(t_begin: int = 0, t_end: int = 40) -> None:
+    info = {
+        "build": [],
+        "line-begin-of-test": t_begin,
+        "line-build": t_begin + 3,
+        "line-state": t_begin + 2,
+        "line-tools": t_begin + 4,
+        "line-version": t_begin + 1,
+        "name": "TestsuitesUnitNoClock0",
+        "state": "EXPECTED_PASS",
+        "tools": "10.4.0",
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    if t_end >= 0:
+        info["line-end-of-test"] = t_end
+    return info
+
+
+def _test_case_0(tc_begin_0: int = 17) -> None:
+    return {
+        "duration": 0.868197,
+        "failed-steps-count": 0,
+        "line-begin": tc_begin_0,
+        "line-duration": tc_begin_0 + 1,
+        "line-end": tc_begin_0 + 1,
+        "line-failed-steps-count": tc_begin_0 + 1,
+        "line-step-count": tc_begin_0 + 1,
+        "name": "ScoreRbtreeUnitRbtree",
+        "runtime-measurements": [],
+        "step-count": 495132
+    }
+
+
+def _test_case_1(tc_begin_1: int = 19) -> None:
+    return {
+        "duration": 0.000291,
+        "failed-steps-count": 0,
+        "line-begin": tc_begin_1,
+        "line-duration": tc_begin_1 + 1,
+        "line-end": tc_begin_1 + 1,
+        "line-failed-steps-count": tc_begin_1 + 1,
+        "line-step-count": tc_begin_1 + 1,
+        "name": "RtemsConfigUnitConfig",
+        "runtime-measurements": [],
+        "step-count": 1
+    }
+
+
+def _test_case_2(tc_begin_2: int = 21) -> None:
+    return {
+        "duration":
+        0.293161,
+        "failed-steps-count":
+        0,
+        "line-begin":
+        tc_begin_2,
+        "line-duration":
+        tc_begin_2 + 15,
+        "line-end":
+        tc_begin_2 + 15,
+        "line-failed-steps-count":
+        tc_begin_2 + 15,
+        "line-step-count":
+        tc_begin_2 + 15,
+        "name":
+        "RtemsTaskValPerf",
+        "runtime-measurements": [{
+            "duration-sum":
+            0.00090888,
+            "duration-total":
+            0.01336819,
+            "line-begin":
+            tc_begin_2 + 1,
+            "line-end":
+            tc_begin_2 + 14,
+            "mad":
+            0.0,
+            "max":
+            6.46e-06,
+            "min":
+            6.46e-06,
+            "name":
+            "RtemsTaskReqPerfConstruct",
+            "p1":
+            6.46e-06,
+            "p99":
+            6.46e-06,
+            "q1":
+            6.46e-06,
+            "q2":
+            6.46e-06,
+            "q3":
+            6.46e-06,
+            "sample-count":
+            6,
+            "samples":
+            [6.46e-06, 6.46e-06, 6.46e-06, 6.46e-06, 6.46e-06, 6.46e-06],
+            "variant":
+            "FullCache"
+        }],
+        "step-count":
+        1007
+    }
+
+
+def _test_suite(ts_begin: int = 5,
+                tc_begin_0: int = 17,
+                tc_begin_1: int = 19,
+                tc_begin_2: int = 21) -> None:
+    test_suite = {
+        "bsp":
+        "xilinx_zynq_a9_qemu",
+        "build-label":
+        "foobar",
+        "compiler":
+        "10.4.0",
+        "duration":
+        0.897917,
+        "failed-steps-count":
+        0,
+        "line-begin":
+        ts_begin,
+        "line-bsp":
+        ts_begin + 4,
+        "line-build-label":
+        ts_begin + 5,
+        "line-compiler":
+        ts_begin + 2,
+        "line-duration":
+        ts_begin + 32,
+        "line-end":
+        ts_begin + 32,
+        "line-failed-steps-count":
+        ts_begin + 32,
+        "line-platform":
+        ts_begin + 1,
+        "line-report-hash":
+        ts_begin + 33,
+        "line-rtems-debug":
+        ts_begin + 7,
+        "line-rtems-multiprocessing":
+        ts_begin + 8,
+        "line-rtems-posix-api":
+        ts_begin + 9,
+        "line-rtems-profiling":
+        ts_begin + 10,
+        "line-rtems-smp":
+        ts_begin + 11,
+        "line-step-count":
+        ts_begin + 32,
+        "line-target-hash":
+        ts_begin + 6,
+        "line-version":
+        ts_begin + 3,
+        "name":
+        "TestsuitesUnitNoClock0",
+        "platform":
+        "RTEMS",
+        "report-hash":
+        "ZNUhinVyKcmR1PY5VSQVJIxxvXK5LMnG9Zf9JU5nOoE=",
+        "report-hash-calculated":
+        "8pQUQEPgSxq0ks4vr8V6icgamZb31yx1Ung90Ri3Gww=",
+        "rtems-debug":
+        False,
+        "rtems-multiprocessing":
+        False,
+        "rtems-posix-api":
+        False,
+        "rtems-profiling":
+        False,
+        "rtems-smp":
+        False,
+        "step-count":
+        495175,
+        "target-hash":
+        "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [
+            _test_case_0(tc_begin_0),
+            _test_case_1(tc_begin_1),
+            _test_case_2(tc_begin_2)
+        ],
+        "version":
+        "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    return test_suite
+
+
+def _data_ranges(range_begin: int = 43) -> list:
+    return [(range_begin, range_begin + 2), (range_begin + 4, range_begin + 6),
+            (range_begin + 8, range_begin + 10)]
+
+
+def _report(t_begin: int = 0,
+            t_end: int = 40,
+            ts_begin: int = 5,
+            tc_begin_0: int = 17,
+            tc_begin_1: int = 19,
+            tc_begin_2: int = 21,
+            data_begin: int = 42,
+            error: int = -1) -> None:
+    report = {
+        "data-ranges": _data_ranges(data_begin + 1),
+        "info": _info(t_begin, t_end),
+        "line-gcov-info-base64-begin": data_begin,
+        "line-gcov-info-base64-end": data_begin + 3,
+        "line-records-base64-begin": data_begin + 4,
+        "line-records-base64-end": data_begin + 7,
+        "line-records-base64-zlib-begin": data_begin + 8,
+        "line-records-base64-zlib-end": data_begin + 11,
+        "test-suite": _test_suite(ts_begin, tc_begin_0, tc_begin_1, tc_begin_2)
+    }
+    if error >= 0:
+        report["line-parser-error"] = error
+    return report
+
+
+def test_testoutputparser():
+    report = {}
+    augment_report(report, [])
+    assert report == {"data-ranges": [], "info": {}}
+    report = {}
+    augment_report(report, _OUTPUT)
+    assert report == _report()
+
+    report = _check("M:N:6", "M:N:7", 24, 35)
+    expected = _report(error=35, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:MI:0.000006460", "M:MI:0.000006461", 26, 35)
+    expected = _report(error=35, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    _OUTPUT[24] = "M:N:0"
+    report = _check("M:S:6:0.000006460", "M:S:0:0.000006460", 25, -1)
+    _OUTPUT[24] = "M:N:6"
+    expected = _report()
+    expected["test-suite"][
+        "report-hash-calculated"] = "VxsveAz1TOzA1UY0rQzNouamPctwlLI0t3V23R7ahKc="
+    expected["test-suite"]["test-cases"][2]["runtime-measurements"][0][
+        "sample-count"] = 0
+    expected["test-suite"]["test-cases"][2]["runtime-measurements"][0][
+        "samples"] = []
+    assert report == expected
+
+    report = _check("M:V:FullCache", "?", 23, 23)
+    expected = _report(error=23, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:N:6", "?", 24, 24)
+    expected = _report(error=24, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:S:6:0.000006460", "?", 25, 25)
+    expected = _report(error=25, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:MX:0.000006460", "M:MX:0.000006461", 32, 35)
+    expected = _report(error=35, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:P1:0.000006460", "?", 27, 27)
+    expected = _report(error=27, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:Q1:0.000006460", "?", 28, 28)
+    expected = _report(error=28, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:Q2:0.000006460", "?", 29, 29)
+    expected = _report(error=29, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:Q3:0.000006460", "?", 30, 30)
+    expected = _report(error=30, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:P99:0.000006460", "?", 31, 31)
+    expected = _report(error=31, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:MX:0.000006460", "?", 32, 32)
+    expected = _report(error=32, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:MAD:0.000000000", "?", 33, 33)
+    expected = _report(error=33, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:D:0.000908880", "?", 34, 34)
+    expected = _report(error=34, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:E:RtemsTaskReqPerfConstruct:D:0.013368190", "?", 35, 35)
+    expected = _report(error=35, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("M:E:RtemsTaskReqPerfConstruct:D:0.013368190",
+                    "M:E:FooBar:D:0.013368190", 35, 35)
+    expected = _report(error=35, t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    del expected["test-suite"]["test-cases"][2]
+    assert report == expected
+
+    report = _check("*** BEGIN OF TEST TestsuitesUnitNoClock0 ***",
+                    "BEGIN OF TEST XYZ", 0, -1)
+    expected = _report(t_end=-1)
+    expected["info"] = {}
+    del expected["test-suite"]
+    assert report == expected
+
+    report = _check(
+        "*** TEST VERSION: 6.0.0.52f06822b8921ad825cb593b792eab7640e26cde",
+        "foobar", 1, -1)
+
+    report = _check("*** TEST STATE: EXPECTED_PASS", "?", 2, 2)
+    expected = _report(error=2)
+    expected["info"] = {
+        "line-begin-of-test": 0,
+        "line-version": 1,
+        "name": "TestsuitesUnitNoClock0",
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    del expected["test-suite"]
+    assert report == expected
+
+    report = _check("*** TEST BUILD:", "?", 3, 3)
+    expected = _report(error=3)
+    expected["info"] = {
+        "line-begin-of-test": 0,
+        "line-state": 2,
+        "line-version": 1,
+        "name": "TestsuitesUnitNoClock0",
+        "state": "EXPECTED_PASS",
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    del expected["test-suite"]
+    assert report == expected
+
+    report = _check("*** TEST BUILD:",
+                    "*** TEST BUILD: RTEMS_DEBUG, RTEMS_SMP", 3, -1)
+    expected = _report()
+    expected["info"]["build"] = ["RTEMS_DEBUG", "RTEMS_SMP"]
+    assert report == expected
+
+    report = _check("*** TEST TOOLS: 10.4.0", "?", 4, 4)
+    expected = _report(error=4)
+    expected["info"] = {
+        "build": [],
+        "line-begin-of-test": 0,
+        "line-build": 3,
+        "line-state": 2,
+        "line-version": 1,
+        "name": "TestsuitesUnitNoClock0",
+        "state": "EXPECTED_PASS",
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    del expected["test-suite"]
+    assert report == expected
+
+    report = _check("*** END OF TEST TestsuitesUnitNoClock0 ***",
+                    "*** END OF TEST FooBar ***", 40, -1)
+    expected = _report(t_end=-1, error=40)
+    assert report == expected
+
+    report = _check("S:Platform:RTEMS", "?", 6, 6)
+    expected = _report(error=6, t_end=-1)
+    expected["test-suite"] = {
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-step-count": "?",
+        "name": "TestsuitesUnitNoClock0",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": []
+    }
+    assert report == expected
+
+    report = _check("S:Compiler:10.4.0", "?", 7, 7)
+    expected = _report(error=7, t_end=-1)
+    expected["test-suite"] = {
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": []
+    }
+    assert report == expected
+
+    report = _check("S:Version:6.0.0.52f06822b8921ad825cb593b792eab7640e26cde",
+                    "?", 8, 8)
+    expected = _report(error=8, t_end=-1)
+    expected["test-suite"] = {
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": []
+    }
+    assert report == expected
+
+    report = _check("S:BSP:xilinx_zynq_a9_qemu", "?", 9, 9)
+    expected = _report(error=9, t_end=-1)
+    expected["test-suite"] = {
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:BuildLabel:foobar", "?", 10, 10)
+    expected = _report(error=10, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check(
+        "S:TargetHash:SHA256:oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "?", 11, 11)
+    expected = _report(error=11, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:RTEMS_DEBUG:0", "?", 12, 12)
+    expected = _report(error=12, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-target-hash": 11,
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "step-count": "?",
+        "target-hash": "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:RTEMS_MULTIPROCESSING:0", "?", 13, 13)
+    expected = _report(error=13, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-rtems-debug": 12,
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-target-hash": 11,
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "rtems-debug": False,
+        "step-count": "?",
+        "target-hash": "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:RTEMS_POSIX_API:0", "?", 14, 14)
+    expected = _report(error=14, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-rtems-debug": 12,
+        "line-rtems-multiprocessing": 13,
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-target-hash": 11,
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "rtems-debug": False,
+        "rtems-multiprocessing": False,
+        "step-count": "?",
+        "target-hash": "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:RTEMS_PROFILING:0", "?", 15, 15)
+    expected = _report(error=15, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-rtems-debug": 12,
+        "line-rtems-multiprocessing": 13,
+        "line-rtems-posix-api": 14,
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-target-hash": 11,
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "rtems-debug": False,
+        "rtems-multiprocessing": False,
+        "rtems-posix-api": False,
+        "step-count": "?",
+        "target-hash": "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("S:RTEMS_SMP:0", "?", 16, 16)
+    expected = _report(error=16, t_end=-1)
+    expected["test-suite"] = {
+        "bsp": "xilinx_zynq_a9_qemu",
+        "build-label": "foobar",
+        "compiler": "10.4.0",
+        "duration": "?",
+        "failed-steps-count": "?",
+        "line-begin": 5,
+        "line-bsp": 9,
+        "line-build-label": 10,
+        "line-compiler": 7,
+        "line-duration": "?",
+        "line-end": "?",
+        "line-failed-steps-count": "?",
+        "line-report-hash": "?",
+        "line-rtems-debug": 12,
+        "line-rtems-multiprocessing": 13,
+        "line-rtems-posix-api": 14,
+        "line-rtems-profiling": 15,
+        "line-platform": 6,
+        "line-step-count": "?",
+        "line-target-hash": 11,
+        "line-version": 8,
+        "name": "TestsuitesUnitNoClock0",
+        "platform": "RTEMS",
+        "report-hash": "?",
+        "report-hash-calculated": "?",
+        "rtems-debug": False,
+        "rtems-multiprocessing": False,
+        "rtems-posix-api": False,
+        "rtems-profiling": False,
+        "step-count": "?",
+        "target-hash": "oqNHrlFi_jsico5ygHk-OcfeM9oaY3JMw_z6dmF09-U=",
+        "test-cases": [],
+        "version": "6.0.0.52f06822b8921ad825cb593b792eab7640e26cde"
+    }
+    assert report == expected
+
+    report = _check("E:ScoreRbtreeUnitRbtree:N:495132:F:0:D:0.868197", "?", 18,
+                    20)
+    expected = _report(t_end=-1, error=20)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    expected["test-suite"]["test-cases"] = []
+    assert report == expected
+
+    report = _check("Z:TestsuitesUnitNoClock0:C:3:N:495175:F:0:D:0.897917",
+                    "Z:FooBar:C:3:N:495175:F:0:D:0.897917", 37, 37)
+    expected = _report(t_end=-1, error=37)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    assert report == expected
+
+    report = _check("Z:TestsuitesUnitNoClock0:C:3:N:495175:F:0:D:0.897917",
+                    "?", 37, -1)
+    expected = _report(t_end=-1)
+    expected["test-suite"].update(_INCOMPLETE_TEST_SUITE)
+    assert report == expected
+
+    report = _check(
+        "Y:ReportHash:SHA256:ZNUhinVyKcmR1PY5VSQVJIxxvXK5LMnG9Zf9JU5nOoE=",
+        "?", 38, -1)
+    expected = _report(t_end=-1)
+    expected["test-suite"]["line-report-hash"] = "?"
+    expected["test-suite"]["report-hash"] = "?"
+    expected["test-suite"]["report-hash-calculated"] = "?"
+    assert report == expected