runtests: Retry run in case of corruption

12 files changed, 76 insertions, 5 deletions

diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py
index a0de436d..2d8f75f9 100644
--- a/rtemsspec/runtests.py
+++ b/rtemsspec/runtests.py
@@ -99,9 +99,7 @@ class RunTests(BuildItem):
 
         # Run the tests with changed executables
         if executables:
-            runner = self.input("runner")
-            assert isinstance(runner, TestRunner)
-            reports.extend(runner.run_tests(executables))
+            reports.extend(self._run_tests(executables))
 
         # Save the reports
         os.makedirs(os.path.dirname(log.file), exist_ok=True)
@@ -113,3 +111,37 @@ class RunTests(BuildItem):
                 "start-time": start_time
             }
             json.dump(data, dst, sort_keys=True, indent=2)
+
+    def _run_tests(self, executables: List[Executable]) -> List[Report]:
+        runner = self.input("runner")
+        assert isinstance(runner, TestRunner)
+        max_run_count = runner["max-retry-count-per-executable"] + 1
+        reports_by_path: Dict[str, Report] = {}
+        while executables and max_run_count:
+            for new_report in runner.run_tests(executables):
+                reports_by_path[new_report["executable"]] = new_report
+            next_executables: List[Executable] = []
+            for executable in executables:
+                report = reports_by_path.get(executable.path, None)
+                if report is None:
+                    logging.warning("%s: no report for: %s", self.uid,
+                                    executable.path)
+                    next_executables.append(executable)
+                    continue
+                if report.get("gcov-info-hash",
+                              "") != report.get("gcov-info-hash-calculated",
+                                                ""):
+                    next_executables.append(executable)
+                    logging.warning("%s: gcov info is corrupt for: %s",
+                                    self.uid, executable.path)
+                    continue
+                test_suite = report.get("test-suite", {})
+                if test_suite.get("report-hash", "") != test_suite.get(
+                        "report-hash-calculated", ""):
+                    next_executables.append(executable)
+                    logging.warning("%s: test suite report is corrupt for: %s",
+                                    self.uid, executable.path)
+                    continue
+            executables = next_executables
+            max_run_count -= 1
+        return list(reports_by_path.values())
diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py
index 94bad5a4..63d92e32 100644
--- a/rtemsspec/testrunner.py
+++ b/rtemsspec/testrunner.py
@@ -34,13 +34,13 @@ from subprocess import run as subprocess_run
 import tarfile
 import time
 import threading
-from typing import Any, Dict, List, NamedTuple, Union
+from typing import Any, Dict, List, NamedTuple
 
 from rtemsspec.items import Item, ItemGetValueContext
 from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
 from rtemsspec.testoutputparser import augment_report
 
-Report = Dict[str, Union[str, List[str]]]
+Report = Dict[str, Any]
 
 
 class Executable(NamedTuple):
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
index 3dfc14e5..218f68bf 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
@@ -4,6 +4,7 @@ copyrights:
 description: Description.
 enabled-by: true
 links: []
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 test-runner-type: dummy
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
index 0325b127..7686556c 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
@@ -9,6 +9,7 @@ copyrights:
 description: Description.
 enabled-by: true
 links: []
+max-retry-count-per-executable: 0
 params:
   x: abc
 qdp-type: test-runner
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
index ffcd549e..13960d48 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
@@ -8,6 +8,7 @@ copyrights:
 description: Description.
 enabled-by: true
 links: []
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 test-runner-type: subprocess
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
index 53dbaa06..46d47a9f 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
@@ -4,6 +4,7 @@ copyrights:
 description: Description.
 enabled-by: true
 links: []
+max-retry-count-per-executable: 3
 params: {}
 qdp-type: test-runner
 test-runner-type: test
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index eff36428..3c002493 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -95,9 +95,30 @@ class _TestItem(BuildItem):
 
 class _TestRunner(TestRunner):
 
+    def __init__(self, director: PackageBuildDirector, item: Item):
+        super().__init__(director, item)
+        self.run_count = 0
+
     def run_tests(self, executables: List[Executable]) -> List[Report]:
         logging.info("executables: %s", executables)
         super().run_tests(executables)
+        self.run_count += 1
+        if self.run_count == 1:
+            return []
+        if self.run_count == 2:
+            return [{
+                "executable": executables[0].path,
+                "gcov-info-hash": "a",
+                "gcov-info-hash-calculated": "b"
+            }]
+        if self.run_count == 3:
+            return [{
+                "executable": executables[0].path,
+                "test-suite": {
+                    "report-hash": "c",
+                    "report-hash-calculated": "d"
+                }
+            }]
         return [{
             "executable": executable.path,
             "executable-sha512": executable.digest
@@ -479,6 +500,9 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
     build_bsp.load()
     director.build_package(None, None)
     log = get_and_clear_log(caplog)
+    assert "no report for" in log
+    assert "gcov info is corrupt for" in log
+    assert "test suite report is corrupt for" in log
     assert (f"executables: [Executable(path='{build_bsp.directory}"
             "/a.exe', digest='z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg_SpIdNs6c5H0NE8"
             "XYXysP-DGNKHfuwvY7kxvUdBeoGlODJ6-SfaPg==', timeout=1800), "
diff --git a/spec-qdp/qdp/test-runner/board-dummy.yml b/spec-qdp/qdp/test-runner/board-dummy.yml
index 961d31ee..548ddfbf 100644
--- a/spec-qdp/qdp/test-runner/board-dummy.yml
+++ b/spec-qdp/qdp/test-runner/board-dummy.yml
@@ -7,6 +7,7 @@ enabled-by: true
 links:
 - role: proxy-member
   uid: board
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 test-runner-type: dummy
diff --git a/spec-qdp/qdp/test-runner/board-grmon-manual.yml b/spec-qdp/qdp/test-runner/board-grmon-manual.yml
index 9a8e4446..7e58cf47 100644
--- a/spec-qdp/qdp/test-runner/board-grmon-manual.yml
+++ b/spec-qdp/qdp/test-runner/board-grmon-manual.yml
@@ -24,6 +24,7 @@ links:
   name: variant
   role: input
   uid: ../variant
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 script-base-path: ${../variant:/build-directory}/grmon
diff --git a/spec-qdp/qdp/test-runner/simulator-dummy.yml b/spec-qdp/qdp/test-runner/simulator-dummy.yml
index a490621b..daa60e0c 100644
--- a/spec-qdp/qdp/test-runner/simulator-dummy.yml
+++ b/spec-qdp/qdp/test-runner/simulator-dummy.yml
@@ -8,6 +8,7 @@ enabled-by:
 links:
 - role: proxy-member
   uid: simulator
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 test-runner-type: dummy
diff --git a/spec-qdp/qdp/test-runner/simulator-sis.yml b/spec-qdp/qdp/test-runner/simulator-sis.yml
index 2a1839fe..b20f5b7b 100644
--- a/spec-qdp/qdp/test-runner/simulator-sis.yml
+++ b/spec-qdp/qdp/test-runner/simulator-sis.yml
@@ -23,6 +23,7 @@ links:
   name: variant
   role: input
   uid: ../variant
+max-retry-count-per-executable: 0
 params: {}
 qdp-type: test-runner
 test-runner-type: subprocess
diff --git a/spec-qdp/spec/qdp-test-runner.yml b/spec-qdp/spec/qdp-test-runner.yml
index 0fd7de54..4ac0da7e 100644
--- a/spec-qdp/spec/qdp-test-runner.yml
+++ b/spec-qdp/spec/qdp-test-runner.yml
@@ -18,6 +18,13 @@ spec-info:
         description: |
           It shall be the test runner description.
         spec-type: str
+      max-retry-count-per-executable:
+        description: |
+          It shall be the maximum count of retries per executable.  With a
+          value of zero exactly one run per executable is performed.  An
+          executable run is retried if no report was produced, the gcov
+          information was corrupt, or the test suite report was corrupt.
+        spec-type: int
       params:
         description: |
           It shall be an optional set of parameters which may be used for