runtests: New

13 files changed, 257 insertions, 2 deletions

diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py
index f62fa78d..9b3e7944 100644
--- a/rtemsspec/packagebuildfactory.py
+++ b/rtemsspec/packagebuildfactory.py
@@ -29,6 +29,7 @@ from rtemsspec.directorystate import DirectoryState
 from rtemsspec.packagebuild import BuildItemFactory, PackageVariant
 from rtemsspec.reposubset import RepositorySubset
 from rtemsspec.runactions import RunActions
+from rtemsspec.runtests import RunTests, TestLog
 from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \
     SubprocessTestRunner
 
@@ -40,8 +41,10 @@ def create_build_item_factory() -> BuildItemFactory:
     factory.add_constructor("qdp/build-step/repository-subset",
                             RepositorySubset)
     factory.add_constructor("qdp/build-step/run-actions", RunActions)
+    factory.add_constructor("qdp/build-step/run-tests", RunTests)
     factory.add_constructor("qdp/directory-state/generic", DirectoryState)
     factory.add_constructor("qdp/directory-state/repository", DirectoryState)
+    factory.add_constructor("qdp/directory-state/test-log", TestLog)
     factory.add_constructor("qdp/directory-state/unpacked-archive",
                             DirectoryState)
     factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner)
diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py
new file mode 100644
index 00000000..a0de436d
--- /dev/null
+++ b/rtemsspec/runtests.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" This module provides a build step to run the RTEMS Tester. """
+
+# Copyright (C) 2022 embedded brains GmbH (http://www.embedded-brains.de)
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+import json
+import os
+import logging
+import time
+from typing import Dict, List
+
+from rtemsspec.directorystate import DirectoryState
+from rtemsspec.items import Item
+from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
+from rtemsspec.testrunner import Executable, Report, TestRunner
+
+
+def _now_utc() -> str:
+    return datetime.datetime.utcnow().isoformat()
+
+
+class TestLog(DirectoryState):
+    """ Maintains a test log. """
+
+    def __init__(self, director: PackageBuildDirector, item: Item):
+        super().__init__(director, item)
+        self.reports: List[Report] = []
+
+    def discard(self) -> None:
+        try:
+            with open(self.file, "r", encoding="utf-8") as src:
+                self.reports = json.load(src)["reports"]
+            logging.info("%s: loaded test log: %s", self.uid, self.file)
+        except FileNotFoundError:
+            self.reports = []
+        super().discard()
+
+    def get_reports_by_hash(self) -> Dict[str, Report]:
+        """ Gets the reports by executable hash. """
+        reports_by_hash: Dict[str, Report] = {}
+        for report in self.reports:
+            digest = report["executable-sha512"]
+            assert digest not in reports_by_hash
+            assert isinstance(digest, str)
+            reports_by_hash[digest] = report
+        return reports_by_hash
+
+
+class RunTests(BuildItem):
+    """ Runs the tests. """
+
+    def run(self) -> None:
+        start_time = _now_utc()
+        begin = time.monotonic()
+        log = self.output("log")
+        assert isinstance(log, TestLog)
+        previous_reports_by_hash = log.get_reports_by_hash()
+
+        # Use previous report if the executable hash did not change
+        source = self.input("source")
+        assert isinstance(source, DirectoryState)
+        reports: List[Report] = []
+        executables: List[Executable] = []
+        for path, digest in source.files_and_hashes():
+            if not path.endswith(".exe") or path.endswith(".norun.exe"):
+                continue
+            assert digest
+            report = previous_reports_by_hash.get(digest, None)
+            if report is None:
+                logging.debug("%s: run: %s", self.uid, path)
+                executables.append(Executable(path, digest, 1800))
+            else:
+                logging.debug("%s: use previous report for: %s", self.uid,
+                              path)
+                report["executable"] = path
+                reports.append(report)
+
+        # Run the tests with changed executables
+        if executables:
+            runner = self.input("runner")
+            assert isinstance(runner, TestRunner)
+            reports.extend(runner.run_tests(executables))
+
+        # Save the reports
+        os.makedirs(os.path.dirname(log.file), exist_ok=True)
+        with open(log.file, "w", encoding="utf-8") as dst:
+            data = {
+                "duration": time.monotonic() - begin,
+                "end-time": _now_utc(),
+                "reports": sorted(reports, key=lambda x: x["executable"]),
+                "start-time": start_time
+            }
+            json.dump(data, dst, sort_keys=True, indent=2)
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml
new file mode 100644
index 00000000..109b8ed8
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml
@@ -0,0 +1,19 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/build-directory}/bsp
+directory-state-type: generic
+enabled-by: true
+files:
+- file: a.exe
+  hash: null
+- file: b.exe
+  hash: null
+- file: b.norun.exe
+  hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
index 02c4b721..2af718ef 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
@@ -14,6 +14,8 @@ links:
 - role: build-step
   uid: steps/run-actions
 - role: build-step
+  uid: steps/run-tests
+- role: build-step
   uid: steps/archive
 qdp-type: package-build
 type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml
new file mode 100644
index 00000000..5e165bdf
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml
@@ -0,0 +1,23 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+build-step-type: run-tests
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: |
+  Run the tests.
+enabled-by: run-tests
+links:
+- role: test-procedure
+  uid: ../variant
+- hash: null
+  name: source
+  role: input
+  uid: ../build/bsp
+- hash: null
+  name: runner
+  role: input
+  uid: ../test-runner/test
+- name: log
+  role: output
+  uid: ../test-logs/bsp
+qdp-type: build-step
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml
new file mode 100644
index 00000000..e679c62a
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml
@@ -0,0 +1,15 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/deployment-directory}
+directory-state-type: generic
+enabled-by: true
+files:
+- file: log-empty.json
+  hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml
new file mode 100644
index 00000000..16838238
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml
@@ -0,0 +1,15 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/deployment-directory}
+directory-state-type: test-log
+enabled-by: true
+files:
+- file: test-log-bsp.json
+  hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
new file mode 100644
index 00000000..53dbaa06
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
@@ -0,0 +1,10 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: test
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml
new file mode 100644
index 00000000..23e75a04
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+  uid: root
+- role: spec-refinement
+  spec-key: test-runner-type
+  spec-value: test
+  uid: qdp-test-runner
+spec-description: null
+spec-example: null
+spec-info:
+  dict:
+    attributes: {}
+    description: |
+      This set of attributes specifies a test test runner.
+    mandatory-attributes: all
+spec-name: Subprocess Test Runner Item Type
+spec-type: qdp-test-runner-test
+type: spec
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/a.exe b/rtemsspec/tests/test-files/pkg/build/bsp/a.exe
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/rtemsspec/tests/test-files/pkg/build/bsp/a.exe
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe
new file mode 100644
index 00000000..61780798
--- /dev/null
+++ b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe
@@ -0,0 +1 @@
+b
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index 2aa54228..fa0309d9 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -32,7 +32,7 @@ from pathlib import Path
 import shutil
 import subprocess
 import tarfile
-from typing import NamedTuple
+from typing import List, NamedTuple
 
 from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext
 from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
@@ -40,10 +40,12 @@ from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
 from rtemsspec.packagebuildfactory import create_build_item_factory
 from rtemsspec.specverify import verify
 import rtemsspec.testrunner
-from rtemsspec.testrunner import Executable
+from rtemsspec.testrunner import Executable, Report, TestRunner
 from rtemsspec.tests.util import get_and_clear_log
 from rtemsspec.util import run_command
 
+TestRunner.__test__ = False
+
 
 def _copy_dir(src, dst):
     dst.mkdir(parents=True, exist_ok=True)
@@ -85,6 +87,17 @@ class _TestItem(BuildItem):
         super().__init__(director, item, BuildItemMapper(item, recursive=True))
 
 
+class _TestRunner(TestRunner):
+
+    def run_tests(self, executables: List[Executable]) -> List[Report]:
+        logging.info("executables: %s", executables)
+        super().run_tests(executables)
+        return [{
+            "executable": executable.path,
+            "executable-sha512": executable.digest
+        } for executable in executables]
+
+
 class _Subprocess(NamedTuple):
     stdout: bytes
 
@@ -349,3 +362,20 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
         "start-time":
         "f"
     }]
+
+    # Test RunTests
+    variant["enabled"] = ["run-tests"]
+    factory.add_constructor("qdp/test-runner/test", _TestRunner)
+    build_bsp = director["/qdp/build/bsp"]
+    build_bsp.load()
+    director.build_package(None, None)
+    log = get_and_clear_log(caplog)
+    assert (f"executables: [Executable(path='{build_bsp.directory}"
+            "/a.exe', digest='z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg_SpIdNs6c5H0NE8"
+            "XYXysP-DGNKHfuwvY7kxvUdBeoGlODJ6-SfaPg==', timeout=1800), "
+            f"Executable(path='{build_bsp.directory}/b.exe', "
+            "digest='hopqxuHQKT10-tB_bZWVKz4B09MVPbZ3p12Ad5g_1OMNtr_Im3YIqT-yZ"
+            "GkjOp8aCVctaHqcXaeLID6xUQQKFQ==', timeout=1800)]") in log
+    director.build_package(None, ["/qdp/steps/run-tests"])
+    log = get_and_clear_log(caplog)
+    assert f"use previous report for: {build_bsp.directory}/a.exe"