testrunner: New

6 files changed, 354 insertions, 1 deletions

diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py
index 22ba8ded..f62fa78d 100644
--- a/rtemsspec/packagebuildfactory.py
+++ b/rtemsspec/packagebuildfactory.py
@@ -29,6 +29,8 @@ from rtemsspec.directorystate import DirectoryState
 from rtemsspec.packagebuild import BuildItemFactory, PackageVariant
 from rtemsspec.reposubset import RepositorySubset
 from rtemsspec.runactions import RunActions
+from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \
+    SubprocessTestRunner
 
 
 def create_build_item_factory() -> BuildItemFactory:
@@ -42,5 +44,9 @@ def create_build_item_factory() -> BuildItemFactory:
     factory.add_constructor("qdp/directory-state/repository", DirectoryState)
     factory.add_constructor("qdp/directory-state/unpacked-archive",
                             DirectoryState)
+    factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner)
+    factory.add_constructor("qdp/test-runner/grmon-manual",
+                            GRMONManualTestRunner)
+    factory.add_constructor("qdp/test-runner/subprocess", SubprocessTestRunner)
     factory.add_constructor("qdp/variant", PackageVariant)
     return factory
diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py
new file mode 100644
index 00000000..94bad5a4
--- /dev/null
+++ b/rtemsspec/testrunner.py
@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" This module provides a build item to run tests. """
+
+# Copyright (C) 2022, 2023 embedded brains GmbH & Co. KG
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+import logging
+import multiprocessing
+import os
+import queue
+import subprocess
+from subprocess import run as subprocess_run
+import tarfile
+import time
+import threading
+from typing import Any, Dict, List, NamedTuple, Union
+
+from rtemsspec.items import Item, ItemGetValueContext
+from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
+from rtemsspec.testoutputparser import augment_report
+
+Report = Dict[str, Union[str, List[str]]]
+
+
+class Executable(NamedTuple):
+    """ This data class represents a test executable. """
+    path: str
+    digest: str
+    timeout: int
+
+
+class TestRunner(BuildItem):
+    """ Runs the tests. """
+
+    def __init__(self, director: PackageBuildDirector, item: Item):
+        super().__init__(director, item)
+        self._executable = "/dev/null"
+        self._executables: List[Executable] = []
+        self.mapper.add_get_value(f"{self.item.type}:/test-executable",
+                                  self._get_test_executable)
+        self.mapper.add_get_value(f"{self.item.type}:/test-executables-grmon",
+                                  self._get_test_executables_grmon)
+
+    def _get_test_executable(self, _ctx: ItemGetValueContext) -> Any:
+        return self._executable
+
+    def _get_test_executables_grmon(self, _ctx: ItemGetValueContext) -> Any:
+        return " \\\n".join(
+            os.path.basename(executable.path)
+            for executable in self._executables)
+
+    def run_tests(self, executables: List[Executable]) -> List[Report]:
+        """
+        Runs the test executables and produces a log file of the test run.
+        """
+        self._executables = executables
+        return []
+
+
+class DummyTestRunner(TestRunner):
+    """ Cannot run the tests. """
+
+    def run_tests(self, _executables: List[Executable]) -> List[Report]:
+        """ Raises an exception.  """
+        raise IOError("this test runner cannot run tests")
+
+
+class GRMONManualTestRunner(TestRunner):
+    """ Provides scripts to run the tests using GRMON. """
+
+    def run_tests(self, executables: List[Executable]) -> List[Report]:
+        super().run_tests(executables)
+        base = self["script-base-path"]
+        dir_name = os.path.basename(base)
+        grmon_name = f"{base}.grmon"
+        shell_name = f"{base}.sh"
+        tar_name = f"{base}.tar.xz"
+        os.makedirs(os.path.dirname(base), exist_ok=True)
+        with tarfile.open(tar_name, "w:xz") as tar_file:
+            with open(grmon_name, "w", encoding="utf-8") as grmon_file:
+                grmon_file.write(self["grmon-script"])
+            tar_file.add(grmon_name, os.path.join(dir_name, "run.grmon"))
+            with open(shell_name, "w", encoding="utf-8") as shell_file:
+                shell_file.write(self["shell-script"])
+            tar_file.add(shell_name, os.path.join(dir_name, "run.sh"))
+            for executable in executables:
+                tar_file.add(
+                    executable.path,
+                    os.path.join(dir_name, os.path.basename(executable.path)))
+        raise IOError(f"Run the tests provided by {tar_name}")
+
+
+def _now_utc() -> str:
+    return datetime.datetime.utcnow().isoformat()
+
+
+class _Job:
+    # pylint: disable=too-few-public-methods
+    def __init__(self, executable: Executable, command: List[str]):
+        self.report: Report = {
+            "executable": executable.path,
+            "executable-sha512": executable.digest,
+            "command-line": command
+        }
+        self.timeout = executable.timeout
+
+
+def _worker(work_queue: queue.Queue, item: BuildItem):
+    with open(os.devnull, "rb") as devnull:
+        while True:
+            try:
+                job = work_queue.get_nowait()
+            except queue.Empty:
+                return
+            logging.info("%s: run: %s", item.uid, job.report["command-line"])
+            job.report["start-time"] = _now_utc()
+            begin = time.monotonic()
+            try:
+                process = subprocess_run(job.report["command-line"],
+                                         check=False,
+                                         stdin=devnull,
+                                         stdout=subprocess.PIPE,
+                                         timeout=job.timeout)
+                stdout = process.stdout.decode("utf-8")
+            except subprocess.TimeoutExpired as timeout:
+                if timeout.stdout is not None:
+                    stdout = timeout.stdout.decode("utf-8")
+                else:
+                    stdout = ""
+            except Exception:  # pylint: disable=broad-exception-caught
+                stdout = ""
+            output = stdout.rstrip().replace("\r\n", "\n").split("\n")
+            augment_report(job.report, output)
+            job.report["output"] = output
+            job.report["duration"] = time.monotonic() - begin
+            logging.debug("%s: done: %s", item.uid, job.report["executable"])
+            work_queue.task_done()
+
+
+class SubprocessTestRunner(TestRunner):
+    """ Runs the tests in subprocesses. """
+
+    def run_tests(self, executables: List[Executable]) -> List[Report]:
+        super().run_tests(executables)
+        work_queue: queue.Queue[_Job] = \
+            queue.Queue()  # pylint: disable=unsubscriptable-object
+        jobs: List[_Job] = []
+        for executable in executables:
+            self._executable = executable.path
+            job = _Job(executable, self["command"])
+            jobs.append(job)
+            work_queue.put(job)
+        for _ in range(min(multiprocessing.cpu_count(), len(executables))):
+            threading.Thread(target=_worker,
+                             args=(work_queue, self),
+                             daemon=True).start()
+        work_queue.join()
+        return [job.report for job in jobs]
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
new file mode 100644
index 00000000..3dfc14e5
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
@@ -0,0 +1,10 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: dummy
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
new file mode 100644
index 00000000..0325b127
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
@@ -0,0 +1,16 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+script-base-path: ${../variant:/prefix-directory}/tests
+grmon-script: |
+  ${.:/test-executables-grmon}
+shell-script: |
+  ${.:/params/x}
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params:
+  x: abc
+qdp-type: test-runner
+test-runner-type: grmon-manual
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
new file mode 100644
index 00000000..ffcd549e
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
@@ -0,0 +1,14 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+command:
+- foo
+- bar
+- ${.:/test-executable}
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: subprocess
+type: qdp
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index aad59c63..2aa54228 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -24,18 +24,23 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+import json
 import logging
 import os
 import pytest
 from pathlib import Path
 import shutil
+import subprocess
 import tarfile
+from typing import NamedTuple
 
 from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext
 from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
     build_item_input, PackageBuildDirector
 from rtemsspec.packagebuildfactory import create_build_item_factory
 from rtemsspec.specverify import verify
+import rtemsspec.testrunner
+from rtemsspec.testrunner import Executable
 from rtemsspec.tests.util import get_and_clear_log
 from rtemsspec.util import run_command
 
@@ -80,7 +85,21 @@ class _TestItem(BuildItem):
         super().__init__(director, item, BuildItemMapper(item, recursive=True))
 
 
-def test_packagebuild(caplog, tmpdir):
+class _Subprocess(NamedTuple):
+    stdout: bytes
+
+
+def _test_runner_subprocess(command, check, stdin, stdout, timeout):
+    if command[2] == "a.exe":
+        raise Exception
+    if command[2] == "b.exe":
+        raise subprocess.TimeoutExpired(command[2], timeout, b"")
+    if command[2] == "c.exe":
+        raise subprocess.TimeoutExpired(command[2], timeout, None)
+    return _Subprocess(b"u\r\nv\nw\n")
+
+
+def test_packagebuild(caplog, tmpdir, monkeypatch):
     tmp_dir = Path(tmpdir)
     item_cache = _create_item_cache(tmp_dir, Path("spec-packagebuild"))
 
@@ -221,3 +240,112 @@ def test_packagebuild(caplog, tmpdir):
     assert not os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c"))
     director.build_package(None, None)
     assert os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c"))
+
+    # Test DummyTestRunner
+    dummy_runner = director["/qdp/test-runner/dummy"]
+    with pytest.raises(IOError):
+        dummy_runner.run_tests([])
+
+    # Test GRMONManualTestRunner
+    grmon_manual_runner = director["/qdp/test-runner/grmon-manual"]
+    exe = tmp_dir / "a.exe"
+    exe.touch()
+    with pytest.raises(IOError):
+        grmon_manual_runner.run_tests([
+            Executable(
+                str(exe), "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA"
+                "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 456)
+        ])
+    with tarfile.open(tmp_dir / "tests.tar.xz", "r:*") as archive:
+        assert archive.getnames() == [
+            "tests/run.grmon", "tests/run.sh", "tests/a.exe"
+        ]
+        with archive.extractfile("tests/run.grmon") as src:
+            assert src.read() == b"a.exe\n"
+        with archive.extractfile("tests/run.sh") as src:
+            assert src.read() == b"abc\n"
+
+    # Test SubprocessTestRunner
+    subprocess_runner = director["/qdp/test-runner/subprocess"]
+    monkeypatch.setattr(rtemsspec.testrunner, "subprocess_run",
+                        _test_runner_subprocess)
+    reports = subprocess_runner.run_tests([
+        Executable(
+            "a.exe", "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA"
+            "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 1),
+        Executable(
+            "b.exe", "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAc"
+            "Au5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==", 2),
+        Executable(
+            "c.exe", "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_"
+            "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 3),
+        Executable(
+            "d.exe", "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_"
+            "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 4)
+    ])
+    monkeypatch.undo()
+    reports[0]["start-time"] = "c"
+    reports[0]["duration"] = 2.
+    reports[1]["start-time"] = "d"
+    reports[1]["duration"] = 3.
+    reports[2]["start-time"] = "e"
+    reports[2]["duration"] = 4.
+    reports[3]["start-time"] = "f"
+    reports[3]["duration"] = 5.
+    assert reports == [{
+        "command-line": ["foo", "bar", "a.exe"],
+        "data-ranges": [],
+        "duration":
+        2.0,
+        "executable":
+        "a.exe",
+        "executable-sha512":
+        "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA_"
+        "l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==",
+        "info": {},
+        "output": [""],
+        "start-time":
+        "c"
+    }, {
+        "command-line": ["foo", "bar", "b.exe"],
+        "data-ranges": [],
+        "duration":
+        3.,
+        "executable":
+        "b.exe",
+        "executable-sha512":
+        "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAcA"
+        "u5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==",
+        "info": {},
+        "output": [""],
+        "start-time":
+        "d"
+    }, {
+        "command-line": ["foo", "bar", "c.exe"],
+        "data-ranges": [],
+        "duration":
+        4.,
+        "executable":
+        "c.exe",
+        "executable-sha512":
+        "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
+        "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
+        "info": {},
+        "output": [""],
+        "start-time":
+        "e"
+    }, {
+        "command-line": ["foo", "bar", "d.exe"],
+        "data-ranges": [],
+        "duration":
+        5.,
+        "executable":
+        "d.exe",
+        "executable-sha512":
+        "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
+        "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
+        "info": {},
+        "output": ["u", "v", "w"],
+        "start-time":
+        "f"
+    }]