testsuite: Fix rtems-test-check not excluding tests.

The include file handling was broken.

Add a test configuration data README.

Closes #2981.

3 files changed, 125 insertions, 14 deletions

diff --git a/testsuites/README.testdata b/testsuites/README.testdata
new file mode 100644
index 0000000000..74a64e6b4e
--- /dev/null
+++ b/testsuites/README.testdata
@@ -0,0 +1,72 @@
+RTEMS Testsuite Configuration
+=============================
+
+The RTEMS Testsuite lets you configure the tests for a BSP. Every BSP can have
+a test configuration data file and this file is read when building the
+tests. The test configuration data can control what tests are build and now the
+tests are built.
+
+The test configuration data files have a `.tcfg` file extension. You can
+include other test configuration data files reducing repeated test
+sequences. There is also a global test configuration data file for global
+configurations.
+
+Command
+-------
+
+The build system invokes the test check tool to determine what it does.
+
+ rtems-test-check mode bsp testconfig includepaths test[s]
+
+Mode (mode)
+~~~~~~~~~~~
+
+The check command modes are:
+
+ 1. `exclude`: The input list of tests is checked against the excluded tests
+    and the tests that are not excluded are printed.
+
+ 2. `flags': The test build flags are returned. These flags can enumerate a
+    test in a specific way.
+
+BSP (bsp)
+~~~~~~~~~
+
+The name of the BSP.
+
+Test Configuration (testconfig)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The path to the BSP's test configration file. If the file does not exist the
+input list of tests is returned and there are no special build flags.
+
+Include Paths (inputpaths)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A colon (`:`) separated list of paths test configuration files are search
+for.
+
+Tests (test[s])
+~~~~~~~~~~~~~~~
+
+If the mode is `exclude` this argument is a list of tests to be checked. If the
+mode is `flags` a single test is required and the flags for the test are returned.
+
+File Format
+-----------
+
+The file is an ASCII text file of lines. And text after and including the `#`
+character is removed. Empty lines are ignored.
+
+A line is either the `include` directive or a test state. The states are:
+
+ exclude       : Exclude the test from being built.
+ expected-fail : The test is built but expected to fail.
+ user-input    : The test requires user input and may be aborted when running
+                 the tests.
+ indeterminate : The test result is indeterminate. This means the test may pass
+                 or may fail therefore it cannot be included in the regression
+                 results.
+ benchmark     : The test is a benchmark and maybe is aborted when running the
+                 tests because of the load and time the test may take to run.
+                 Performance changes are currently not viewed as a regression.
diff --git a/testsuites/automake/subdirs.am b/testsuites/automake/subdirs.am
index 70a7ecb78c..63b1da1049 100644
--- a/testsuites/automake/subdirs.am
+++ b/testsuites/automake/subdirs.am
@@ -15,7 +15,8 @@ all-local:
 	  target="all"; \
 	fi; \
 	tcheck="$(top_srcdir)/../../tools/build/rtems-test-check-py"; \
-	tdata="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
+	tdata="$(RTEMS_BSP)-testsuite.tcfg"; \
+	tincludes="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom:$(top_srcdir)/.."; \
 	if test -f "$$tdata"; then \
 	  vtdata="$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
 	else \
@@ -23,7 +24,7 @@ all-local:
 	fi; \
 	echo "BSP Testsuite Data: $$vtdata"; \
 	if test -f $$tcheck; then \
-	  list=`$$tcheck exclude $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $(_SUBDIRS)`; \
+	  list=`$$tcheck exclude $(RTEMS_BSP) $$tdata $$tincludes $(_SUBDIRS)`; \
 	else \
 	  list=$(_SUBDIRS); \
 	fi; \
@@ -31,7 +32,7 @@ all-local:
 	  echo "Making $$target in $$subdir"; \
 	  if test "$$subdir" != "."; then \
 	    if test -f $$tcheck; then \
-	      test_FLAGS=`$$tcheck flags $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $$subdir`; \
+	      test_FLAGS=`$$tcheck flags $(RTEMS_BSP) $$tdata $$tincludes $$subdir`; \
 	    fi; \
 	    local_target="$$target"; \
 	    if test -z "$$test_FLAGS"; then \
diff --git a/tools/build/rtems-test-check-py b/tools/build/rtems-test-check-py
index 04630d25f7..5cf957049f 100755
--- a/tools/build/rtems-test-check-py
+++ b/tools/build/rtems-test-check-py
@@ -16,6 +16,16 @@ def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
 #
+# Search the include paths for a file.
+#
+def find_testdata(paths, name):
+    for p in paths:
+        fn = os.path.join(p, name)
+        if os.path.exists(fn):
+            return fn
+    return None
+
+#
 # Arguments. Keep it simple.
 #
 if len(sys.argv) < 4:
@@ -23,11 +33,18 @@ if len(sys.argv) < 4:
     print('INVALID-TEST-DATA')
     sys.exit(2)
 
-mode = sys.argv[1]
-testconfig = [sys.argv[2]]
-includepath = sys.argv[3]
-bsp = sys.argv[4]
-tests = sys.argv[5:]
+verbose = False
+args = 0
+
+if sys.argv[1] == '-v':
+    verbose = True
+    args = 1
+
+mode = sys.argv[args + 1]
+bsp = sys.argv[args + 2]
+includepaths = sys.argv[args + 4].split(':')
+testconfig = [find_testdata(includepaths, sys.argv[args + 3])]
+tests = sys.argv[args + 5:]
 
 #
 # Handle the modes.
@@ -47,8 +64,8 @@ else:
 #
 # Common RTEMS testsuite configuration. Load first.
 #
-rtems_testdata = os.path.join(includepath, 'testdata', 'rtems.tcfg')
-if os.path.exists(rtems_testdata):
+rtems_testdata = find_testdata(includepaths, os.path.join('testdata', 'rtems.tcfg'))
+if rtems_testdata is not None:
     testconfig.insert(0, rtems_testdata)
 
 states = ['exclude',
@@ -63,6 +80,13 @@ defines = { 'expected-fail' : '-DTEST_STATE_EXPECTED_FAIL=1',
 output = []
 testdata = {}
 
+if verbose:
+    eprint('mode: %s' % (mode))
+    eprint('testconfig: %s' % (', '.join(testconfig)))
+    eprint('includepaths: %s' % (includepaths))
+    eprint('bsp: %s' % (bsp))
+    eprint('tests: %s' % (', '.join(tests)))
+
 def clean(line):
     line = line[0:-1]
     b = line.find('#')
@@ -73,10 +97,16 @@ def clean(line):
 #
 # Load the test data.
 #
-for tc in range(0, len(testconfig)):
-    if not os.path.exists(testconfig[tc]):
+while len(testconfig):
+    tc = testconfig[0]
+    testconfig.remove(tc)
+    if verbose:
+        eprint('reading: %s' % (tc))
+    if not os.path.exists(tc):
+        if verbose:
+            eprint('%s: not found' % (tc))
         continue
-    with open(testconfig[tc]) as f:
+    with open(tc) as f:
         tdata = [clean(l) for l in f.readlines()]
     lc = 0
     for line in tdata:
@@ -84,6 +114,8 @@ for tc in range(0, len(testconfig)):
         ls = [s.strip() for s in line.split(':')]
         if len(line) == 0:
             continue
+        if verbose:
+            eprint('%4d: %s' % (lc, line))
         if len(ls) != 2:
             eprint('error: syntax error: %s:%d' % (tc, lc))
             print('INVALID-TEST-DATA')
@@ -91,7 +123,13 @@ for tc in range(0, len(testconfig)):
         state = ls[0]
         test = ls[1]
         if state == 'include':
-            testconfig.insert(tc, test)
+            td = find_testdata(includepaths, test)
+            if td is None:
+                eprint('error: include not found: %s:%d' % (tc, lc))
+                print('INVALID-TEST-DATA')
+            testconfig.insert(0, td)
+            if verbose:
+                eprint('include: %s' % (', '.join(testconfig)))
         elif state in states:
             if state not in testdata:
                 testdata[state] = [test]