From 18f63c0004cc3348bc785a642da82d2a3d46db5d Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Sun, 16 Apr 2017 09:51:57 +1000 Subject: testsuite: Fix rtems-test-check not excluding tests. The include file handling was broken. Add a test configuration data README. Closes #2981. --- testsuites/README.testdata | 72 +++++++++++++++++++++++++++++++++++++++++ testsuites/automake/subdirs.am | 7 ++-- tools/build/rtems-test-check-py | 60 +++++++++++++++++++++++++++------- 3 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 testsuites/README.testdata diff --git a/testsuites/README.testdata b/testsuites/README.testdata new file mode 100644 index 0000000000..74a64e6b4e --- /dev/null +++ b/testsuites/README.testdata @@ -0,0 +1,72 @@ +RTEMS Testsuite Configuration +============================= + +The RTEMS Testsuite lets you configure the tests for a BSP. Every BSP can have +a test configuration data file and this file is read when building the +tests. The test configuration data can control what tests are build and now the +tests are built. + +The test configuration data files have a `.tcfg` file extension. You can +include other test configuration data files reducing repeated test +sequences. There is also a global test configuration data file for global +configurations. + +Command +------- + +The build system invokes the test check tool to determine what it does. + + rtems-test-check mode bsp testconfig includepaths test[s] + +Mode (mode) +~~~~~~~~~~~ + +The check command modes are: + + 1. `exclude`: The input list of tests is checked against the excluded tests + and the tests that are not excluded are printed. + + 2. `flags': The test build flags are returned. These flags can enumerate a + test in a specific way. + +BSP (bsp) +~~~~~~~~~ + +The name of the BSP. + +Test Configuration (testconfig) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The path to the BSP's test configration file. If the file does not exist the +input list of tests is returned and there are no special build flags. + +Include Paths (inputpaths) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A colon (`:`) separated list of paths test configuration files are search +for. + +Tests (test[s]) +~~~~~~~~~~~~~~~ + +If the mode is `exclude` this argument is a list of tests to be checked. If the +mode is `flags` a single test is required and the flags for the test are returned. + +File Format +----------- + +The file is an ASCII text file of lines. And text after and including the `#` +character is removed. Empty lines are ignored. + +A line is either the `include` directive or a test state. The states are: + + exclude : Exclude the test from being built. + expected-fail : The test is built but expected to fail. + user-input : The test requires user input and may be aborted when running + the tests. + indeterminate : The test result is indeterminate. This means the test may pass + or may fail therefore it cannot be included in the regression + results. + benchmark : The test is a benchmark and maybe is aborted when running the + tests because of the load and time the test may take to run. + Performance changes are currently not viewed as a regression. diff --git a/testsuites/automake/subdirs.am b/testsuites/automake/subdirs.am index 70a7ecb78c..63b1da1049 100644 --- a/testsuites/automake/subdirs.am +++ b/testsuites/automake/subdirs.am @@ -15,7 +15,8 @@ all-local: target="all"; \ fi; \ tcheck="$(top_srcdir)/../../tools/build/rtems-test-check-py"; \ - tdata="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \ + tdata="$(RTEMS_BSP)-testsuite.tcfg"; \ + tincludes="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom:$(top_srcdir)/.."; \ if test -f "$$tdata"; then \ vtdata="$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \ else \ @@ -23,7 +24,7 @@ all-local: fi; \ echo "BSP Testsuite Data: $$vtdata"; \ if test -f $$tcheck; then \ - list=`$$tcheck exclude $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $(_SUBDIRS)`; \ + list=`$$tcheck exclude $(RTEMS_BSP) $$tdata $$tincludes $(_SUBDIRS)`; \ else \ list=$(_SUBDIRS); \ fi; \ @@ -31,7 +32,7 @@ all-local: echo "Making $$target in $$subdir"; \ if test "$$subdir" != "."; then \ if test -f $$tcheck; then \ - test_FLAGS=`$$tcheck flags $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $$subdir`; \ + test_FLAGS=`$$tcheck flags $(RTEMS_BSP) $$tdata $$tincludes $$subdir`; \ fi; \ local_target="$$target"; \ if test -z "$$test_FLAGS"; then \ diff --git a/tools/build/rtems-test-check-py b/tools/build/rtems-test-check-py index 04630d25f7..5cf957049f 100755 --- a/tools/build/rtems-test-check-py +++ b/tools/build/rtems-test-check-py @@ -15,6 +15,16 @@ import sys def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) +# +# Search the include paths for a file. +# +def find_testdata(paths, name): + for p in paths: + fn = os.path.join(p, name) + if os.path.exists(fn): + return fn + return None + # # Arguments. Keep it simple. # @@ -23,11 +33,18 @@ if len(sys.argv) < 4: print('INVALID-TEST-DATA') sys.exit(2) -mode = sys.argv[1] -testconfig = [sys.argv[2]] -includepath = sys.argv[3] -bsp = sys.argv[4] -tests = sys.argv[5:] +verbose = False +args = 0 + +if sys.argv[1] == '-v': + verbose = True + args = 1 + +mode = sys.argv[args + 1] +bsp = sys.argv[args + 2] +includepaths = sys.argv[args + 4].split(':') +testconfig = [find_testdata(includepaths, sys.argv[args + 3])] +tests = sys.argv[args + 5:] # # Handle the modes. @@ -47,8 +64,8 @@ else: # # Common RTEMS testsuite configuration. Load first. # -rtems_testdata = os.path.join(includepath, 'testdata', 'rtems.tcfg') -if os.path.exists(rtems_testdata): +rtems_testdata = find_testdata(includepaths, os.path.join('testdata', 'rtems.tcfg')) +if rtems_testdata is not None: testconfig.insert(0, rtems_testdata) states = ['exclude', @@ -63,6 +80,13 @@ defines = { 'expected-fail' : '-DTEST_STATE_EXPECTED_FAIL=1', output = [] testdata = {} +if verbose: + eprint('mode: %s' % (mode)) + eprint('testconfig: %s' % (', '.join(testconfig))) + eprint('includepaths: %s' % (includepaths)) + eprint('bsp: %s' % (bsp)) + eprint('tests: %s' % (', '.join(tests))) + def clean(line): line = line[0:-1] b = line.find('#') @@ -73,10 +97,16 @@ def clean(line): # # Load the test data. # -for tc in range(0, len(testconfig)): - if not os.path.exists(testconfig[tc]): +while len(testconfig): + tc = testconfig[0] + testconfig.remove(tc) + if verbose: + eprint('reading: %s' % (tc)) + if not os.path.exists(tc): + if verbose: + eprint('%s: not found' % (tc)) continue - with open(testconfig[tc]) as f: + with open(tc) as f: tdata = [clean(l) for l in f.readlines()] lc = 0 for line in tdata: @@ -84,6 +114,8 @@ for tc in range(0, len(testconfig)): ls = [s.strip() for s in line.split(':')] if len(line) == 0: continue + if verbose: + eprint('%4d: %s' % (lc, line)) if len(ls) != 2: eprint('error: syntax error: %s:%d' % (tc, lc)) print('INVALID-TEST-DATA') @@ -91,7 +123,13 @@ for tc in range(0, len(testconfig)): state = ls[0] test = ls[1] if state == 'include': - testconfig.insert(tc, test) + td = find_testdata(includepaths, test) + if td is None: + eprint('error: include not found: %s:%d' % (tc, lc)) + print('INVALID-TEST-DATA') + testconfig.insert(0, td) + if verbose: + eprint('include: %s' % (', '.join(testconfig))) elif state in states: if state not in testdata: testdata[state] = [test] -- cgit v1.2.3