From 7f7363f5f46f0e762bbaee089db9904324ce11f9 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Tue, 15 Sep 2020 11:20:44 +1000 Subject: waf: Implement module dependency checking in the build system - Do not build a test if a dependency is not enabled. - Perform a dependency check and generate an error if an enabled module depends on a disabled module. Closes #4077 --- builder.py | 114 ++++++++++++++++++++++++++++++++++++++++++++-------- freebsd-to-rtems.py | 1 + libbsd.py | 29 +++++++------ waf_libbsd.py | 22 +++++++--- wscript | 81 +++++++------------------------------ 5 files changed, 143 insertions(+), 104 deletions(-) diff --git a/builder.py b/builder.py index fe2158a9..7e79d5f9 100755 --- a/builder.py +++ b/builder.py @@ -42,6 +42,11 @@ import os import re import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser + # # Global controls. # @@ -63,6 +68,9 @@ verboseDetail = 2 verboseMoreDetail = 3 verboseDebug = 4 +BUILDSET_DIR = "buildset" +BUILDSET_DEFAULT = "buildset/default.ini" + def verbose(level=verboseInfo): return verboseLevel >= level @@ -761,8 +769,10 @@ class File(object): state = state and (self.pathComposer == self.pathComposer) state = state and (self.originPath == self.originPath) state = state and (self.forwardConverter == self.forwardConverter) - state = state and (self.self.reverseConverter == self.self.reverseConverter) - state = state and (self.buildSystemComposer == self.buildSystemComposer) + state = state and (self.self.reverseConverter + == self.self.reverseConverter) + state = state and (self.buildSystemComposer + == self.buildSystemComposer) return state def processSource(self, forward): @@ -794,13 +804,12 @@ class Module(object): def __init__(self, manager, name, enabled=True): self.manager = manager self.name = name - self.conditionalOn = "none" self.files = [] self.cpuDependentSourceFiles = {} self.dependencies = [] def __str__(self): - out = [self.name + ': conditional-on=' + self.conditionalOn] + out = [self.name + ':'] if len(self.dependencies) > 0: out += [' Deps: ' + str(len(self.dependencies))] out += [' ' + type(d).__name__ for d in self.dependencies] @@ -981,13 +990,16 @@ class Module(object): NoConverter(), assertSourceFile, sourceFileBuildComposer) - def addTest(self, testFragementComposer): + def addTest(self, testFragementComposer, dependencies=[]): self.files += [ File('user', testFragementComposer.testName, PathComposer(), NoConverter(), NoConverter(), testFragementComposer) ] + self.dependencies += dependencies def addDependency(self, dep): + if not isinstance(dep, str): + raise TypeError('dependencies are a string: %s' % (self.name)) self.dependencies += [dep] @@ -1010,18 +1022,75 @@ class ModuleManager(object): out += [str(self.modules[m]), ''] return os.linesep.join(out) + def _loadIni(self, ini_file): + if not os.path.exists(ini_file): + raise FileNotFoundError('file not found: %s' % (ini_file)) + ini = configparser.ConfigParser() + ini.read(ini_file) + if not ini.has_section('general'): + raise Exception( + "'{}' is missing a general section.".format(ini_file)) + if not ini.has_option('general', 'name'): + raise Exception("'{}' is missing a general/name.".format(ini_file)) + if ini.has_option('general', 'extends'): + extends = ini.get('general', 'extends') + extendfile = None + basepath = os.path.dirname(ini_file) + if os.path.isfile(os.path.join(basepath, extends)): + extendfile = os.path.join(basepath, extends) + elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)): + extendfile = os.path.join(BUILDSET_DIR, extends) + else: + raise Exception( + "'{}': Invalid file given for general/extends:'{}'".format( + ini_file, extends)) + base = self._loadIni(extendfile) + for s in ini.sections(): + if not base.has_section(s): + base.add_section(s) + for o in ini.options(s): + val = ini.get(s, o) + base.set(s, o, val) + ini = base + return ini + + def _checkDependencies(self): + enabled_modules = self.getEnabledModules() + enabled_modules.remove('tests') + for mod in enabled_modules: + if mod not in self.modules: + raise KeyError('enabled module not found: %s' % (mod)) + for dep in self.modules[mod].dependencies: + if dep not in self.modules: + print(type(dep)) + raise KeyError('dependent module not found: %s' % (dep)) + if dep not in enabled_modules: + raise Exception('module "%s" dependency "%s" not enabled' % + (mod, dep)) + def getAllModules(self): if 'modules' in self.configuration: - return self.configuration['modules'] + return sorted(self.configuration['modules']) return [] def getEnabledModules(self): if 'modules-enabled' in self.configuration: - return self.configuration['modules-enabled'] + return sorted(self.configuration['modules-enabled']) return [] def addModule(self, module): - self.modules[module.name] = module + name = module.name + if name in self.modules: + raise KeyError('module already added: %' % (name)) + self.modules[name] = module + if 'modules' not in self.configuration: + self.configuration['modules'] = [] + if 'modules-enabled' not in self.configuration: + self.configuration['modules-enabled'] = [] + self.configuration['modules'] += [name] + self.configuration['modules-enabled'] += [name] + self.configuration['modules'].sort() + self.configuration['modules-enabled'].sort def processSource(self, direction): if verbose(verboseDetail): @@ -1035,15 +1104,6 @@ class ModuleManager(object): def getConfiguration(self): return copy.deepcopy(self.configuration) - def updateConfiguration(self, config): - self.configuration.update(config) - - def setModuleConfigiuration(self): - mods = sorted(self.modules.keys()) - self.configuration['modules'] = mods - # Enabled modules are overwritten by config file. Default to all. - self.configuration['modules-enabled'] = mods - def generateBuild(self, only_enabled=True): modules_to_process = self.getEnabledModules() # Used for copy between FreeBSD and RTEMS @@ -1053,6 +1113,7 @@ class ModuleManager(object): if m not in self.modules: raise KeyError('enabled module not registered: %s' % (m)) self.modules[m].generate() + self._checkDependencies() def duplicateCheck(self): dups = [] @@ -1068,6 +1129,25 @@ class ModuleManager(object): dups += [(m, mod, fm.getPath(), fm.getSpace())] return dups + def loadConfig(self, config=BUILDSET_DEFAULT): + if 'name' in self.configuration: + raise KeyError('configuration already loaded: %s (%s)' % \ + (self.configuration['name'], config)) + ini = self._loadIni(config) + self.configuration['name'] = ini.get('general', 'name') + self.configuration['modules-enabled'] = [] + mods = [] + if ini.has_section('modules'): + mods = ini.options('modules') + for mod in mods: + if ini.getboolean('modules', mod): + self.configuration['modules-enabled'].append(mod) + + def getName(self): + if 'name' not in self.configuration: + raise KeyError('configuration not loaded') + return self.configuration['name'] + def setGenerators(self): self.generator['convert'] = Converter self.generator['no-convert'] = NoConverter diff --git a/freebsd-to-rtems.py b/freebsd-to-rtems.py index 8f66e589..cda3ff3a 100755 --- a/freebsd-to-rtems.py +++ b/freebsd-to-rtems.py @@ -150,6 +150,7 @@ try: build = builder.ModuleManager() libbsd.load(build) + build.loadConfig() build.generateBuild(only_enabled=False) dups = build.duplicateCheck() diff --git a/libbsd.py b/libbsd.py index d86cdd39..58438100 100644 --- a/libbsd.py +++ b/libbsd.py @@ -960,7 +960,7 @@ class dev_usb_controller(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/controller/ohci.h', @@ -999,7 +999,7 @@ class dev_usb_input(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/input/usb_rdesc.h', @@ -1027,7 +1027,7 @@ class dev_usb_net(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/net/if_auereg.h', @@ -1077,7 +1077,7 @@ class dev_usb_quirk(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/quirk/usb_quirk.h', @@ -1100,7 +1100,7 @@ class dev_usb_serial(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/serial/uftdi_reg.h', @@ -1145,7 +1145,7 @@ class dev_usb_storage(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceSourceFiles( [ 'sys/dev/usb/storage/umass.c', @@ -1163,7 +1163,7 @@ class dev_usb_controller_bbb(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/arm/ti/ti_cpuid.h', @@ -1200,7 +1200,7 @@ class dev_usb_wlan(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/usb/wlan/if_rsureg.h', @@ -1250,7 +1250,7 @@ class dev_wlan_rtwn(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['dev_usb']) + self.addDependency('dev_usb') self.addKernelSpaceHeaderFiles( [ 'sys/dev/rtwn/if_rtwn_beacon.h', @@ -2875,7 +2875,7 @@ class nfsv2(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['rpc_user']) + self.addDependency('rpc_user') self.addRTEMSUserSourceFiles( [ 'nfsclient/mount_prot_xdr.c', @@ -3276,7 +3276,7 @@ class crypto_openssl(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['user_space']) + self.addDependency('user_space') self.addUserSpaceHeaderFiles( [ 'crypto/openssl/crypto/aes/aes_locl.h', @@ -4258,7 +4258,7 @@ class usr_bin_openssl(builder.Module): def generate(self): mm = self.manager - self.addDependency(mm['crypto_openssl']) + self.addDependency('crypto_openssl') self.addUserSpaceHeaderFiles( [ 'crypto/openssl/apps/apps.h', @@ -5328,7 +5328,8 @@ class tests(builder.Module): def generate(self): mm = self.manager self.addTest(mm.generator['test']('epoch01', ['test_main'], extraLibs = ['rtemstest'])) - self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True)) + self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True), + ['nfsv2']) self.addTest(mm.generator['test']('foobarclient', ['test_main'], runTest = False, netTest = True)) self.addTest(mm.generator['test']('foobarserver', ['test_main'], @@ -5498,7 +5499,5 @@ def load(mm): mm.addModule(tests(mm)) - mm.setModuleConfigiuration() - # XXX TODO Check that no file is also listed in empty # XXX TODO Check that no file in in two modules diff --git a/waf_libbsd.py b/waf_libbsd.py index c457cbbd..558690b1 100644 --- a/waf_libbsd.py +++ b/waf_libbsd.py @@ -46,6 +46,10 @@ import builder import rtems_waf.rtems as rtems + +BUILDSET_DIR = builder.BUILDSET_DIR +BUILDSET_DEFAULT = builder.BUILDSET_DEFAULT + windows = os.name == 'nt' if windows: @@ -138,16 +142,22 @@ class Builder(builder.ModuleManager): self.data = {} - for mn in self.getEnabledModules(): + enabled_modules = self.getEnabledModules() + for mn in enabled_modules: m = self[mn] - if m.conditionalOn == "none": + enabled = True + for dep in m.dependencies: + if dep not in enabled_modules: + enabled = False + break + if enabled: for f in m.files: _dataInsert(self.data, 'all', f.getSpace(), f.getFragment()) - for cpu, files in sorted(m.cpuDependentSourceFiles.items()): - for f in files: - _dataInsert(self.data, cpu, f.getSpace(), - f.getFragment()) + for cpu, files in sorted(m.cpuDependentSourceFiles.items()): + for f in files: + _dataInsert(self.data, cpu, f.getSpace(), + f.getFragment()) # Start here if you need to understand self.data. Add 'True or' if self.trace: diff --git a/wscript b/wscript index 99a60978..745ee6f8 100644 --- a/wscript +++ b/wscript @@ -45,76 +45,24 @@ except: import sys sys.exit(1) -import libbsd -import waf_libbsd import os.path import runpy import sys -try: - import configparser -except ImportError: - import ConfigParser as configparser -import waflib.Options -builders = {} - -BUILDSET_DIR = "buildset" -BUILDSET_DEFAULT = "buildset/default.ini" - - -def load_ini(conf, f): - ini = configparser.ConfigParser() - ini.read(f) - if not ini.has_section('general'): - conf.fatal("'{}' is missing a general section.".format(f)) - if not ini.has_option('general', 'name'): - conf.fatal("'{}' is missing a general/name.".format(f)) - if ini.has_option('general', 'extends'): - extends = ini.get('general', 'extends') - extendfile = None - basepath = os.path.dirname(f) - if os.path.isfile(os.path.join(basepath, extends)): - extendfile = os.path.join(basepath, extends) - elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)): - extendfile = os.path.join(BUILDSET_DIR, extends) - else: - conf.fatal( - "'{}': Invalid file given for general/extends:'{}'".format( - f, extends)) - base = load_ini(conf, extendfile) - for s in ini.sections(): - if not base.has_section(s): - base.add_section(s) - for o in ini.options(s): - val = ini.get(s, o) - base.set(s, o, val) - ini = base - return ini - - -def load_config(conf, f): - ini = load_ini(conf, f) - config = {} +import waflib.Options - config['name'] = ini.get('general', 'name') +import libbsd +import waf_libbsd - config['modules-enabled'] = [] - mods = [] - if ini.has_section('modules'): - mods = ini.options('modules') - for mod in mods: - if ini.getboolean('modules', mod): - config['modules-enabled'].append(mod) - return config +builders = {} def update_builders(ctx, buildset_opt): global builders builders = {} - buildsets = [] if buildset_opt == []: - buildset_opt.append(BUILDSET_DEFAULT) + buildset_opt.append(waf_libbsd.BUILDSET_DEFAULT) for bs in buildset_opt: if os.path.isdir(bs): for f in os.listdir(bs): @@ -123,15 +71,16 @@ def update_builders(ctx, buildset_opt): else: for f in bs.split(','): buildsets += [f] - for bs in buildsets: - builder = waf_libbsd.Builder() - libbsd.load(builder) - bsconfig = load_config(ctx, bs) - bsname = bsconfig['name'] - builder.updateConfiguration(bsconfig) - builder.generate(rtems_version) - builders[bsname] = builder + try: + builder = waf_libbsd.Builder() + libbsd.load(builder) + builder.loadConfig(bs) + builder.generate(rtems_version) + except Exception as exc: + raise + ctx.fatal(str(exc)) + builders[builder.getName()] = builder def bsp_init(ctx, env, contexts): @@ -250,7 +199,7 @@ def configure(conf): conf.env.OPTIMIZATION = conf.options.optimization conf.env.BUILDSET = conf.options.buildset if len(conf.env.BUILDSET) == 0: - conf.env.BUILDSET += [BUILDSET_DEFAULT] + conf.env.BUILDSET += [waf_libbsd.BUILDSET_DEFAULT] update_builders(conf, conf.env.BUILDSET) rtems.configure(conf, bsp_configure) -- cgit v1.2.3