PR 2108, PR 2109. Add --jobs and --always-clean.

Refactor the options handling in defaults.py to allow the --jobs
option have varing specific parameters. The option supports 'none',
'max' and 'half' or a fraction to divide the number of CPUs or
an integer value which is the number of jobs. The --no-smp has
been removed.

The host specific modules have been changed to set the number of
CPUs in the defaults table.

Fixed the --keep-going to clean up is --always-clean is provided
even if the build has an error.

1 files changed, 204 insertions, 180 deletions

diff --git a/source-builder/sb/defaults.py b/source-builder/sb/defaults.py
index 469ed88..4f38e73 100644
--- a/source-builder/sb/defaults.py
+++ b/source-builder/sb/defaults.py
@@ -251,50 +251,162 @@ FFLAGS="${FFLAGS:-%optflags}" ; export FFLAGS ;
 class command_line:
     """Process the command line in a common way for all Tool Builder commands."""
 
-    #
-    # The define and if it is a path and needs conversion.
-    #
-    _long_opts = { '--prefix'         : ('_prefix', True),
-                   '--prefixbase'     : ('_prefixbase', True),
-                   '--topdir'         : ('_topdir', True),
-                   '--configdir'      : ('_configdir', True),
-                   '--builddir'       : ('_builddir', True),
-                   '--sourcedir'      : ('_sourcedir', True),
-                   '--tmppath'        : ('_tmppath', True),
-                   '--log'            : ('_logfile', False),
-                   '--url'            : ('_url_base', False),
-                   '--targetcflags'   : ('_targetcflags', False),
-                   '--targetcxxflags' : ('_targetcxxflags', False),
-                   '--libstdcxxflags' : ('_libstdcxxflags', False) }
-
-    _long_true_opts = { '--force'    : '_force',
-                        '--trace'    : '_trace',
-                        '--dry-run'  : '_dry_run',
-                        '--warn-all' : '_warn_all',
-                        '--no-clean' : '_no_clean',
-                        '--no-smp'   : '_no_smp',
-                        '--rebuild'  : '_rebuild' }
-
-    _target_triplets = { '--host'   : '_host',
-                         '--build'  : '_build',
-                         '--target' : '_target' }
+    def __init__(self, argv, optargs):
+        self._long_opts = {
+            # key                 macro              handler            param  defs    init
+            '--prefix'         : ('_prefix',         self._lo_path,     True,  None,  False),
+            '--topdir'         : ('_topdir',         self._lo_path,     True,  None,  False),
+            '--configdir'      : ('_configdir',      self._lo_path,     True,  None,  False),
+            '--builddir'       : ('_builddir',       self._lo_path,     True,  None,  False),
+            '--sourcedir'      : ('_sourcedir',      self._lo_path,     True,  None,  False),
+            '--tmppath'        : ('_tmppath',        self._lo_path,     True,  None,  False),
+            '--jobs'           : ('_jobs',           self._lo_jobs,     True,  'max', True),
+            '--log'            : ('_logfile',        self._lo_string,   True,  None,  False),
+            '--url'            : ('_url_base',       self._lo_string,   True,  None,  False),
+            '--targetcflags'   : ('_targetcflags',   self._lo_string,   True,  None,  False),
+            '--targetcxxflags' : ('_targetcxxflags', self._lo_string,   True,  None,  False),
+            '--libstdcxxflags' : ('_libstdcxxflags', self._lo_string,   True,  None,  False),
+            '--force'          : ('_force',          self._lo_bool,     False, '0',   True),
+            '--quiet'          : ('_quiet',          self._lo_bool,     False, '0',   True),
+            '--trace'          : ('_trace',          self._lo_bool,     False, '0',   True),
+            '--dry-run'        : ('_dry_run',        self._lo_bool,     False, '0',   True),
+            '--warn-all'       : ('_warn_all',       self._lo_bool,     False, '0',   True),
+            '--no-clean'       : ('_no_clean',       self._lo_bool,     False, '0',   True),
+            '--keep-going'     : ('_keep_going',     self._lo_bool,     False, '0',   True),
+            '--always-clean'   : ('_always_clean',   self._lo_bool,     False, '0',   True),
+            '--host'           : ('_host',           self._lo_triplets, True,  None,  False),
+            '--build'          : ('_build',          self._lo_triplets, True,  None,  False),
+            '--target'         : ('_target',         self._lo_triplets, True,  None,  False),
+            '--help'           : (None,              self._lo_help,     False, None,  False)
+            }
+
+        self.command_path = path.dirname(argv[0])
+        if len(self.command_path) == 0:
+            self.command_path = '.'
+        self.command_name = path.basename(argv[0])
+        self.argv = argv
+        self.args = argv[1:]
+        self.optargs = optargs
+        self.defaults = {}
+        self.defaults['_sbdir'] = ('dir', 'required', path.shell(self.command_path))
+        self.opts = { 'params' : [] }
+        for lo in self._long_opts:
+            self.opts[lo[2:]] = self._long_opts[lo][3]
+            if self._long_opts[lo][4]:
+                self.defaults[self._long_opts[lo][0]] = ('none', 'none', self._long_opts[lo][3])
+        self._process()
+
+    def __str__(self):
+        def _dict(dd):
+            s = ''
+            ddl = dd.keys()
+            ddl.sort()
+            for d in ddl:
+                s += '  ' + d + ': ' + str(dd[d]) + '\n'
+            return s
+
+        s = 'command: ' + self.command() + \
+            '\nargs: ' + str(self.args) + \
+            '\nopts:\n' + _dict(self.opts)
+
+        return s
+
+    def _lo_string(self, opt, macro, value):
+        if value is None:
+            raise error.general('option requires a value: %s' % (opt))
+        self.opts[opt[2:]] = value
+        self.defaults[macro] = ('none', 'none', value)
+
+    def _lo_path(self, opt, macro, value):
+        if value is None:
+            raise error.general('option requires a path: %s' % (opt))
+        value = path.shell(value)
+        self.opts[opt[2:]] = value
+        self.defaults[macro] = ('none', 'none', value)
+
+    def _lo_jobs(self, opt, macro, value):
+        if value is None:
+            raise error.general('option requires a value: %s' % (opt))
+        ok = False
+        if value in ['max', 'none', 'half']:
+            ok = True
+        else:
+            try:
+                i = int(value)
+                ok = True
+            except:
+                pass
+            if not ok:
+                try:
+                    f = float(value)
+                    ok = True
+                except:
+                    pass
+        if not ok:
+            raise error.general('invalid jobs option: %s' % (value))
+        self.defaults[macro] = ('none', 'none', value)
+        self.opts[opt[2:]] = value
+
+    def _lo_bool(self, opt, macro, value):
+        if value is not None:
+            raise error.general('option does not take a value: %s' % (opt))
+        self.opts[opt[2:]] = '1'
+        self.defaults[macro] = ('none', 'none', '1')
+
+    def _lo_triplets(self, opt, macro, value):
+        #
+        # This is a target triplet. Run it past config.sub to make make sure it
+        # is ok.  The target triplet is 'cpu-vendor-os'.
+        #
+        e = execute.capture_execution()
+        config_sub = path.join(self.command_path,
+                               basepath, 'config.sub')
+        exit_code, proc, output = e.shell(config_sub + ' ' + value)
+        if exit_code == 0:
+            value = output
+        self.defaults[macro] = ('triplet', 'none', value)
+        self.opts[opt[2:]] = value
+        _cpu = macro + '_cpu'
+        _arch = macro + '_arch'
+        _vendor = macro + '_vendor'
+        _os = macro + '_os'
+        _arch_value = ''
+        _vendor_value = ''
+        _os_value = ''
+        dash = value.find('-')
+        if dash >= 0:
+            _arch_value = value[:dash]
+            value = value[dash + 1:]
+        dash = value.find('-')
+        if dash >= 0:
+            _vendor_value = value[:dash]
+            value = value[dash + 1:]
+        if len(value):
+            _os_value = value
+        self.defaults[_cpu] = ('none', 'none', _arch_value)
+        self.defaults[_arch] = ('none', 'none', _arch_value)
+        self.defaults[_vendor] = ('none', 'none', _vendor_value)
+        self.defaults[_os] = ('none', 'none', _os_value)
+
+    def _lo_help(self, opt, macro, value):
+        self.help()
 
     def _help(self):
         print '%s: [options] [args]' % (self.command_name)
         print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
         print 'Options and arguments:'
         print '--force                : Force the build to proceed'
-        print '--trace                : Trace the execution (not current used)'
+        print '--quiet                : Quiet output (not used)'
+        print '--trace                : Trace the execution'
         print '--dry-run              : Do everything but actually run the build'
         print '--warn-all             : Generate warnings'
         print '--no-clean             : Do not clean up the build tree'
-        print '--no-smp               : Run with 1 job and not as many as CPUs'
-        print '--rebuild              : Rebuild (not used)'
+        print '--always-clean         : Always clean the build tree, even with an error'
+        print '--jobs                 : Run with specified number of jobs, default: num CPUs.'
         print '--host                 : Set the host triplet'
         print '--build                : Set the build triplet'
         print '--target               : Set the target triplet'
         print '--prefix path          : Tools build prefix, ie where they are installed'
-        print '--prefixbase path      : '
         print '--topdir path          : Top of the build tree, default is $PWD'
         print '--configdir path       : Path to the configuration directory, default: ./config'
         print '--builddir path        : Path to the build directory, default: ./build'
@@ -312,154 +424,42 @@ class command_line:
                 print '%-22s : %s' % (a, self.optargs[a])
         raise error.exit()
 
-    def __init__(self, argv, optargs):
-        self.command_path = path.dirname(argv[0])
-        if len(self.command_path) == 0:
-            self.command_path = '.'
-        self.command_name = path.basename(argv[0])
-        self.argv = argv
-        self.args = argv[1:]
-        self.optargs = optargs
-        self.defaults = {}
-        for to in command_line._long_true_opts:
-            self.defaults[command_line._long_true_opts[to]] = ('none', 'none', '0')
-        self.defaults['_sbdir'] = ('dir', 'required', path.shell(self.command_path))
-        self.opts = { 'params'   : [],
-                      'warn-all' : '0',
-                      'quiet'    : '0',
-                      'force'    : '0',
-                      'trace'    : '0',
-                      'dry-run'  : '0',
-                      'no-clean' : '0',
-                      'no-smp'   : '0',
-                      'rebuild'  : '0' }
-        self._process()
-
-    def __str__(self):
-        def _dict(dd):
-            s = ''
-            ddl = dd.keys()
-            ddl.sort()
-            for d in ddl:
-                s += '  ' + d + ': ' + str(dd[d]) + '\n'
-            return s
-
-        s = 'command: ' + self.command() + \
-            '\nargs: ' + str(self.args) + \
-            '\nopts:\n' + _dict(self.opts)
-
-        return s
-
     def _process(self):
-
-        def _process_lopt(opt, arg, long_opts, args, values = False):
-            for lo in long_opts:
-                if values and opt.startswith(lo):
-                    equals = opt.find('=')
-                    if equals < 0:
-                        if arg == len(args) - 1:
-                            raise error.general('missing option value: ' + lo)
-                        arg += 1
-                        value = args[arg]
-                    else:
-                        value = opt[equals + 1:]
-                    if type(long_opts[lo]) is tuple:
-                        if long_opts[lo][1]:
-                            value = path.shell(value)
-                        macro = long_opts[lo][0]
-                    else:
-                        macro = long_opts[lo]
-                    return lo, macro, value, arg
-                elif opt == lo:
-                    return lo, long_opts[lo], True, arg
-            return None, None, None, arg
-
-        i = 0
-        while i < len(self.args):
-            a = self.args[i]
-            if a.startswith('-'):
-                if a.startswith('--'):
-                    if a.startswith('--warn-all'):
-                        self.opts['warn-all'] = True
-                    elif a == '--help':
-                        self._help()
-                    else:
-                        lo, macro, value, i = _process_lopt(a, i,
-                                                            command_line._long_true_opts,
-                                                            self.args)
-                        if lo:
-                            self.defaults[macro] = ('none', 'none', '1')
-                            self.opts[lo[2:]] = '1'
+        arg = 0
+        while arg < len(self.args):
+            a = self.args[arg]
+            if a == '-?':
+                self._help()
+            elif a.startswith('--'):
+                los = a.split('=')
+                lo = los[0]
+                if lo in self._long_opts:
+                    long_opt = self._long_opts[lo]
+                    if len(los) == 1:
+                        if long_opt[2]:
+                            if arg == len(args) - 1:
+                                raise error.general('option requires a parameter: %s' % (lo))
+                            arg += 1
+                            value = args[arg]
                         else:
-                            lo, macro, value, i = _process_lopt(a, i,
-                                                                command_line._long_opts,
-                                                                self.args, True)
-                            if lo:
-                                self.defaults[macro] = ('none', 'none', value)
-                                self.opts[lo[2:]] = value
-                            else:
-                                #
-                                # The target triplet is 'cpu-vendor-os'.
-                                #
-                                lo, macro, value, i = _process_lopt(a, i,
-                                                                    command_line._target_triplets,
-                                                                    self.args, True)
-                                if lo:
-                                    #
-                                    # This is a target triplet. Run it past config.sub to make
-                                    # make sure it is ok.
-                                    #
-                                    e = execute.capture_execution()
-                                    config_sub = path.join(self.command_path,
-                                                           basepath, 'config.sub')
-                                    exit_code, proc, output = e.shell(config_sub + ' ' + value)
-                                    if exit_code == 0:
-                                        value = output
-                                    self.defaults[macro] = ('triplet', 'none', value)
-                                    self.opts[lo[2:]] = value
-                                    _arch = macro + '_cpu'
-                                    _vendor = macro + '_vendor'
-                                    _os = macro + '_os'
-                                    _arch_value = ''
-                                    _vendor_value = ''
-                                    _os_value = ''
-                                    dash = value.find('-')
-                                    if dash >= 0:
-                                        _arch_value = value[:dash]
-                                        value = value[dash + 1:]
-                                    dash = value.find('-')
-                                    if dash >= 0:
-                                        _vendor_value = value[:dash]
-                                        value = value[dash + 1:]
-                                    if len(value):
-                                        _os_value = value
-                                    self.defaults[_arch] = ('none', 'none', _arch_value)
-                                    self.defaults[_vendor] = ('none', 'none', _vendor_value)
-                                    self.defaults[_os] = ('none', 'none', _os_value)
-                                if not lo:
-                                    sa = a.split('=')
-                                    if sa[0] not in self.optargs:
-                                        raise error.general('invalid argument (try --help): %s' % (a))
-                else:
-                    if a == '-f':
-                        self.opts['force'] = '1'
-                    elif a == '-n':
-                        self.opts['dry-run'] = '1'
-                    elif a == '-q':
-                        self.opts['quiet'] = '1'
-                    elif a == '-?':
-                        self._help()
+                            value = None
                     else:
-                        raise error.general('invalid argument (try --help): %s' % (a))
+                        value = '='.join(los[1:])
+                    long_opt[1](lo, long_opt[0], value)
             else:
                 self.opts['params'].append(a)
-            i += 1
+            arg += 1
 
     def _post_process(self, _defaults):
-        if self.no_smp():
-            _defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
         if _defaults['_host'][2] == _defaults['nil'][2]:
             raise error.general('host not set')
+        if '_ncpus' not in _defaults:
+            raise error.general('host number of CPUs not set')
+        ncpus = self.jobs(_defaults['_ncpus'][2])
+        if ncpus > 1:
+            _defaults['_smp_mflags'] = ('none', 'none', '-j %d' % (ncpus))
+        else:
+            _defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
         return _defaults
 
     def define(self, _defaults, key, value = '1'):
@@ -506,14 +506,43 @@ class command_line:
     def warn_all(self):
         return self.opts['warn-all'] != '0'
 
+    def keep_going(self):
+        return self.opts['keep-going'] != '0'
+
     def no_clean(self):
         return self.opts['no-clean'] != '0'
 
-    def no_smp(self):
-        return self.opts['no-smp'] != '0'
+    def always_clean(self):
+        return self.opts['always-clean'] != '0'
 
-    def rebuild(self):
-        return self.opts['rebuild'] != '0'
+    def jobs(self, cpus):
+        cpus = int(cpus)
+        if self.opts['jobs'] == 'none':
+            cpus = 0
+        elif self.opts['jobs'] == 'max':
+            pass
+        elif self.opts['jobs'] == 'half':
+            cpus = cpus / 2
+        else:
+            ok = False
+            try:
+                i = int(self.opts['jobs'])
+                cpus = i
+                ok = True
+            except:
+                pass
+            if not ok:
+                try:
+                    f = float(self.opts['jobs'])
+                    cpus = f * cpus
+                    ok = True
+                except:
+                    pass
+                if not ok:
+                    raise error.internal('bad jobs option: %s' % (self.opts['jobs']))
+        if cpus <= 0:
+            cpu = 1
+        return cpus
 
     def params(self):
         return self.opts['params']
@@ -566,15 +595,10 @@ class command_line:
         return ['stdout']
 
     def urls(self):
-        if 'url' in self.opts:
+        if self.opts['url'] is not None:
             return self.opts['url'].split(',')
         return None
 
-    def prefixbase(self):
-        if 'prefixbase' in self.opts:
-            return self.opts['prefixbase']
-        return None
-
 def load(args, optargs = None):
     """
     Copy the defaults, get the host specific values and merge them overriding