path: root/rtemstoolkit/options.py

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

#
# Determine the defaults and load the specific file.
#

from __future__ import print_function

import copy
import glob
import pprint
import re
import os
import string
import sys

from rtemstoolkit import error
from rtemstoolkit import execute
from rtemstoolkit import git
from rtemstoolkit import host
from rtemstoolkit import log
from rtemstoolkit import macros
from rtemstoolkit import path
from rtemstoolkit import version

basepath = 'tb'

class command_line(object):
    """Process the command line in a common way for all Tool Builder commands."""

    def __init__(self, base_path = None, argv = None, optargs = None,
                 defaults = None, long_opts = None, long_opts_help = None,
                 command_path = '', log_default = None):

        if argv is None:
            return

        global basepath

        if long_opts == None:
            long_opts = {}

        basepath = base_path

        if log_default is not None and type(log_default) is not list:
            raise error.general('log default is not a list')
        self.log_default = log_default

        if defaults is None:
            defaults = macros.macros()

        self.long_opts = {
            # key                 macro                handler            param  defs       init
            '--jobs'           : ('_jobs',             self._lo_jobs,     True,  'default', True),
            '--log'            : ('_logfile',          self._lo_string,   True,  None,      False),
            '--macros'         : ('_macros',           self._lo_string,   True,  None,      False),
            '--force'          : ('_force',            self._lo_bool,     False, '0',       True),
            '--quiet'          : ('_quiet',            self._lo_bool,     False, '0',       True),
            '--trace'          : ('_trace',            self._lo_bool,     False, '0',       True),
            '--dry-run'        : ('_dry_run',          self._lo_bool,     False, '0',       True),
            '--warn-all'       : ('_warn_all',         self._lo_bool,     False, '0',       True),
            '--no-clean'       : ('_no_clean',         self._lo_bool,     False, '0',       True),
            '--keep-going'     : ('_keep_going',       self._lo_bool,     False, '0',       True),
            '--always-clean'   : ('_always_clean',     self._lo_bool,     False, '0',       True),
            '--no-install'     : ('_no_install',       self._lo_bool,     False, '0',       True),
            '--help'           : (None,                self._lo_help,     False, None,      False)
        }
        self.long_opts_help = {
            '--force':                      'Force the build to proceed',
            '--quiet':                      'Quiet output (not used)',
            '--trace':                      'Trace the execution',
            '--dry-run':                    'Do everything but actually run the build',
            '--warn-all':                   'Generate warnings',
            '--no-clean':                   'Do not clean up the build tree',
            '--always-clean':               'Always clean the build tree, even with an error',
            '--keep-going':                 'Do not stop on an error.',
            '--jobs=[0..n,none,half,full]': 'Run with specified number of jobs, default: num CPUs.',
            '--macros file[,file]':         'Macro format files to load after the defaults',
            '--log file':                   'Log file where all build output is written to',
        }
        self.opts = { 'params' : [] }
        self.command_path = command_path
        self.command_name = path.basename(argv[0])
        self.argv = argv
        self.args = argv[1:]
        self.optargs = optargs
        self.defaults = defaults
        for lo in self.long_opts:
            self.opts[lo[2:]] = self.long_opts[lo][3]
            if self.long_opts[lo][4]:
                self.defaults[self.long_opts[lo][0]] = ('none', 'none', self.long_opts[lo][3])
        for lo in long_opts:
            if lo in self.long_opts:
                raise error.general('suplicate option: %s' % (lo))
            self.opts[lo[2:]] = long_opts[lo][3]
            if long_opts[lo][4]:
                self.defaults[long_opts[lo][0]] = ('none', 'none', long_opts[lo][3])
            if long_opts[lo][1] == 'int':
                handler = self._lo_int
            elif long_opts[lo][1] == 'string':
                handler = self._lo_string
            elif long_opts[lo][1] == 'path':
                handler = self._lo_path
            elif long_opts[lo][1] == 'jobs':
                handler = self._lo_jobs
            elif long_opts[lo][1] == 'bool':
                handler = self._lo_bool
            elif long_opts[lo][1] == 'triplet':
                handler = self._lo_triplets
            else:
                raise error.general('invalid option handler: %s: %s' % (lo, long_opts[lo][1]))
            self.long_opts[lo] = (long_opts[lo][0], handler, long_opts[lo][2],
                                   long_opts[lo][3], long_opts[lo][4])
            if long_opts_help is not None:
                if lo not in long_opts_help:
                    raise error.general('no help for option: %s' % (lo))
                self.long_opts_help[lo] = long_opts_help[lo]

    def __copy__(self):
        new = type(self)()
        #new.__dict__.update(self.__dict__)
        new.opts = copy.copy(self.opts)
        new.command_path = copy.copy(self.command_path)
        new.command_name = copy.copy(self.command_name)
        new.argv = self.argv
        new.args = self.args
        new.optargs = copy.copy(self.optargs)
        new.defaults = copy.copy(self.defaults)
        new.long_opts = copy.copy(self.long_opts)
        return new

    def __str__(self):
        def _dict(dd):
            s = ''
            ddl = list(dd.keys())
            ddl.sort()
            for d in ddl:
                s += '  ' + d + ': ' + str(dd[d]) + '\n'
            return s

        s = 'command: ' + self.command() + \
            '\nargs: ' + str(self.args) + \
            '\nopts:\n' + _dict(self.opts)

        return s

    def _lo_int(self, opt, macro, value):
        if value is None:
            raise error.general('option requires a value: %s' % (opt))
        try:
            num = int(value)
        except:
            raise error.general('option conversion to int failed: %s' % (opt))
        self.opts[opt[2:]] = value
        self.defaults[macro] = value

    def _lo_string(self, opt, macro, value):
        if value is None:
            raise error.general('option requires a value: %s' % (opt))
        self.opts[opt[2:]] = value
        self.defaults[macro] = value

    def _lo_path(self, opt, macro, value):
        if value is None:
            raise error.general('option requires a path: %s' % (opt))
        value = path.abspath(value)
        self.opts[opt[2:]] = value
        self.defaults[macro] = value

    def _lo_jobs(self, opt, macro, value):
        if value is None:
            raise error.general('option requires a value: %s' % (opt))
        ok = False
        if value in ['max', 'none', 'half']:
            ok = True
        else:
            try:
                i = int(value)
                ok = True
            except:
                pass
            if not ok:
                try:
                    f = float(value)
                    ok = True
                except:
                    pass
        if not ok:
            raise error.general('invalid jobs option: %s' % (value))
        self.defaults[macro] = value
        self.opts[opt[2:]] = value

    def _lo_bool(self, opt, macro, value):
        if value is not None:
            raise error.general('option does not take a value: %s' % (opt))
        self.opts[opt[2:]] = '1'
        self.defaults[macro] = '1'

    def _lo_triplets(self, opt, macro, value):
        #
        # This is a target triplet. Run it past config.sub to make make sure it
        # is ok. The target triplet is 'cpu-vendor-os'.
        #
        e = execute.capture_execution()
        config_sub = path.join(self.command_path,
                               basepath, 'config.sub')
        exit_code, proc, output = e.shell(config_sub + ' ' + value)
        if exit_code == 0:
            value = output
        self.defaults[macro] = ('triplet', 'none', value)
        self.opts[opt[2:]] = value
        _cpu = macro + '_cpu'
        _arch = macro + '_arch'
        _vendor = macro + '_vendor'
        _os = macro + '_os'
        _arch_value = ''
        _vendor_value = ''
        _os_value = ''
        dash = value.find('-')
        if dash >= 0:
            _arch_value = value[:dash]
            value = value[dash + 1:]
        dash = value.find('-')
        if dash >= 0:
            _vendor_value = value[:dash]
            value = value[dash + 1:]
        if len(value):
            _os_value = value
        self.defaults[_cpu]    = _arch_value
        self.defaults[_arch]   = _arch_value
        self.defaults[_vendor] = _vendor_value
        self.defaults[_os]     = _os_value

    def _lo_help(self, opt, macro, value):
        self.help()

    def _help_indent(self):
        indent = 0
        if self.optargs:
            for o in self.optargs:
                if len(o) > indent:
                    indent = len(o)
        for o in self.long_opts_help:
            if len(o) > indent:
                indent = len(o)
        return indent

    def help(self):
        print('%s: [options] [args]' % (self.command_name))
        print('RTEMS Tools Project, %s' % (version.string()))
        print('Options and arguments:')
        opts = list(self.long_opts_help.keys())
        if self.optargs:
            opts += self.optargs.keys()
        indent = self._help_indent()
        for o in sorted(opts):
            if o in self.long_opts_help:
                h = self.long_opts_help[o]
            elif self.optargs:
                h = self.optargs[o]
            else:
                raise error.general('invalid help data: %s' %(o))
            print('%-*s : %s' % (indent, o, h))
        raise error.exit()

    def process(self):
        arg = 0
        while arg < len(self.args):
            a = self.args[arg]
            if a == '-?':
                self.help()
            elif a.startswith('--'):
                los = a.split('=')
                lo = los[0]
                if lo in self.long_opts:
                    long_opt = self.long_opts[lo]
                    if len(los) == 1:
                        if long_opt[2]:
                            if arg == len(self.args) - 1:
                                raise error.general('option requires a parameter: %s' % (lo))
                            arg += 1
                            value = self.args[arg]
                        else:
                            value = None
                    else:
                        value = '='.join(los[1:])
                    long_opt[1](lo, long_opt[0], value)
            else:
                self.opts['params'].append(a)
            arg += 1

    def post_process(self):
        # Handle the log first.
        log.default = log.log(self.logfiles())
        if self.trace():
            log.tracing = True
        if self.quiet():
            log.quiet = True
        # Handle the jobs for make
        if '_ncpus' not in self.defaults:
            raise error.general('host number of CPUs not set')
        ncpus = self.jobs(self.defaults['_ncpus'])
        if ncpus > 1:
            self.defaults['_smp_mflags'] = '-j %d' % (ncpus)
        else:
            self.defaults['_smp_mflags'] = self.defaults['nil']
        # Load user macro files
        um = self.user_macros()
        if um:
            checked = path.exists(um)
            if False in checked:
                raise error.general('macro file not found: %s' % (um[checked.index(False)]))
            for m in um:
                self.defaults.load(m)
        # Check if the user has a private set of macros to load
        if 'RSB_MACROS' in os.environ:
            if path.exists(os.environ['RSB_MACROS']):
                self.defaults.load(os.environ['RSB_MACROS'])
        if 'HOME' in os.environ:
            rsb_macros = path.join(os.environ['HOME'], '.rsb_macros')
            if path.exists(rsb_macros):
                self.defaults.load(rsb_macros)

    def local_git(self):
        repo = git.repo(self.defaults.expand('%{_rtdir}'), self)
        if repo.valid():
            repo_valid = '1'
            repo_head = repo.head()
            repo_clean = not repo.dirty()
            repo_id = repo_head
            if not repo_clean:
                repo_id += '-modified'
            repo_mail = repo.email()
        else:
            repo_valid = '0'
            repo_head = '%{nil}'
            repo_clean = '%{nil}'
            repo_id = 'no-repo'
            repo_mail = None
        self.defaults['_local_git_valid'] = repo_valid
        self.defaults['_local_git_head']  = repo_head
        self.defaults['_local_git_clean'] = str(repo_clean)
        self.defaults['_local_git_id']    = repo_id
        if repo_mail is not None:
            self.defaults['_localgit_mail'] = repo_mail

    def command(self):
        return path.join(self.command_path, self.command_name)

    def force(self):
        return self.opts['force'] != '0'

    def dry_run(self):
        return self.opts['dry-run'] != '0'

    def set_dry_run(self):
        self.opts['dry-run'] = '1'

    def quiet(self):
        return self.opts['quiet'] != '0'

    def trace(self):
        return self.opts['trace'] != '0'

    def warn_all(self):
        return self.opts['warn-all'] != '0'

    def keep_going(self):
        return self.opts['keep-going'] != '0'

    def no_clean(self):
        return self.opts['no-clean'] != '0'

    def always_clean(self):
        return self.opts['always-clean'] != '0'

    def no_install(self):
        return self.opts['no-install'] != '0'

    def user_macros(self):
        #
        # Return something even if it does not exist.
        #
        if self.opts['macros'] is None:
            return None
        um = []
        configs = self.defaults.expand('%{_configdir}').split(':')
        for m in self.opts['macros'].split(','):
            if path.exists(m):
                um += [m]
            else:
                # Get the expanded config macros then check them.
                cm = path.expand(m, configs)
                ccm = path.exists(cm)
                if True in ccm:
                    # Pick the first found
                    um += [cm[ccm.index(True)]]
                else:
                    um += [m]
        return um if len(um) else None

    def jobs(self, cpus):
        try:
            cpus = int(cpus)
        except:
            raise error.general('invalid host cpu value')
        opt_jobs = self.opts['jobs']
        if opt_jobs == 'default':
            _jobs = self.defaults.get_value('jobs')
            if _jobs is not None:
                if _jobs == 'none':
                    cpus = 0
                elif _jobs == 'max':
                    pass
                elif _jobs == 'half':
                    cpus = cpus / 2
                else:
                    try:
                        cpus = int(_jobs)
                    except:
                        raise error.general('invalid %%{jobs} value: %s' % (_jobs))
            else:
                opt_jobs = 'max'
        if opt_jobs != 'default':
            if opt_jobs == 'none':
                cpus = 0
            elif opt_jobs == 'max':
                pass
            elif opt_jobs == 'half':
                cpus = cpus / 2
            else:
                ok = False
                try:
                    i = int(opt_jobs)
                    cpus = i
                    ok = True
                except:
                    pass
                if not ok:
                    try:
                        f = float(opt_jobs)
                        cpus = f * cpus
                        ok = True
                    except:
                        pass
                    if not ok:
                        raise error.internal('bad jobs option: %s' % (opt_jobs))
        if cpus <= 0:
            cpu = 1
        return cpus

    def params(self):
        return self.opts['params']

    def get_args(self):
        for arg in self.args:
            yield arg

    def find_arg(self, arg):
        if self.optargs is None or arg not in self.optargs:
            raise error.internal('bad arg: %s' % (arg))
        for a in self.args:
            sa = a.split('=')
            if sa[0].startswith(arg):
                return sa
        return None

    def logfiles(self):
        if 'log' in self.opts and self.opts['log'] is not None:
            log = self.opts['log'].split(',')
        elif self.log_default is None:
            log = ['stdout']
        else:
            log = self.log_default
        return log

    def urls(self):
        if self.opts['url'] is not None:
            return self.opts['url'].split(',')
        return None

    def log_info(self):
        # Filter potentially sensitive mail options out.
        filtered_args = [
            arg for arg in self.argv
            if all(
                smtp_opt not in arg
                for smtp_opt in [
                    '--smtp-host',
                    '--mail-to',
                    '--mail-from',
                    '--smtp-user',
                    '--smtp-password',
                    '--smtp-port'
                ]
            )
        ]
        log.output(log.info(filtered_args))

def load(opts):
    """
    Copy the defaults, get the host specific values and merge them overriding
    any matching defaults, then create an options object to handle the command
    line merging in any command line overrides. Finally post process the
    command line.
    """

    if not isinstance(opts, command_line):
        raise error.general('invalid options type passed to options loader')

    overrides = host.overrides()
    for k in overrides:
        opts.defaults[k] = overrides[k]

    opts.local_git()
    opts.process()
    opts.post_process()

def run(args):
    try:
        long_opts = {
            # key              macro         handler   param  defs   init
            '--test-path'  : ('_test_path',  'path',   True,  None,  False),
            '--test-jobs'  : ('_test_jobs',  'jobs',   True,  'max', True),
            '--test-bool'  : ('_test_bool',  'bool',   False, '0',   True)
        }
        opts = command_line(base_path = '.',
                            argv = args,
                            optargs = None,
                            defaults = macros.macros(),
                            long_opts = long_opts,
                            command_path = '.')
        load(opts)
        log.notice('RTEMS Tools Project - Defaults, v%s' % (version.string()))
        opts.log_info()
        log.notice('Options:')
        log.notice(str(opts))
        log.notice('Defaults:')
        log.notice(str(opts.defaults))
    except error.general as gerr:
        print(gerr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr)
        sys.exit(1)
    except error.exit:
        pass
    except KeyboardInterrupt:
        _notice(opts, 'abort: user terminated')
        sys.exit(1)
    sys.exit(0)

if __name__ == '__main__':
    run(sys.argv)