Canandian Cross Compiling and RTEMS 3rd party package building Fixes.

The change fixes installing for RTEMS 3rd Party packages where the
RSB considered them Canadian Cross Compiling (Cxc). Fixing the
Cxc issue broke real Cxc builds. The change corrects the issue of
macros being changed in the Cxc and the prep data not being udpated.
The configuration is loaded again after the updated macros. The
macros are also copied and restored to ensure a clean stable base.

The change also introduces --rtems-tools and --rtems-bsp to align
the command line with the waf configure process or RTEMS application.

10 files changed, 198 insertions, 88 deletions

diff --git a/rtems/config/rtems-bsp.cfg b/rtems/config/rtems-bsp.cfg
index 3b24ec8..d5868ca 100644
--- a/rtems/config/rtems-bsp.cfg
+++ b/rtems/config/rtems-bsp.cfg
@@ -3,21 +3,25 @@
 #
 
 #
-# The RTEMS BSP support requires the host turple, the RTEMS BSP
-# and the path to the tools. The prefix is set to an installed
-# RTEMS. The built package is installed into the prefix.
+# The RTEMS BSP support requires the host turple, the RTEMS BSP and the path to
+# the tools. The prefix is set to an installed RTEMS. The built package is
+# installed into the prefix.
 #
-# Keeping the package's installed path in the RTEMS install path
-# and separate to the tools lets the tools version vary
-# independently. If --with-tools is not provided use the prefix.
+# Keeping the package's installed path in the RTEMS install path and separate
+# to the tools lets the tools version vary independently. If --rtems-tools
+# (--with-tools) is not provided use the prefix.
 #
 
-%if %{_host} == %{nil}
- %error No RTEMS target specified: --host=host
+%if %{_target} == %{nil}
+ %error No RTEMS target specified: --rtems-bsp=arch/bsp (or --target=target)
+%endif
+
+%if %{_host} != %{_build}
+ %error Canadian cross building for BSP is not supported.
 %endif
 
 %ifn %{defined with_rtems_bsp}
- %error No RTEMS BSP specified: --with-rtems-bsp=bsp
+ %error No RTEMS BSP specified: --rtems-bsp=arch/bsp (or --with-rtems-bsp=bsp)
 %endif
 
 %ifn %{defined with_tools}
@@ -25,6 +29,11 @@
 %endif
 
 #
+# Set the host to the target.
+#
+%define _host %{_target}
+
+#
 # Set the path to the tools.
 #
 %{path prepend %{with_tools}/bin}
@@ -47,6 +56,10 @@
 %define rtems_bsp_ldflags   %{pkgconfig ldflags %{_host}-%{rtems_bsp}}
 %define rtems_bsp_libs      %{pkgconfig libs    %{_host}-%{rtems_bsp}}
 
+%if %{rtems_bsp_cflags} == %{nil}
+ %error No RTEMS target CFLAGS found; Please check the --rtems-bsp option.
+%endif
+
 %if %{rtems_bsp_ccflags} == %{nil}
   %define rtems_bsp_ccflags %{rtems_bsp_cflags}
 %endif
diff --git a/rtems/config/tools/rtems-autoconf-2.69-1.cfg b/rtems/config/tools/rtems-autoconf-2.69-1.cfg
index 1db8746..71ef4ed 100644
--- a/rtems/config/tools/rtems-autoconf-2.69-1.cfg
+++ b/rtems/config/tools/rtems-autoconf-2.69-1.cfg
@@ -2,6 +2,10 @@
 # Autoconf 2.69.
 #
 
+%if %{rtems_arch} == none
+ %define _target %{_host}
+%endif
+
 %include %{_configdir}/checks.cfg
 %include %{_configdir}/base.cfg
 %include %{_configdir}/versions.cfg
diff --git a/rtems/config/tools/rtems-automake-1.12.6-1.cfg b/rtems/config/tools/rtems-automake-1.12.6-1.cfg
index 1585cf4..f844013 100644
--- a/rtems/config/tools/rtems-automake-1.12.6-1.cfg
+++ b/rtems/config/tools/rtems-automake-1.12.6-1.cfg
@@ -2,6 +2,10 @@
 # Automake 1.12.6
 #
 
+%if %{rtems_arch} == none
+ %define _target %{_host}
+%endif
+
 %include %{_configdir}/checks.cfg
 %include %{_configdir}/base.cfg
 %include %{_configdir}/versions.cfg
diff --git a/source-builder/defaults.mc b/source-builder/defaults.mc
index ce81c35..d38d090 100644
--- a/source-builder/defaults.mc
+++ b/source-builder/defaults.mc
@@ -79,12 +79,14 @@ _sourcedir:          dir,     optional, '%{_topdir}/sources'
 _patchdir:           dir,     optional, '%{_topdir}/patches:%{_sbdir}/patches'
 _builddir:           dir,     optional, '%{_topdir}/build/%{buildname}'
 _buildcxcdir:        dir,     optional, '%{_topdir}/build/%{buildname}-cxc'
+_buildxcdir:         dir,     optional, '%{_topdir}/build/%{buildname}-xc'
 _docdir:             dir,     none,     '%{_defaultdocdir}'
 _tmppath:            dir,     none,     '%{_topdir}/build/tmp'
 _tmproot:            dir,     none,     '%{_tmppath}/sb-%{_uid}/%{_bset}'
 _tmpcxcroot:         dir,     none,     '%{_tmppath}/sb-%{_uid}-cxc/%{_bset}'
 buildroot:           dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}'
 buildcxcroot:        dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-cxc'
+buildxcroot:         dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-xx'
 _datadir:            dir,     none,     '%{_prefix}/share'
 _defaultdocdir:      dir,     none,     '%{_prefix}/share/doc'
 _exeext:             none,    none,     ''
@@ -301,7 +303,7 @@ fi'''
 # Host/build flags.
 host_build_flags:    none,    none,     '''
 # Host and build flags, Cross build if host and build are different and
-# Cxc build idf target is deifned and also different.
+# Cxc build if target is deifned and also different.
 # Note, gcc is not ready to be compiled with -std=gnu99 (this needs to be checked).
 if test "%{_build}" != "%{_host}" ; then
   # Cross build
diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py
index 630a1a0..3241133 100644
--- a/source-builder/sb/build.py
+++ b/source-builder/sb/build.py
@@ -22,6 +22,7 @@
 # installed not to be package unless you run a packager around this.
 #
 
+import copy
 import getopt
 import glob
 import os
@@ -116,12 +117,12 @@ class build:
     def __init__(self, name, create_tar_files, opts, macros = None):
         try:
             self.opts = opts
-            if macros is None:
-                self.macros = opts.defaults
-            else:
-                self.macros = macros
+            self.init_name = name
+            self.init_macros = macros
+            self.config = None
             self.create_tar_files = create_tar_files
             log.notice('config: ' + name)
+            self.set_macros(macros)
             self.config = config.file(name, opts, self.macros)
             self.script = script()
             self.macros['buildname'] = self._name_(self.macros['name'])
@@ -136,6 +137,20 @@ class build:
         except:
             raise
 
+    def copy_init_macros(self):
+        return copy.copy(self.init_macros)
+
+    def copy_macros(self):
+        return copy.copy(self.macros)
+
+    def set_macros(self, macros):
+        if macros is None:
+            self.macros = copy.copy(opts.defaults)
+        else:
+            self.macros = copy.copy(macros)
+        if self.config:
+            self.config.set_macros(self.macros)
+
     def rmdir(self, rmpath):
         log.output('removing: %s' % (path.host(rmpath)))
         if not self.opts.dry_run():
@@ -151,9 +166,30 @@ class build:
         _host = self.config.expand('%{_host}')
         _build = self.config.expand('%{_build}')
         _target = self.config.expand('%{_target}')
-        return self.config.defined('%{allow_cxc}') and \
-            len(_host) and len(_build) and (_target) and \
-            _host != _build and _host != _target
+        _allowed = self.config.defined('%{allow_cxc}')
+        if len(_host) and len(_build) and (_target) and \
+           _allowed and _host != _build and _host != _target:
+            return True
+        return False
+
+    def installable(self):
+        _host = self.config.expand('%{_host}')
+        _build = self.config.expand('%{_build}')
+        _canadian_cross = self.canadian_cross()
+        if self.macros.get('_disable_installing') and \
+           self.config.expand('%{_disable_installing}') == 'yes':
+            _disable_installing = True
+        else:
+            _disable_installing = False
+        _no_install = self.opts.no_install()
+        log.trace('_build: installable: host=%s build=%s ' \
+                  'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
+                  (_host, _build, _no_install, _canadian_cross, _disable_installing, \
+                   self.disabled()))
+        return len(_host) and len(_build) and \
+            not self.disabled() and \
+            not _disable_installing and \
+            not _canadian_cross
 
     def source(self, name):
         #
@@ -313,7 +349,8 @@ class build:
                         raise error.general('%s: %s' % (package, msg))
                     if args[0] == '%setup':
                         if len(args) == 1:
-                            raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
+                            raise error.general('invalid %%setup directive: %s' % \
+                                                (' '.join(args)))
                         if args[1] == 'source':
                             self.source_setup(package, args[1:])
                         elif args[1] == 'patch':
@@ -371,6 +408,8 @@ class build:
 
     def build_package(self, package):
         if self.canadian_cross():
+            if not self.config.defined('%{allow_cxc}'):
+                raise error.general('Canadian Cross is not allowed')
             self.script.append('echo "==> Candian-cross build/target:"')
             self.script.append('SB_CXC="yes"')
         else:
@@ -402,6 +441,9 @@ class build:
         packages = self.config.packages()
         return packages['main']
 
+    def reload(self):
+        self.config.load(self.init_name)
+
     def make(self):
         package = self.main_package()
         if package.disabled():
@@ -410,12 +452,13 @@ class build:
             try:
                 name = package.name()
                 if self.canadian_cross():
-                    log.notice('package: (Cxc) %s' % (name))
+                    cxc_label = '(Cxc) '
                 else:
-                    log.notice('package: %s' % (name))
-                    log.trace('---- macro maps %s' % ('-' * 55))
-                    log.trace('%s' % (str(self.config.macros)))
-                    log.trace('-' * 70)
+                    cxc_label = ''
+                log.notice('package: %s%s' % (cxc_label, name))
+                log.trace('---- macro maps %s' % ('-' * 55))
+                log.trace('%s' % (str(self.config.macros)))
+                log.trace('-' * 70)
                 self.script.reset()
                 self.script.append(self.config.expand('%{___build_template}'))
                 self.script.append('echo "=> ' + name + ':"')
@@ -426,10 +469,7 @@ class build:
                     sn = path.join(self.config.expand('%{_builddir}'), 'doit')
                     log.output('write script: ' + sn)
                     self.script.write(sn)
-                    if self.canadian_cross():
-                        log.notice('building: (Cxc) %s' % (name))
-                    else:
-                        log.notice('building: %s' % (name))
+                    log.notice('building: %s%s' % (cxc_label, name))
                     self.run(sn)
             except error.general, gerr:
                 log.notice(str(gerr))
diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py
index 1687d06..127112b 100644
--- a/source-builder/sb/config.py
+++ b/source-builder/sb/config.py
@@ -236,35 +236,14 @@ class file:
                 re.compile('%disable') ]
 
     def __init__(self, name, opts, macros = None):
+        log.trace('config: %s: initialising' % (name))
         self.opts = opts
-        if macros is None:
-            self.macros = opts.defaults
-        else:
-            self.macros = macros
         self.init_name = name
-        log.trace('config: %s' % (name))
-        self.disable_macro_reassign = False
-        self.configpath = []
         self.wss = re.compile(r'\s+')
         self.tags = re.compile(r':+')
         self.sf = re.compile(r'%\([^\)]+\)')
-        for arg in self.opts.args:
-            if arg.startswith('--with-') or arg.startswith('--without-'):
-                if '=' in arg:
-                    label, value = arg.split('=', 1)
-                else:
-                    label = arg
-                    value = None
-                label = label[2:].lower().replace('-', '_')
-                if value:
-                    self.macros.define(label, value)
-                else:
-                    self.macros.define(label)
-        self._includes = []
-        self.load_depth = 0
-        self.pkgconfig_prefix = None
-        self.pkgconfig_crosscompile = False
-        self.pkgconfig_filter_flags = False
+        self.set_macros(macros)
+        self._reset(name)
         self.load(name)
 
     def __str__(self):
@@ -286,6 +265,34 @@ class file:
             s += str(self._packages[_package])
         return s
 
+    def _reset(self, name):
+        self.name = name
+        self.load_depth = 0
+        self.configpath = []
+        self._includes = []
+        self._packages = {}
+        self.in_error = False
+        self.lc = 0
+        self.conditionals = {}
+        self._packages = {}
+        self.package = 'main'
+        self.disable_macro_reassign = False
+        self.pkgconfig_prefix = None
+        self.pkgconfig_crosscompile = False
+        self.pkgconfig_filter_flags = False
+        for arg in self.opts.args:
+            if arg.startswith('--with-') or arg.startswith('--without-'):
+                if '=' in arg:
+                    label, value = arg.split('=', 1)
+                else:
+                    label = arg
+                    value = None
+                label = label[2:].lower().replace('-', '_')
+                if value:
+                    self.macros.define(label, value)
+                else:
+                    self.macros.define(label)
+
     def _relative_path(self, p):
         sbdir = None
         if '_sbdir' in self.macros:
@@ -1053,6 +1060,12 @@ class file:
     def _info_append(self, info, data):
         self._packages[self.package].info_append(info, data)
 
+    def set_macros(self, macros):
+        if macros is None:
+            self.macros = opts.defaults
+        else:
+            self.macros = macros
+
     def load(self, name):
 
         def common_end(left, right):
@@ -1066,12 +1079,7 @@ class file:
             return end
 
         if self.load_depth == 0:
-            self.in_error = False
-            self.lc = 0
-            self.name = name
-            self.conditionals = {}
-            self._packages = {}
-            self.package = 'main'
+            self._reset(name)
             self._packages[self.package] = package(self.package,
                                                    self.define('%{_arch}'),
                                                    self)
diff --git a/source-builder/sb/options.py b/source-builder/sb/options.py
index e4964f0..c611dfe 100644
--- a/source-builder/sb/options.py
+++ b/source-builder/sb/options.py
@@ -221,6 +221,9 @@ class command_line:
         print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code'
         print '--with-<label>         : Add the --with-<label> to the build'
         print '--without-<label>      : Add the --without-<label> to the build'
+        print '--rtems-tools path     : Path to an install RTEMS tool set'
+        print '--rtems-bsp arc/bsp    : Standard RTEMS architecure and BSP specifier'
+        print '--rtems-version ver    : The RTEMS major/minor version string'
         if self.optargs:
             for a in self.optargs:
                 print '%-22s : %s' % (a, self.optargs[a])
@@ -297,10 +300,6 @@ class command_line:
             rsb_macros = path.join(os.environ['HOME'], '.rsb_macros')
             if path.exists(rsb_macros):
                 self.defaults.load(rsb_macros)
-        # If a Cxc build disable installing.
-        if self.canadian_cross():
-            self.opts['no-install'] = '1'
-            self.defaults['_no_install'] = '1'
 
     def sb_git(self):
         repo = git.repo(self.defaults.expand('%{_sbdir}'), self)
@@ -450,7 +449,7 @@ class command_line:
 
     def get_arg(self, arg):
         if self.optargs is None or arg not in self.optargs:
-            raise error.internal('bad arg: %s' % (arg))
+            return None
         return self.parse_args(arg)
 
     def with_arg(self, label):
@@ -516,6 +515,9 @@ class command_line:
     def download_disabled(self):
         return self.opts['no-download'] != '0'
 
+    def disable_install(self):
+        self.opts['no-install'] = '1'
+
     def info(self):
         s = ' Command Line: %s%s' % (' '.join(self.argv), os.linesep)
         s += ' Python: %s' % (sys.version.replace('\n', ''))
@@ -524,6 +526,28 @@ class command_line:
     def log_info(self):
         log.output(self.info())
 
+    def rtems_options(self):
+        # Check for RTEMS specific helper options.
+        rtems_tools = self.parse_args('--rtems-tools')
+        if rtems_tools is not None:
+            if self.get_arg('--with-tools') is not None:
+                raise error.general('--rtems-tools and --with-tools cannot be used together')
+            self.args.append('--with-tools=%s' % (rtems_tools[1]))
+        rtems_arch_bsp = self.parse_args('--rtems-bsp')
+        if rtems_arch_bsp is not None:
+            if self.get_arg('--target') is not None:
+                raise error.general('--rtems-bsp and --target cannot be used together')
+            ab = rtems_arch_bsp[1].split('/')
+            if len(ab) != 2:
+                raise error.general('invalid --rtems-bsp option')
+            rtems_version = self.parse_args('--rtems-version')
+            if rtems_version is None:
+                rtems_version = '%d.%d' % (version.major, version.minor)
+            else:
+                rtems_version = rtems_version[1]
+            self.args.append('--target=%s-rtems%s' % (ab[0], rtems_version))
+            self.args.append('--with-rtems-bsp=%s' % (ab[1]))
+
 def load(args, optargs = None, defaults = '%{_sbdir}/defaults.mc'):
     """
     Copy the defaults, get the host specific values and merge them overriding
@@ -590,6 +614,7 @@ def load(args, optargs = None, defaults = '%{_sbdir}/defaults.mc'):
         o.defaults[k] = overrides[k]
 
     o.sb_git()
+    o.rtems_options()
     o.process()
     o.post_process()
 
diff --git a/source-builder/sb/path.py b/source-builder/sb/path.py
index 824b8f5..df76634 100644
--- a/source-builder/sb/path.py
+++ b/source-builder/sb/path.py
@@ -201,7 +201,11 @@ def copy_tree(src, dst):
     if not os.path.isdir(hdst):
         if trace:
             print ' mkdir: %s' % (hdst)
-        os.makedirs(hdst)
+        try:
+            os.makedirs(hdst)
+        except OSError, why:
+            raise error.general('copying tree: cannot create target directory %s: %s' % \
+                                (hdst, str(why)))
 
     for name in names:
         srcname = host(os.path.join(hsrc, name))
diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py
index ae1b670..3b05cff 100644
--- a/source-builder/sb/setbuilder.py
+++ b/source-builder/sb/setbuilder.py
@@ -99,7 +99,8 @@ class buildset:
                 format = _build.opts.get_arg('--report-format')
                 if format is not None:
                     if len(format) != 2:
-                        raise error.general('invalid report format option: %s' % ('='.join(format)))
+                        raise error.general('invalid report format option: %s' % \
+                                            ('='.join(format)))
                     format = format[1]
             if format is None:
                 format = 'text'
@@ -156,27 +157,28 @@ class buildset:
         self.copy(src, dst)
 
     def canadian_cross(self, _build):
-        # @fixme Switch to using a private macros map.
-        macros_to_save = ['%{_prefix}',
-                          '%{_tmproot}',
-                          '%{buildroot}',
-                          '%{_builddir}',
-                          '%{_host}']
-        macros_to_copy = [('%{_host}',     '%{_build}'),
-                          ('%{_tmproot}',  '%{_tmpcxcroot}'),
-                          ('%{buildroot}', '%{buildcxcroot}'),
-                          ('%{_builddir}', '%{_buildcxcdir}')]
-        orig_macros = {}
-        for m in macros_to_save:
-            orig_macros[m] = _build.config.macro(m)
+        log.trace('_bset: Cxc for build machine: _build => _host')
+        macros_to_copy = [('%{_host}',        '%{_build}'),
+                          ('%{_host_alias}',  '%{_build_alias}'),
+                          ('%{_host_arch}',   '%{_build_arch}'),
+                          ('%{_host_cpu}',    '%{_build_cpu}'),
+                          ('%{_host_os}',     '%{_build_os}'),
+                          ('%{_host_vendor}', '%{_build_vendor}'),
+                          ('%{_tmproot}',     '%{_tmpcxcroot}'),
+                          ('%{buildroot}',    '%{buildcxcroot}'),
+                          ('%{_builddir}',    '%{_buildcxcdir}')]
+        cxc_macros = _build.copy_init_macros()
         for m in macros_to_copy:
-            _build.config.set_define(m[0], _build.config.macro(m[1]))
+            log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
+            cxc_macros[m[0]] = cxc_macros[m[1]]
+        _build.set_macros(cxc_macros)
+        _build.reload()
         _build.make()
-        for m in macros_to_save:
-            _build.config.set_define(m, orig_macros[m])
         if not _build.macros.get('%{_disable_collecting}'):
-            self.root_copy(_build.config.expand('%{buildcxcroot}'),
-                           _build.config.expand('%{_tmpcxcroot}'))
+            self.root_copy(_build.config.expand('%{buildroot}'),
+                           _build.config.expand('%{_tmproot}'))
+        _build.set_macros(_build.copy_init_macros())
+        _build.reload()
 
     def build_package(self, _config, _build):
         if not _build.disabled():
@@ -189,7 +191,7 @@ class buildset:
 
     def bset_tar(self, _build):
         tardir = _build.config.expand('%{_tardir}')
-        if self.opts.get_arg('--bset-tar-file') \
+        if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
            and not _build.macros.get('%{_disable_packaging}'):
             path.mkdir(tardir)
             tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (self.bset_pkg)))
@@ -380,16 +382,21 @@ class buildset:
                             raise
                     else:
                         raise
+            #
+            # Installing ...
+            #
+            log.trace('_bset: installing: deps:%r no-install:%r' % \
+                      (deps is None, self.opts.no_install()))
             if deps is None \
                and not self.opts.no_install() \
                and not have_errors:
                 for b in builds:
-                    if not b.canadian_cross() \
-                       and not b.disabled() \
-                       and not b.macros.get('%{_disable_installing}'):
+                    log.trace('_bset: installing: %r' % b.installable())
+                    if b.installable():
                         self.install(b.name(),
                                      b.config.expand('%{buildroot}'),
                                      b.config.expand('%{_prefix}'))
+
             if deps is None and \
                     (not self.opts.no_clean() or self.opts.always_clean()):
                 for b in builds:
@@ -472,6 +479,9 @@ def run():
             deps = None
         if not list_bset_cfg_files(opts, configs):
             prefix = opts.defaults.expand('%{_prefix}')
+            if opts.canadian_cross():
+                opts.disable_install()
+
             if not opts.dry_run() and \
                not opts.canadian_cross() and \
                not opts.no_install() and \
diff --git a/source-builder/sb/version.py b/source-builder/sb/version.py
index bdc0b3c..802fafa 100644
--- a/source-builder/sb/version.py
+++ b/source-builder/sb/version.py
@@ -23,8 +23,8 @@
 # level. This allows macro expansion to work.
 #
 
-major = 0
-minor = 5
+major = 4
+minor = 11
 revision = 0
 
 def str():