sb-bootstrap: Update to support python34.11

Closes #4587

2 files changed, 237 insertions, 318 deletions

diff --git a/source-builder/sb-bootstrap b/source-builder/sb-bootstrap
index a1ee9a7..6519710 100755
--- a/source-builder/sb-bootstrap
+++ b/source-builder/sb-bootstrap
@@ -1,31 +1,246 @@
 #! /usr/bin/env python
+
 #
-# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2013 Chris Johns (chrisj@rtems.org)
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (C) 2013-2019 Chris Johns (chrisj@rtems.org)
 # All rights reserved.
 #
-# This file is part of the RTEMS Tools package in 'rtems-tools'.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
 #
-# Permission to use, copy, modify, and/or distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
+
+#
+# RTEMS Tools Project (http://www.rtems.org/)
 #
-# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 from __future__ import print_function
 
-import sys, os
-base = os.path.dirname(sys.argv[0])
-sys.path.insert(0, base + '/sb')
-try:
-    import bootstrap
-    bootstrap.run(sys.argv)
-except ImportError:
-    print("Incorrect Source Builder installation", file = sys.stderr)
-    sys.exit(1)
+import argparse
+import datetime
+import multiprocessing
+import os
+import re
+import sys
+import threading
+import time
+
+version = "1.0"
+
+class error(Exception):
+    """Base class for Builder exceptions."""
+    def set_output(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class general_error(error):
+    """Raise for a general error."""
+    def __init__(self, what):
+        self.set_output('error: ' + str(what))
+
+def _collect(path_, file):
+    confs = []
+    for root, dirs, files in os.walk(path_, topdown = True):
+        for f in files:
+            if f == file:
+                confs += [os.path.join(root, f)]
+    return confs
+
+def _grep(file, pattern):
+    rege = re.compile(pattern)
+    try:
+        f = open(file, 'r')
+        matches = [rege.match(l) != None for l in f.readlines()]
+        f.close()
+    except IOError as err:
+        raise general_error('reading: %s' % (file))
+    return True in matches
+
+class command:
+
+    def __init__(self, cmd, cwd):
+        self.exit_code = 0
+        self.thread = None
+        self.output = None
+        self.cmd = cmd
+        self.cwd = cwd
+        self.result = None
+
+    def runner(self):
+
+        import subprocess
+
+        #
+        # Support Python 2.6
+        #
+        if "check_output" not in dir(subprocess):
+            def f(*popenargs, **kwargs):
+                if 'stdout' in kwargs:
+                    raise ValueError('stdout argument not allowed, it will be overridden.')
+                process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+                output, unused_err = process.communicate()
+                retcode = process.poll()
+                if retcode:
+                    cmd = kwargs.get("args")
+                    if cmd is None:
+                        cmd = popenargs[0]
+                    raise subprocess.CalledProcessError(retcode, cmd)
+                return output
+            subprocess.check_output = f
+
+        self.start_time = datetime.datetime.now()
+        self.exit_code = 0
+        try:
+            try:
+                if os.name == 'nt':
+                    cmd = ['sh', '-c'] + self.cmd
+                else:
+                    cmd = self.cmd
+                self.output = subprocess.check_output(cmd, cwd = self.cwd)
+            except subprocess.CalledProcessError as cpe:
+                self.exit_code = cpe.returncode
+                self.output = cpe.output
+            except OSError as ose:
+                raise general_error('bootstrap failed: %s in %s: %s' % \
+                                        (' '.join(cmd), self.cwd, (str(ose))))
+            except KeyboardInterrupt:
+                pass
+            except:
+                raise
+        except:
+            self.result = sys.exc_info()
+        self.end_time = datetime.datetime.now()
+
+    def run(self):
+        self.thread = threading.Thread(target = self.runner)
+        self.thread.start()
+
+    def is_alive(self):
+        return self.thread and self.thread.is_alive()
+
+    def reraise(self):
+        if self.result is not None:
+            raise self.result[0](self.result[1])
+
+class autoreconf:
+
+    def __init__(self, topdir, configure):
+        self.topdir = topdir
+        self.configure = configure
+        self.cwd = os.path.dirname(self.configure)
+        self.command = command(['autoreconf', '-i', '--no-recursive'], self.cwd)
+        self.command.run()
+
+    def is_alive(self):
+        return self.command.is_alive()
+
+    def post_process(self):
+        if self.command is not None:
+            self.command.reraise()
+            if self.command.exit_code != 0:
+                raise general_error('error: autoreconf: %s' % (' '.join(self.command.cmd)))
+            makefile = os.path.join(self.cwd, 'Makefile.am')
+            if os.path.exists(makefile):
+                if _grep(makefile, 'stamp-h\.in'):
+                    stamp_h = os.path.join(self.cwd, 'stamp-h.in')
+                    try:
+                        t = open(os.path.host(stamp_h), 'w')
+                        t.write('timestamp')
+                        t.close()
+                    except IOError as err:
+                        raise general_error('writing: %s' % (stamp_h))
+
+def generate(topdir, jobs):
+    if type(jobs) is str:
+        jobs = int(jobs)
+    start_time = datetime.datetime.now()
+    confs = _collect(topdir, 'configure.ac')
+    next = 0
+    autoreconfs = []
+    while next < len(confs) or len(autoreconfs) > 0:
+        if next < len(confs) and len(autoreconfs) < jobs:
+            print('%3d/%3d: autoreconf: %s' % \
+                  (next + 1, len(confs), confs[next][len(topdir) + 1:]))
+            autoreconfs += [autoreconf(topdir, confs[next])]
+            next += 1
+        else:
+            for ac in autoreconfs:
+                if not ac.is_alive():
+                    ac.post_process()
+                    autoreconfs.remove(ac)
+                    del ac
+            if len(autoreconfs) >= jobs:
+                time.sleep(1)
+    end_time = datetime.datetime.now()
+    print('Bootstrap time: %s' % (str(end_time - start_time)))
+
+def run(args):
+    try:
+        #
+        # On Windows MSYS2 prepends a path to itself to the environment
+        # path. This means the RTEMS specific automake is not found and which
+        # breaks the bootstrap. We need to remove the prepended path. Also
+        # remove any ACLOCAL paths from the environment.
+        #
+        if os.name == 'nt':
+            cspath = os.environ['PATH'].split(os.pathsep)
+            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
+                os.environ['PATH'] = os.pathsep.join(cspath[1:])
+            if 'ACLOCAL_PATH' in os.environ:
+                #
+                # The clear fails on a current MSYS2 python (Feb 2016). Delete
+                # the entry if the clear fails.
+                #
+                try:
+                    os.environ['ACLOCAL_PATH'].clear()
+                except:
+                    del os.environ['ACLOCAL_PATH']
+
+        argsp = argparse.ArgumentParser(prog = 'rtems-bootstrap',
+                                        description = "Bootstrap in parallel")
+        argsp.add_argument('-j', '--jobs',
+                           help = 'number of jobs to run (default: %(default)s).',
+                           type = int, default = multiprocessing.cpu_count())
+        argsp.add_argument('-r', '--rtems',
+                           type = str, default = os.getcwd(),
+                           help = 'path to the rtems kernel source (default: %(default)s).')
+        argopts = argsp.parse_args(args[1:])
+
+        print('RTEMS Bootstrap, %s' % (version))
+
+        if not os.path.exists(argopts.rtems):
+            raise general_error('path does not exist: %s' % (argopts.rtems))
+        if not os.path.isdir(argopts.rtems):
+            raise general_error('path not a directory: %s' % (argopts.rtems))
+
+        generate(argopts.rtems, argopts.jobs)
+    except general_error as gerr:
+        print(gerr)
+        print('Bootstrap FAILED', file = sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        log.notice('abort: user terminated')
+        sys.exit(1)
+    sys.exit(0)
+
+if __name__ == "__main__":
+    run(sys.argv)
diff --git a/source-builder/sb/bootstrap.py b/source-builder/sb/bootstrap.py
deleted file mode 100644
index c9acf5c..0000000
--- a/source-builder/sb/bootstrap.py
+++ /dev/null
@@ -1,296 +0,0 @@
-#
-# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2013-2016 Chris Johns (chrisj@rtems.org)
-# All rights reserved.
-#
-# This file is part of the RTEMS Tools package in 'rtems-tools'.
-#
-# Permission to use, copy, modify, and/or distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-#
-
-from __future__ import print_function
-
-import datetime
-import operator
-import os
-import re
-import sys
-import threading
-import time
-
-import error
-import log
-import options
-import path
-import version
-
-def _collect(path_, file):
-    confs = []
-    for root, dirs, files in os.walk(path.host(path_), topdown = True):
-        for f in files:
-            if f == file:
-                confs += [path.shell(path.join(root, f))]
-    return confs
-
-def _grep(file, pattern):
-    rege = re.compile(pattern)
-    try:
-        f = open(path.host(file), 'r')
-        matches = [rege.match(l) != None for l in f.readlines()]
-        f.close()
-    except IOError as err:
-        raise error.general('reading: %s' % (file))
-    return True in matches
-
-class command:
-
-    def __init__(self, cmd, cwd):
-        self.exit_code = 0
-        self.thread = None
-        self.output = None
-        self.cmd = cmd
-        self.cwd = cwd
-        self.result = None
-
-    def runner(self):
-
-        import subprocess
-
-        #
-        # Support Python 2.6
-        #
-        if "check_output" not in dir(subprocess):
-            def f(*popenargs, **kwargs):
-                if 'stdout' in kwargs:
-                    raise ValueError('stdout argument not allowed, it will be overridden.')
-                process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
-                output, unused_err = process.communicate()
-                retcode = process.poll()
-                if retcode:
-                    cmd = kwargs.get("args")
-                    if cmd is None:
-                        cmd = popenargs[0]
-                    raise subprocess.CalledProcessError(retcode, cmd)
-                return output
-            subprocess.check_output = f
-
-        self.start_time = datetime.datetime.now()
-        self.exit_code = 0
-        try:
-            try:
-                if os.name == 'nt':
-                    cmd = ['sh', '-c'] + self.cmd
-                else:
-                    cmd = self.cmd
-                self.output = subprocess.check_output(cmd, cwd = path.host(self.cwd))
-            except subprocess.CalledProcessError as cpe:
-                self.exit_code = cpe.returncode
-                self.output = cpe.output
-            except OSError as ose:
-                raise error.general('bootstrap failed: %s in %s: %s' % \
-                                        (' '.join(cmd), path.host(self.cwd), (str(ose))))
-            except KeyboardInterrupt:
-                pass
-            except:
-                raise
-        except:
-            self.result = sys.exc_info()
-        self.end_time = datetime.datetime.now()
-
-    def run(self):
-        self.thread = threading.Thread(target = self.runner)
-        self.thread.start()
-
-    def is_alive(self):
-        return self.thread and self.thread.is_alive()
-
-    def reraise(self):
-        if self.result is not None:
-            raise self.result[0](self.result[1]).with_traceback(self.result[2])
-
-class autoreconf:
-
-    def __init__(self, topdir, configure):
-        self.topdir = topdir
-        self.configure = configure
-        self.cwd = path.dirname(self.configure)
-        self.bspopts()
-        self.command = command(['autoreconf', '-i', '--no-recursive'], self.cwd)
-        self.command.run()
-
-    def bspopts(self):
-        if _grep(self.configure, 'RTEMS_CHECK_BSPDIR'):
-            bsp_specs = _collect(self.cwd, 'bsp_specs')
-            try:
-                acinclude = path.join(self.cwd, 'acinclude.m4')
-                b = open(path.host(acinclude), 'w')
-                b.write('# RTEMS_CHECK_BSPDIR(RTEMS_BSP_FAMILY)' + os.linesep)
-                b.write('AC_DEFUN([RTEMS_CHECK_BSPDIR],' + os.linesep)
-                b.write('[' + os.linesep)
-                b.write('  case "$1" in' + os.linesep)
-                for bs in sorted(bsp_specs):
-                    dir = path.dirname(bs)[len(self.cwd) + 1:]
-                    b.write('  %s )%s' % (dir, os.linesep))
-                    b.write('    AC_CONFIG_SUBDIRS([%s]);;%s' % (dir, os.linesep))
-                b.write('  *)' + os.linesep)
-                b.write('    AC_MSG_ERROR([Invalid BSP]);;' + os.linesep)
-                b.write('  esac' + os.linesep)
-                b.write('])' + os.linesep)
-                b.close()
-            except IOError as err:
-                raise error.general('writing: %s' % (acinclude))
-
-    def is_alive(self):
-        return self.command.is_alive()
-
-    def post_process(self):
-        if self.command is not None:
-            self.command.reraise()
-            if self.command.exit_code != 0:
-                raise error.general('error: autoreconf: %s' % (' '.join(self.command.cmd)))
-            makefile = path.join(self.cwd, 'Makefile.am')
-            if path.exists(makefile):
-                if _grep(makefile, 'stamp-h\.in'):
-                    stamp_h = path.join(self.cwd, 'stamp-h.in')
-                    try:
-                        t = open(path.host(stamp_h), 'w')
-                        t.write('timestamp')
-                        t.close()
-                    except IOError as err:
-                        raise error.general('writing: %s' % (stamp_h))
-
-def generate(topdir, jobs):
-    if type(jobs) is str:
-        jobs = int(jobs)
-    start_time = datetime.datetime.now()
-    confs = _collect(topdir, 'configure.ac')
-    next = 0
-    autoreconfs = []
-    while next < len(confs) or len(autoreconfs) > 0:
-        if next < len(confs) and len(autoreconfs) < jobs:
-            log.notice('%3d/%3d: autoreconf: %s' % \
-                           (next + 1, len(confs), confs[next][len(topdir) + 1:]))
-            autoreconfs += [autoreconf(topdir, confs[next])]
-            next += 1
-        else:
-            for ac in autoreconfs:
-                if not ac.is_alive():
-                    ac.post_process()
-                    autoreconfs.remove(ac)
-                    del ac
-            if len(autoreconfs) >= jobs:
-                time.sleep(1)
-    end_time = datetime.datetime.now()
-    log.notice('Bootstrap time: %s' % (str(end_time - start_time)))
-
-class ampolish3:
-
-    def __init__(self, topdir, makefile):
-        self.topdir = topdir
-        self.makefile = makefile
-        self.preinstall = path.join(path.dirname(makefile), 'preinstall.am')
-        self.command = command([path.join(topdir, 'ampolish3'), makefile], self.topdir)
-        self.command.run()
-
-    def is_alive(self):
-        return self.command.is_alive()
-
-    def post_process(self):
-        if self.command is not None:
-            if self.command.exit_code != 0:
-                raise error.general('error: ampolish3: %s' % (' '.join(self.command.cmd)))
-            try:
-                p = open(path.host(self.preinstall), 'w')
-                for l in self.command.output:
-                    p.write(l)
-                p.close()
-            except IOError as err:
-                raise error.general('writing: %s' % (self.preinstall))
-
-def preinstall(topdir, jobs):
-    if type(jobs) is str:
-        jobs = int(jobs)
-    start_time = datetime.datetime.now()
-    makes = []
-    for am in _collect(topdir, 'Makefile.am'):
-        if _grep(am, 'include .*/preinstall\.am'):
-            makes += [am]
-    next = 0
-    ampolish3s = []
-    while next < len(makes) or len(ampolish3s) > 0:
-        if next < len(makes) and len(ampolish3s) < jobs:
-            log.notice('%3d/%3d: ampolish3: %s' % \
-                           (next + 1, len(makes), makes[next][len(topdir) + 1:]))
-            ampolish3s += [ampolish3(topdir, makes[next])]
-            next += 1
-        else:
-            for ap in ampolish3s:
-                if not ap.is_alive():
-                    ap.post_process()
-                    ampolish3s.remove(ap)
-                    del ap
-            if len(ampolish3s) >= jobs:
-                time.sleep(1)
-    end_time = datetime.datetime.now()
-    log.notice('Preinstall time: %s' % (str(end_time - start_time)))
-
-def run(args):
-    try:
-        #
-        # On Windows MSYS2 prepends a path to itself to the environment
-        # path. This means the RTEMS specific automake is not found and which
-        # breaks the bootstrap. We need to remove the prepended path. Also
-        # remove any ACLOCAL paths from the environment.
-        #
-        if os.name == 'nt':
-            cspath = os.environ['PATH'].split(os.pathsep)
-            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
-                os.environ['PATH'] = os.pathsep.join(cspath[1:])
-            if 'ACLOCAL_PATH' in os.environ:
-                #
-                # The clear fails on a current MSYS2 python (Feb 2016). Delete
-                # the entry if the clear fails.
-                #
-                try:
-                    os.environ['ACLOCAL_PATH'].clear()
-                except:
-                    del os.environ['ACLOCAL_PATH']
-        optargs = { '--rtems':       'The RTEMS source directory',
-                    '--preinstall':  'Preinstall AM generation' }
-        log.notice('RTEMS Source Builder - RTEMS Bootstrap, %s' % (version.str()))
-        opts = options.load(sys.argv, optargs)
-        if opts.get_arg('--rtems'):
-            topdir = opts.get_arg('--rtems')
-        else:
-            topdir = os.getcwd()
-        if opts.get_arg('--preinstall'):
-            preinstall(topdir, opts.jobs(opts.defaults['_ncpus']))
-        else:
-            generate(topdir, opts.jobs(opts.defaults['_ncpus']))
-    except error.general as gerr:
-        print(gerr)
-        print('Bootstrap FAILED', file = sys.stderr)
-        sys.exit(1)
-    except error.internal as ierr:
-        print(ierr)
-        print('Bootstrap FAILED', file = sys.stderr)
-        sys.exit(1)
-    except error.exit as eerr:
-        pass
-    except KeyboardInterrupt:
-        log.notice('abort: user terminated')
-        sys.exit(1)
-    sys.exit(0)
-
-if __name__ == "__main__":
-    run(sys.argv)