From d8be0e080a9544835d6b48d05026e7332995009b Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Tue, 25 Jan 2022 03:13:51 +0000 Subject: sb-bootstrap: Update to support python3 Closes #4587 --- source-builder/sb-bootstrap | 259 +++++++++++++++++++++++++++++++++--- source-builder/sb/bootstrap.py | 296 ----------------------------------------- 2 files changed, 237 insertions(+), 318 deletions(-) delete mode 100644 source-builder/sb/bootstrap.py diff --git a/source-builder/sb-bootstrap b/source-builder/sb-bootstrap index a1ee9a7..6519710 100755 --- a/source-builder/sb-bootstrap +++ b/source-builder/sb-bootstrap @@ -1,31 +1,246 @@ #! /usr/bin/env python + # -# RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2013 Chris Johns (chrisj@rtems.org) +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (C) 2013-2019 Chris Johns (chrisj@rtems.org) # All rights reserved. # -# This file is part of the RTEMS Tools package in 'rtems-tools'. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. # -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. + +# +# RTEMS Tools Project (http://www.rtems.org/) # -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import print_function -import sys, os -base = os.path.dirname(sys.argv[0]) -sys.path.insert(0, base + '/sb') -try: - import bootstrap - bootstrap.run(sys.argv) -except ImportError: - print("Incorrect Source Builder installation", file = sys.stderr) - sys.exit(1) +import argparse +import datetime +import multiprocessing +import os +import re +import sys +import threading +import time + +version = "1.0" + +class error(Exception): + """Base class for Builder exceptions.""" + def set_output(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class general_error(error): + """Raise for a general error.""" + def __init__(self, what): + self.set_output('error: ' + str(what)) + +def _collect(path_, file): + confs = [] + for root, dirs, files in os.walk(path_, topdown = True): + for f in files: + if f == file: + confs += [os.path.join(root, f)] + return confs + +def _grep(file, pattern): + rege = re.compile(pattern) + try: + f = open(file, 'r') + matches = [rege.match(l) != None for l in f.readlines()] + f.close() + except IOError as err: + raise general_error('reading: %s' % (file)) + return True in matches + +class command: + + def __init__(self, cmd, cwd): + self.exit_code = 0 + self.thread = None + self.output = None + self.cmd = cmd + self.cwd = cwd + self.result = None + + def runner(self): + + import subprocess + + # + # Support Python 2.6 + # + if "check_output" not in dir(subprocess): + def f(*popenargs, **kwargs): + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output + subprocess.check_output = f + + self.start_time = datetime.datetime.now() + self.exit_code = 0 + try: + try: + if os.name == 'nt': + cmd = ['sh', '-c'] + self.cmd + else: + cmd = self.cmd + self.output = subprocess.check_output(cmd, cwd = self.cwd) + except subprocess.CalledProcessError as cpe: + self.exit_code = cpe.returncode + self.output = cpe.output + except OSError as ose: + raise general_error('bootstrap failed: %s in %s: %s' % \ + (' '.join(cmd), self.cwd, (str(ose)))) + except KeyboardInterrupt: + pass + except: + raise + except: + self.result = sys.exc_info() + self.end_time = datetime.datetime.now() + + def run(self): + self.thread = threading.Thread(target = self.runner) + self.thread.start() + + def is_alive(self): + return self.thread and self.thread.is_alive() + + def reraise(self): + if self.result is not None: + raise self.result[0](self.result[1]) + +class autoreconf: + + def __init__(self, topdir, configure): + self.topdir = topdir + self.configure = configure + self.cwd = os.path.dirname(self.configure) + self.command = command(['autoreconf', '-i', '--no-recursive'], self.cwd) + self.command.run() + + def is_alive(self): + return self.command.is_alive() + + def post_process(self): + if self.command is not None: + self.command.reraise() + if self.command.exit_code != 0: + raise general_error('error: autoreconf: %s' % (' '.join(self.command.cmd))) + makefile = os.path.join(self.cwd, 'Makefile.am') + if os.path.exists(makefile): + if _grep(makefile, 'stamp-h\.in'): + stamp_h = os.path.join(self.cwd, 'stamp-h.in') + try: + t = open(os.path.host(stamp_h), 'w') + t.write('timestamp') + t.close() + except IOError as err: + raise general_error('writing: %s' % (stamp_h)) + +def generate(topdir, jobs): + if type(jobs) is str: + jobs = int(jobs) + start_time = datetime.datetime.now() + confs = _collect(topdir, 'configure.ac') + next = 0 + autoreconfs = [] + while next < len(confs) or len(autoreconfs) > 0: + if next < len(confs) and len(autoreconfs) < jobs: + print('%3d/%3d: autoreconf: %s' % \ + (next + 1, len(confs), confs[next][len(topdir) + 1:])) + autoreconfs += [autoreconf(topdir, confs[next])] + next += 1 + else: + for ac in autoreconfs: + if not ac.is_alive(): + ac.post_process() + autoreconfs.remove(ac) + del ac + if len(autoreconfs) >= jobs: + time.sleep(1) + end_time = datetime.datetime.now() + print('Bootstrap time: %s' % (str(end_time - start_time))) + +def run(args): + try: + # + # On Windows MSYS2 prepends a path to itself to the environment + # path. This means the RTEMS specific automake is not found and which + # breaks the bootstrap. We need to remove the prepended path. Also + # remove any ACLOCAL paths from the environment. + # + if os.name == 'nt': + cspath = os.environ['PATH'].split(os.pathsep) + if 'msys' in cspath[0] and cspath[0].endswith('bin'): + os.environ['PATH'] = os.pathsep.join(cspath[1:]) + if 'ACLOCAL_PATH' in os.environ: + # + # The clear fails on a current MSYS2 python (Feb 2016). Delete + # the entry if the clear fails. + # + try: + os.environ['ACLOCAL_PATH'].clear() + except: + del os.environ['ACLOCAL_PATH'] + + argsp = argparse.ArgumentParser(prog = 'rtems-bootstrap', + description = "Bootstrap in parallel") + argsp.add_argument('-j', '--jobs', + help = 'number of jobs to run (default: %(default)s).', + type = int, default = multiprocessing.cpu_count()) + argsp.add_argument('-r', '--rtems', + type = str, default = os.getcwd(), + help = 'path to the rtems kernel source (default: %(default)s).') + argopts = argsp.parse_args(args[1:]) + + print('RTEMS Bootstrap, %s' % (version)) + + if not os.path.exists(argopts.rtems): + raise general_error('path does not exist: %s' % (argopts.rtems)) + if not os.path.isdir(argopts.rtems): + raise general_error('path not a directory: %s' % (argopts.rtems)) + + generate(argopts.rtems, argopts.jobs) + except general_error as gerr: + print(gerr) + print('Bootstrap FAILED', file = sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + log.notice('abort: user terminated') + sys.exit(1) + sys.exit(0) + +if __name__ == "__main__": + run(sys.argv) diff --git a/source-builder/sb/bootstrap.py b/source-builder/sb/bootstrap.py deleted file mode 100644 index c9acf5c..0000000 --- a/source-builder/sb/bootstrap.py +++ /dev/null @@ -1,296 +0,0 @@ -# -# RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2013-2016 Chris Johns (chrisj@rtems.org) -# All rights reserved. -# -# This file is part of the RTEMS Tools package in 'rtems-tools'. -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -from __future__ import print_function - -import datetime -import operator -import os -import re -import sys -import threading -import time - -import error -import log -import options -import path -import version - -def _collect(path_, file): - confs = [] - for root, dirs, files in os.walk(path.host(path_), topdown = True): - for f in files: - if f == file: - confs += [path.shell(path.join(root, f))] - return confs - -def _grep(file, pattern): - rege = re.compile(pattern) - try: - f = open(path.host(file), 'r') - matches = [rege.match(l) != None for l in f.readlines()] - f.close() - except IOError as err: - raise error.general('reading: %s' % (file)) - return True in matches - -class command: - - def __init__(self, cmd, cwd): - self.exit_code = 0 - self.thread = None - self.output = None - self.cmd = cmd - self.cwd = cwd - self.result = None - - def runner(self): - - import subprocess - - # - # Support Python 2.6 - # - if "check_output" not in dir(subprocess): - def f(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output - subprocess.check_output = f - - self.start_time = datetime.datetime.now() - self.exit_code = 0 - try: - try: - if os.name == 'nt': - cmd = ['sh', '-c'] + self.cmd - else: - cmd = self.cmd - self.output = subprocess.check_output(cmd, cwd = path.host(self.cwd)) - except subprocess.CalledProcessError as cpe: - self.exit_code = cpe.returncode - self.output = cpe.output - except OSError as ose: - raise error.general('bootstrap failed: %s in %s: %s' % \ - (' '.join(cmd), path.host(self.cwd), (str(ose)))) - except KeyboardInterrupt: - pass - except: - raise - except: - self.result = sys.exc_info() - self.end_time = datetime.datetime.now() - - def run(self): - self.thread = threading.Thread(target = self.runner) - self.thread.start() - - def is_alive(self): - return self.thread and self.thread.is_alive() - - def reraise(self): - if self.result is not None: - raise self.result[0](self.result[1]).with_traceback(self.result[2]) - -class autoreconf: - - def __init__(self, topdir, configure): - self.topdir = topdir - self.configure = configure - self.cwd = path.dirname(self.configure) - self.bspopts() - self.command = command(['autoreconf', '-i', '--no-recursive'], self.cwd) - self.command.run() - - def bspopts(self): - if _grep(self.configure, 'RTEMS_CHECK_BSPDIR'): - bsp_specs = _collect(self.cwd, 'bsp_specs') - try: - acinclude = path.join(self.cwd, 'acinclude.m4') - b = open(path.host(acinclude), 'w') - b.write('# RTEMS_CHECK_BSPDIR(RTEMS_BSP_FAMILY)' + os.linesep) - b.write('AC_DEFUN([RTEMS_CHECK_BSPDIR],' + os.linesep) - b.write('[' + os.linesep) - b.write(' case "$1" in' + os.linesep) - for bs in sorted(bsp_specs): - dir = path.dirname(bs)[len(self.cwd) + 1:] - b.write(' %s )%s' % (dir, os.linesep)) - b.write(' AC_CONFIG_SUBDIRS([%s]);;%s' % (dir, os.linesep)) - b.write(' *)' + os.linesep) - b.write(' AC_MSG_ERROR([Invalid BSP]);;' + os.linesep) - b.write(' esac' + os.linesep) - b.write('])' + os.linesep) - b.close() - except IOError as err: - raise error.general('writing: %s' % (acinclude)) - - def is_alive(self): - return self.command.is_alive() - - def post_process(self): - if self.command is not None: - self.command.reraise() - if self.command.exit_code != 0: - raise error.general('error: autoreconf: %s' % (' '.join(self.command.cmd))) - makefile = path.join(self.cwd, 'Makefile.am') - if path.exists(makefile): - if _grep(makefile, 'stamp-h\.in'): - stamp_h = path.join(self.cwd, 'stamp-h.in') - try: - t = open(path.host(stamp_h), 'w') - t.write('timestamp') - t.close() - except IOError as err: - raise error.general('writing: %s' % (stamp_h)) - -def generate(topdir, jobs): - if type(jobs) is str: - jobs = int(jobs) - start_time = datetime.datetime.now() - confs = _collect(topdir, 'configure.ac') - next = 0 - autoreconfs = [] - while next < len(confs) or len(autoreconfs) > 0: - if next < len(confs) and len(autoreconfs) < jobs: - log.notice('%3d/%3d: autoreconf: %s' % \ - (next + 1, len(confs), confs[next][len(topdir) + 1:])) - autoreconfs += [autoreconf(topdir, confs[next])] - next += 1 - else: - for ac in autoreconfs: - if not ac.is_alive(): - ac.post_process() - autoreconfs.remove(ac) - del ac - if len(autoreconfs) >= jobs: - time.sleep(1) - end_time = datetime.datetime.now() - log.notice('Bootstrap time: %s' % (str(end_time - start_time))) - -class ampolish3: - - def __init__(self, topdir, makefile): - self.topdir = topdir - self.makefile = makefile - self.preinstall = path.join(path.dirname(makefile), 'preinstall.am') - self.command = command([path.join(topdir, 'ampolish3'), makefile], self.topdir) - self.command.run() - - def is_alive(self): - return self.command.is_alive() - - def post_process(self): - if self.command is not None: - if self.command.exit_code != 0: - raise error.general('error: ampolish3: %s' % (' '.join(self.command.cmd))) - try: - p = open(path.host(self.preinstall), 'w') - for l in self.command.output: - p.write(l) - p.close() - except IOError as err: - raise error.general('writing: %s' % (self.preinstall)) - -def preinstall(topdir, jobs): - if type(jobs) is str: - jobs = int(jobs) - start_time = datetime.datetime.now() - makes = [] - for am in _collect(topdir, 'Makefile.am'): - if _grep(am, 'include .*/preinstall\.am'): - makes += [am] - next = 0 - ampolish3s = [] - while next < len(makes) or len(ampolish3s) > 0: - if next < len(makes) and len(ampolish3s) < jobs: - log.notice('%3d/%3d: ampolish3: %s' % \ - (next + 1, len(makes), makes[next][len(topdir) + 1:])) - ampolish3s += [ampolish3(topdir, makes[next])] - next += 1 - else: - for ap in ampolish3s: - if not ap.is_alive(): - ap.post_process() - ampolish3s.remove(ap) - del ap - if len(ampolish3s) >= jobs: - time.sleep(1) - end_time = datetime.datetime.now() - log.notice('Preinstall time: %s' % (str(end_time - start_time))) - -def run(args): - try: - # - # On Windows MSYS2 prepends a path to itself to the environment - # path. This means the RTEMS specific automake is not found and which - # breaks the bootstrap. We need to remove the prepended path. Also - # remove any ACLOCAL paths from the environment. - # - if os.name == 'nt': - cspath = os.environ['PATH'].split(os.pathsep) - if 'msys' in cspath[0] and cspath[0].endswith('bin'): - os.environ['PATH'] = os.pathsep.join(cspath[1:]) - if 'ACLOCAL_PATH' in os.environ: - # - # The clear fails on a current MSYS2 python (Feb 2016). Delete - # the entry if the clear fails. - # - try: - os.environ['ACLOCAL_PATH'].clear() - except: - del os.environ['ACLOCAL_PATH'] - optargs = { '--rtems': 'The RTEMS source directory', - '--preinstall': 'Preinstall AM generation' } - log.notice('RTEMS Source Builder - RTEMS Bootstrap, %s' % (version.str())) - opts = options.load(sys.argv, optargs) - if opts.get_arg('--rtems'): - topdir = opts.get_arg('--rtems') - else: - topdir = os.getcwd() - if opts.get_arg('--preinstall'): - preinstall(topdir, opts.jobs(opts.defaults['_ncpus'])) - else: - generate(topdir, opts.jobs(opts.defaults['_ncpus'])) - except error.general as gerr: - print(gerr) - print('Bootstrap FAILED', file = sys.stderr) - sys.exit(1) - except error.internal as ierr: - print(ierr) - print('Bootstrap FAILED', file = sys.stderr) - sys.exit(1) - except error.exit as eerr: - pass - except KeyboardInterrupt: - log.notice('abort: user terminated') - sys.exit(1) - sys.exit(0) - -if __name__ == "__main__": - run(sys.argv) -- cgit v1.2.3