From 3a972f610213bab8374b5518f6a139ceed95823a Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Mon, 7 Mar 2016 11:56:02 +1100 Subject: sb: Update code base to support Python3 and Python2. Fix Windows support to allow MSYS2 Python to be used. Updates #2619. --- source-builder/pkg-config | 24 +++++----- source-builder/sb-bootstrap | 4 +- source-builder/sb-builder | 4 +- source-builder/sb-check | 4 +- source-builder/sb-defaults | 4 +- source-builder/sb-reports | 4 +- source-builder/sb-rtems-config | 4 +- source-builder/sb-set-builder | 7 ++- source-builder/sb/bootstrap.py | 30 ++++++------ source-builder/sb/build.py | 30 ++++++------ source-builder/sb/check.py | 20 ++++---- source-builder/sb/config.py | 41 ++++++++-------- source-builder/sb/cvs.py | 12 +++-- source-builder/sb/download.py | 41 +++++++++------- source-builder/sb/error.py | 10 ++-- source-builder/sb/execute.py | 28 ++++++----- source-builder/sb/git.py | 18 +++---- source-builder/sb/linux.py | 2 +- source-builder/sb/log.py | 46 +++++++++--------- source-builder/sb/macros.py | 67 ++++++++++++++++++-------- source-builder/sb/mailer.py | 12 +++-- source-builder/sb/options.py | 101 +++++++++++++++++++++------------------ source-builder/sb/path.py | 64 +++++++++++++------------ source-builder/sb/pkgconfig.py | 8 ++-- source-builder/sb/reports.py | 38 ++++++++------- source-builder/sb/rtemsconfig.py | 26 +++++----- source-builder/sb/setbuilder.py | 27 ++++++----- source-builder/sb/version.py | 10 ++-- source-builder/sb/windows.py | 36 +++++++------- 29 files changed, 410 insertions(+), 312 deletions(-) diff --git a/source-builder/pkg-config b/source-builder/pkg-config index c2378d6..d2afff0 100755 --- a/source-builder/pkg-config +++ b/source-builder/pkg-config @@ -1,7 +1,7 @@ #! /usr/bin/env python # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2014 Chris Johns (chrisj@rtems.org) +# Copyright 2014-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -29,6 +29,8 @@ # POSSIBILITY OF SUCH DAMAGE. # +from __future__ import print_function + import os import sys @@ -42,13 +44,13 @@ except: try: import argparse except: - print >> sys.stderr, "Incorrect Source Builder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) try: import pkgconfig except ImportError: - print >> sys.stderr, "Incorrect Source Builder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) # @@ -81,12 +83,12 @@ def log(s, lf = True): out = sys.stdout if lf: if out != sys.stdout and trace_stdout: - print s - print >> out, s + print(s) + print(s, file = out) else: if out != sys.stdout and trace_stdout: - print s, - print >> out, s, + print(s, end = '', flush = True) + print(out, s, end = '', flush = True) def run(argv): @@ -191,13 +193,13 @@ def run(argv): if ec == 0: if args.cflags: if len(flags['cflags']): - print flags['cflags'] + print(flags['cflags']) log('cflags: %s' % (flags['cflags'])) else: log('cflags: empty') if args.libs: if len(flags['libs']): - print flags['libs'] + print(flags['libs']) log('libs: %s' % (flags['libs'])) else: log('libs: empty') @@ -215,9 +217,9 @@ try: ec = run(sys.argv) log('ec = %d' % (ec)) except ImportError: - print >> sys.stderr, "incorrect package config installation" + print("incorrect package config installation", file = sys.stderr) sys.exit(1) except pkgconfig.error, e: - print >> sys.stderr, 'error: %s' % (e) + print('error: %s' % (e), file = sys.stderr) sys.exit(1) sys.exit(ec) diff --git a/source-builder/sb-bootstrap b/source-builder/sb-bootstrap index 0befd8d..a1ee9a7 100755 --- a/source-builder/sb-bootstrap +++ b/source-builder/sb-bootstrap @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import bootstrap bootstrap.run(sys.argv) except ImportError: - print >> sys.stderr, "Incorrect Source Builder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-builder b/source-builder/sb-builder index 440d481..8721b56 100755 --- a/source-builder/sb-builder +++ b/source-builder/sb-builder @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import build build.run(sys.argv) except ImportError: - print >> sys.stderr, "Incorrect Source Builder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-check b/source-builder/sb-check index 43aae67..d23b799 100755 --- a/source-builder/sb-check +++ b/source-builder/sb-check @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import check check.run() except ImportError: - print >> sys.stderr, "Incorrect Set Bulder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-defaults b/source-builder/sb-defaults index 37dac2a..790a5e5 100755 --- a/source-builder/sb-defaults +++ b/source-builder/sb-defaults @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import options options.run(sys.argv) except ImportError: - print >> sys.stderr, "Incorrect Defaults installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-reports b/source-builder/sb-reports index ba9db1c..3330be3 100755 --- a/source-builder/sb-reports +++ b/source-builder/sb-reports @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import reports reports.run(sys.argv) except ImportError: - print >> sys.stderr, "Incorrect Defaults installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-rtems-config b/source-builder/sb-rtems-config index ac5b269..1633b6b 100755 --- a/source-builder/sb-rtems-config +++ b/source-builder/sb-rtems-config @@ -18,6 +18,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') @@ -25,5 +27,5 @@ try: import rtemsconfig rtemsconfig.run(sys.argv) except ImportError: - print >> sys.stderr, "Incorrect Set Bulder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb-set-builder b/source-builder/sb-set-builder index 7fb4f3a..52e122b 100755 --- a/source-builder/sb-set-builder +++ b/source-builder/sb-set-builder @@ -18,12 +18,17 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + import sys, os base = os.path.dirname(sys.argv[0]) sys.path.insert(0, base + '/sb') + +import setbuilder + try: import setbuilder setbuilder.run() except ImportError: - print >> sys.stderr, "Incorrect Set Bulder installation" + print("Incorrect Source Builder installation", file = sys.stderr) sys.exit(1) diff --git a/source-builder/sb/bootstrap.py b/source-builder/sb/bootstrap.py index 2e585c2..c9acf5c 100644 --- a/source-builder/sb/bootstrap.py +++ b/source-builder/sb/bootstrap.py @@ -18,6 +18,8 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # +from __future__ import print_function + import datetime import operator import os @@ -46,7 +48,7 @@ def _grep(file, pattern): f = open(path.host(file), 'r') matches = [rege.match(l) != None for l in f.readlines()] f.close() - except IOError, err: + except IOError as err: raise error.general('reading: %s' % (file)) return True in matches @@ -91,10 +93,10 @@ class command: else: cmd = self.cmd self.output = subprocess.check_output(cmd, cwd = path.host(self.cwd)) - except subprocess.CalledProcessError, cpe: + except subprocess.CalledProcessError as cpe: self.exit_code = cpe.returncode self.output = cpe.output - except OSError, ose: + except OSError as ose: raise error.general('bootstrap failed: %s in %s: %s' % \ (' '.join(cmd), path.host(self.cwd), (str(ose)))) except KeyboardInterrupt: @@ -114,7 +116,7 @@ class command: def reraise(self): if self.result is not None: - raise self.result[0], self.result[1], self.result[2] + raise self.result[0](self.result[1]).with_traceback(self.result[2]) class autoreconf: @@ -145,7 +147,7 @@ class autoreconf: b.write(' esac' + os.linesep) b.write('])' + os.linesep) b.close() - except IOError, err: + except IOError as err: raise error.general('writing: %s' % (acinclude)) def is_alive(self): @@ -164,7 +166,7 @@ class autoreconf: t = open(path.host(stamp_h), 'w') t.write('timestamp') t.close() - except IOError, err: + except IOError as err: raise error.general('writing: %s' % (stamp_h)) def generate(topdir, jobs): @@ -212,7 +214,7 @@ class ampolish3: for l in self.command.output: p.write(l) p.close() - except IOError, err: + except IOError as err: raise error.general('writing: %s' % (self.preinstall)) def preinstall(topdir, jobs): @@ -275,15 +277,15 @@ def run(args): preinstall(topdir, opts.jobs(opts.defaults['_ncpus'])) else: generate(topdir, opts.jobs(opts.defaults['_ncpus'])) - except error.general, gerr: - print gerr - print >> sys.stderr, 'Bootstrap FAILED' + except error.general as gerr: + print(gerr) + print('Bootstrap FAILED', file = sys.stderr) sys.exit(1) - except error.internal, ierr: - print ierr - print >> sys.stderr, 'Bootstrap FAILED' + except error.internal as ierr: + print(ierr) + print('Bootstrap FAILED', file = sys.stderr) sys.exit(1) - except error.exit, eerr: + except error.exit as eerr: pass except KeyboardInterrupt: log.notice('abort: user terminated') diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index ad24e86..4d8787c 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -22,6 +22,8 @@ # installed not to be package unless you run a packager around this. # +from __future__ import print_function + import copy import getopt import glob @@ -29,8 +31,6 @@ import os import shutil import stat import sys -import urllib2 -import urlparse try: import check @@ -45,10 +45,10 @@ try: import sources import version except KeyboardInterrupt: - print 'abort: user terminated' + print('abort: user terminated') sys.exit(1) except: - print 'error: unknown application load error' + print('error: unknown application load error') sys.exit(1) class script: @@ -81,7 +81,7 @@ class script: os.chmod(path.host(name), stat.S_IRWXU | \ stat.S_IRGRP | stat.S_IXGRP | \ stat.S_IROTH | stat.S_IXOTH) - except IOError, err: + except IOError as err: raise error.general('creating script: ' + name) except: if s is not None: @@ -128,11 +128,11 @@ class build: self.config = config.file(name, opts, self.macros) self.script = script() self.macros['buildname'] = self._name_(self.macros['name']) - except error.general, gerr: + except error.general as gerr: log.notice(str(gerr)) log.stderr('Build FAILED') raise - except error.internal, ierr: + except error.internal as ierr: log.notice(str(ierr)) log.stderr('Internal Build FAILED') raise @@ -232,7 +232,7 @@ class build: args = args[1:] try: opts, args = getopt.getopt(args[1:], 'qDcn:ba') - except getopt.GetoptError, ge: + except getopt.GetoptError as ge: raise error.general('source setup error: %s' % str(ge)) quiet = False unpack_before_chdir = True @@ -473,12 +473,12 @@ class build: self.script.write(sn) log.notice('building: %s%s' % (cxc_label, name)) self.run(sn) - except error.general, gerr: + except error.general as gerr: log.notice(str(gerr)) log.stderr('Build FAILED') self._generate_report_('Build: %s' % (gerr)) raise - except error.internal, ierr: + except error.internal as ierr: log.notice(str(ierr)) log.stderr('Internal Build FAILED') self._generate_report_('Build: %s' % (ierr)) @@ -546,22 +546,22 @@ def run(args): if opts.get_arg('--list-configs'): configs = get_configs(opts) for p in configs['paths']: - print 'Examining: %s' % (os.path.relpath(p)) + print('Examining: %s' % (os.path.relpath(p))) for c in configs['files']: if c.endswith('.cfg'): - print ' %s' % (c) + print(' %s' % (c)) else: for config_file in opts.config_files(): b = build(config_file, True, opts) b.make() b = None - except error.general, gerr: + except error.general as gerr: log.stderr('Build FAILED') ec = 1 - except error.internal, ierr: + except error.internal as ierr: log.stderr('Internal Build FAILED') ec = 1 - except error.exit, eerr: + except error.exit as eerr: pass except KeyboardInterrupt: log.notice('abort: user terminated') diff --git a/source-builder/sb/check.py b/source-builder/sb/check.py index 9930fca..3e847ce 100644 --- a/source-builder/sb/check.py +++ b/source-builder/sb/check.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2012 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -21,6 +21,8 @@ # Check the defaults for a specific host. # +from __future__ import print_function + import os import error @@ -130,7 +132,7 @@ def host_setup(opts): sane = True - for d in opts.defaults.keys(): + for d in list(opts.defaults.keys()): try: (test, constraint, value) = opts.defaults.get(d) except: @@ -164,16 +166,16 @@ def run(): _opts = options.load(args = sys.argv) log.notice('RTEMS Source Builder - Check, %s' % (version.str())) if host_setup(_opts): - print 'Environment is ok' + print('Environment is ok') else: - print 'Environment is not correctly set up' - except error.general, gerr: - print gerr + print('Environment is not correctly set up') + except error.general as gerr: + print(gerr) sys.exit(1) - except error.internal, ierr: - print ierr + except error.internal as ierr: + print(ierr) sys.exit(1) - except error.exit, eerr: + except error.exit as eerr: pass except KeyboardInterrupt: log.notice('abort: user terminated') diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py index 441ce6c..8a0c01b 100644 --- a/source-builder/sb/config.py +++ b/source-builder/sb/config.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2013 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -25,7 +25,10 @@ # other software modules. # +from __future__ import print_function + import copy +from functools import reduce import os import re import sys @@ -39,10 +42,10 @@ try: import pkgconfig import sources except KeyboardInterrupt: - print 'user terminated' + print('user terminated', file = sys.stderr) sys.exit(1) except: - print 'error: unknown application load error' + print('error: unknown application load error', file = sys.stderr) sys.exit(1) def _check_bool(value): @@ -75,7 +78,7 @@ class package: def _dictlist(dl): s = '' - dll = dl.keys() + dll = list(dl.keys()) dll.sort() for d in dll: if d: @@ -250,7 +253,7 @@ class file: def _dict(dd): s = '' - ddl = dd.keys() + ddl = list(dd.keys()) ddl.sort() for d in ddl: s += ' ' + d + ': ' + dd[d] + '\n' @@ -339,14 +342,14 @@ class file: outter level. Nested levels will need to split with futher calls.''' trace_me = False if trace_me: - print '------------------------------------------------------' + print('------------------------------------------------------') macros = [] nesting = [] has_braces = False c = 0 while c < len(s): if trace_me: - print 'ms:', c, '"' + s[c:] + '"', has_braces, len(nesting), nesting + print('ms:', c, '"' + s[c:] + '"', has_braces, len(nesting), nesting) # # We need to watch for shell type variables or the form '${var}' because # they can upset the brace matching. @@ -394,9 +397,9 @@ class file: macros.append(s[macro_start:c + 1].strip()) c += 1 if trace_me: - print 'ms:', macros + print('ms:', macros) if trace_me: - print '-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=' + print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') return macros def _shell(self, line): @@ -435,7 +438,7 @@ class file: op = test[1] ver = self.macros.expand(test[2]) ok = pkg.check(op, ver) - except pkgconfig.error, pe: + except pkgconfig.error as pe: self._error('pkgconfig: check: %s' % (pe)) except: raise error.internal('pkgconfig failure') @@ -459,7 +462,7 @@ class file: fflags += [f] pkg_flags = ' '.join(fflags) log.trace('pkgconfig: %s: %s' % (flags, pkg_flags)) - except pkgconfig.error, pe: + except pkgconfig.error as pe: self._error('pkgconfig: %s: %s' % (flags, pe)) except: raise error.internal('pkgconfig failure') @@ -1127,7 +1130,7 @@ class file: try: log.trace('config: %s: _open: %s' % (self.name, path.host(configname))) config = open(path.host(configname), 'r') - except IOError, err: + except IOError as err: raise error.general('error opening config file: %s' % (path.host(configname))) self.configpath += [configname] @@ -1171,7 +1174,7 @@ class file: self.load_depth -= 1 def defined(self, name): - return self.macros.has_key(name) + return name in self.macros def define(self, name): if name in self.macros: @@ -1229,14 +1232,14 @@ def run(): opts = options.load(sys.argv, defaults = 'defaults.mc') log.trace('config: count %d' % (len(opts.config_files()))) for config_file in opts.config_files(): - s = file(config_file, opts) - print s + s = open(config_file, opts) + print(s) del s - except error.general, gerr: - print gerr + except error.general as gerr: + print(gerr) sys.exit(1) - except error.internal, ierr: - print ierr + except error.internal as ierr: + print(ierr) sys.exit(1) except KeyboardInterrupt: log.notice('abort: user terminated') diff --git a/source-builder/sb/cvs.py b/source-builder/sb/cvs.py index b07b836..2ff7417 100644 --- a/source-builder/sb/cvs.py +++ b/source-builder/sb/cvs.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2013 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -21,6 +21,8 @@ # Provide some basic access to the cvs command. # +from __future__ import print_function + import os import error @@ -151,8 +153,8 @@ if __name__ == '__main__': if not path.exists(ldir): path.mkdir(ldir) c.checkout(':pserver:anoncvs@sourceware.org:/cvs/src', module = 'newlib') - print c.cvs_version() - print c.valid() - print c.status() + print(c.cvs_version()) + print(c.valid()) + print(c.status()) c.reset() - print c.clean() + print(c.clean()) diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index ffed422..f6f6622 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -22,12 +22,20 @@ # installed not to be package unless you run a packager around this. # +from __future__ import print_function + import hashlib import os import stat import sys -import urllib2 -import urlparse +try: + import urllib.request as urllib_request + import urllib.parse as urllib_parse + import urllib.error as urllib_error +except ImportError: + import urllib as urllib_request + import urllib as urllib_parse + import urllib as urllib_error import cvs import error @@ -49,11 +57,11 @@ def _do_download(opts): def _humanize_bytes(bytes, precision = 1): abbrevs = ( - (1 << 50L, 'PB'), - (1 << 40L, 'TB'), - (1 << 30L, 'GB'), - (1 << 20L, 'MB'), - (1 << 10L, 'kB'), + (1 << 50, 'PB'), + (1 << 40, 'TB'), + (1 << 30, 'GB'), + (1 << 20, 'MB'), + (1 << 10, 'kB'), (1, ' bytes') ) if bytes == 1: @@ -82,7 +90,7 @@ def _hash_check(file_, absfile, macros, remove = True): hasher = hashlib.new(hash[0]) _in = open(path.host(absfile), 'rb') hasher.update(_in.read()) - except IOError, err: + except IOError as err: log.notice('hash: %s: read error: %s' % (file_, str(err))) failed = True except: @@ -103,7 +111,7 @@ def _hash_check(file_, absfile, macros, remove = True): if path.exists(absfile): try: os.remove(path.host(absfile)) - except IOError, err: + except IOError as err: raise error.general('hash: %s: remove: %s' % (absfile, str(err))) except: raise error.general('hash: %s: remove error' % (file_)) @@ -317,7 +325,7 @@ def _http_downloader(url, local, config, opts): # Hack for GitHub. # if url.startswith('https://api.github.com'): - url = urlparse.urljoin(url, config.expand('tarball/%{version}')) + url = urllib_parse.urljoin(url, config.expand('tarball/%{version}')) dst = os.path.relpath(path.host(local)) log.notice('download: %s -> %s' % (url, dst)) failed = False @@ -335,14 +343,15 @@ def _http_downloader(url, local, config, opts): try: _in = None _ssl_context = None + _urllib_url = url try: import ssl _ssl_context = ssl._create_unverified_context() - _in = urllib2.urlopen(url, context = _ssl_context) + _in = urllib_request.urlopen(_urllib_url, context = _ssl_context) except: _ssl_context = None if _ssl_context is None: - _in = urllib2.urlopen(url) + _in = urllib_request.urlopen(_urllib_url) if url != _in.geturl(): log.notice(' redirect: %s' % (_in.geturl())) _out = open(path.host(local), 'wb') @@ -372,12 +381,12 @@ def _http_downloader(url, local, config, opts): except: log.stdout_raw('\n') raise - except IOError, err: + except IOError as err: log.notice('download: %s: error: %s' % (url, str(err))) if path.exists(local): os.remove(path.host(local)) failed = True - except ValueError, err: + except ValueError as err: log.notice('download: %s: error: %s' % (url, str(err))) if path.exists(local): os.remove(path.host(local)) @@ -570,7 +579,7 @@ def get_file(url, local, opts, config): # # Split up the URL we are being asked to download. # - url_path = urlparse.urlsplit(url)[2] + url_path = urllib_parse.urlsplit(url)[2] slash = url_path.rfind('/') if slash < 0: url_file = url_path @@ -580,7 +589,7 @@ def get_file(url, local, opts, config): for base in url_bases: if base[-1:] != '/': base += '/' - next_url = urlparse.urljoin(base, url_file) + next_url = urllib_parse.urljoin(base, url_file) log.trace('url: %s' %(next_url)) urls.append(next_url) urls += url.split() diff --git a/source-builder/sb/error.py b/source-builder/sb/error.py index 5617c17..6a99b0e 100644 --- a/source-builder/sb/error.py +++ b/source-builder/sb/error.py @@ -17,6 +17,8 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import print_function + # # Various errors we can raise. # @@ -46,9 +48,9 @@ class exit(error): if __name__ == '__main__': try: raise general('a general error') - except general, gerr: - print 'caught:', gerr + except general as gerr: + print('caught:', gerr) try: raise internal('an internal error') - except internal, ierr: - print 'caught:', ierr + except internal as ierr: + print('caught:', ierr) diff --git a/source-builder/sb/execute.py b/source-builder/sb/execute.py index 92a5e83..12d8114 100755 --- a/source-builder/sb/execute.py +++ b/source-builder/sb/execute.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2012 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -23,6 +23,9 @@ # Note, the subprocess module is only in Python 2.4 or higher. # +from __future__ import print_function + +import functools import os import re import sys @@ -81,7 +84,7 @@ def arg_subst(command, substs): def arg_subst_str(command, subst): cmd = arg_subst(command, subst) def add(x, y): return x + ' ' + str(y) - return reduce(add, cmd, '') + return functools.reduce(add, cmd, '') class execute: """Execute commands or scripts. The 'output' is a funtion @@ -105,6 +108,9 @@ class execute: count = 0 while True: line = fh.readline() + # str and bytes are the same type in Python2 + if type(line) is not str and type(line) is bytes: + line = line.decode(sys.stdout.encoding) count += 1 if len(line) == 0: break @@ -155,7 +161,7 @@ class execute: s = command if type(command) is list: def add(x, y): return x + ' ' + str(y) - s = reduce(add, command, '')[1:] + s = functools.reduce(add, command, '')[1:] what = 'spawn' if shell: what = 'shell' @@ -191,7 +197,7 @@ class execute: exit_code = self.capture(proc) if self.verbose: log.output('exit: ' + str(exit_code)) - except OSError, ose: + except OSError as ose: exit_code = ose.errno if self.verbose: log.output('exit: ' + str(ose)) @@ -325,9 +331,9 @@ if __name__ == "__main__": ec, proc = e.command(commands['pipe'][0], commands['pipe'][1], capture = False, stdin = subprocess.PIPE) if ec == 0: - print 'piping input into ' + commands['pipe'][0] + ': ' + \ - commands['pipe'][2] - proc.stdin.write(commands['pipe'][2]) + print('piping input into ' + commands['pipe'][0] + ': ' + \ + commands['pipe'][2]) + proc.stdin.write(bytes(commands['pipe'][2], sys.stdin.encoding)) proc.stdin.close() e.capture(proc) del proc @@ -352,10 +358,10 @@ if __name__ == "__main__": ('date %0 %1', ['-u', '+%d %D %S'])] commands['unix']['pipe'] = ('grep', 'hello', 'hello world') - print arg_list('cmd a1 a2 "a3 is a string" a4') - print arg_list('cmd b1 b2 "b3 is a string a4') - print arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'], - ['subst0', 'subst1', 'subst2']) + print(arg_list('cmd a1 a2 "a3 is a string" a4')) + print(arg_list('cmd b1 b2 "b3 is a string a4')) + print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'], + ['subst0', 'subst1', 'subst2'])) e = execute(error_prefix = 'ERR: ', verbose = True) if sys.platform == "win32": diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py index 19fbed0..f35c335 100644 --- a/source-builder/sb/git.py +++ b/source-builder/sb/git.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2013 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -21,6 +21,8 @@ # Provide some basic access to the git command. # +from __future__ import print_function + import os import error @@ -201,10 +203,10 @@ if __name__ == '__main__': import sys opts = options.load(sys.argv) g = repo('.', opts) - print g.git_version() - print g.valid() - print g.status() - print g.clean() - print g.remotes() - print g.email() - print g.head() + print(g.git_version()) + print(g.valid()) + print(g.status()) + print(g.clean()) + print(g.remotes()) + print(g.email()) + print(g.head()) diff --git a/source-builder/sb/linux.py b/source-builder/sb/linux.py index 4ab63c5..9d90288 100644 --- a/source-builder/sb/linux.py +++ b/source-builder/sb/linux.py @@ -121,7 +121,7 @@ def load(): '__chown': ('exe', 'required', '/usr/sbin/chown') }, } - if variations.has_key(distro): + if distro in variations: for v in variations[distro]: if path.exists(variations[distro][v][2]): defines[v] = variations[distro][v] diff --git a/source-builder/sb/log.py b/source-builder/sb/log.py index ba31654..cc92dd3 100755 --- a/source-builder/sb/log.py +++ b/source-builder/sb/log.py @@ -21,6 +21,8 @@ # Log output to stdout and/or a file. # +from __future__ import print_function + import os import sys @@ -56,16 +58,16 @@ def _output(text = os.linesep, log = None): default.output(text) else: for l in text.replace(chr(13), '').splitlines(): - print l + print(l) sys.stdout.flush() def stdout_raw(text = os.linesep): - print text, + print(text, end=' ') sys.stdout.flush() def stderr(text = os.linesep, log = None): for l in text.replace(chr(13), '').splitlines(): - print >> sys.stderr, l + print(l, file = sys.stderr) sys.stderr.flush() def output(text = os.linesep, log = None): @@ -75,7 +77,7 @@ def output(text = os.linesep, log = None): def notice(text = os.linesep, log = None): if not quiet and default is not None and not default.has_stdout(): for l in text.replace(chr(13), '').splitlines(): - print l + print(l) sys.stdout.flush() _output(text, log) @@ -114,8 +116,8 @@ class log: self.fhs[1] = sys.stderr else: try: - self.fhs.append(file(s, 'w')) - except IOError, ioe: + self.fhs.append(open(s, 'w')) + except IOError as ioe: raise error.general("creating log file '" + s + \ "': " + str(ioe)) @@ -168,41 +170,41 @@ if __name__ == "__main__": l.output('log: hello world CRLF\r\n') l.output('log: hello world NONE') l.flush() - print '=-' * 40 - print 'tail: %d' % (len(l.tail)) - print l - print '=-' * 40 + print('=-' * 40) + print('tail: %d' % (len(l.tail))) + print(l) + print('=-' * 40) for i in range(0, 10): l.output('log: hello world 2: %d\n' % (i)) l.flush() - print '=-' * 40 - print 'tail: %d' % (len(l.tail)) - print l - print '=-' * 40 + print('=-' * 40) + print('tail: %d' % (len(l.tail))) + print(l) + print('=-' * 40) for i in [0, 1]: quiet = False tracing = False - print '- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30) + print('- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30)) trace('trace with quiet and trace off') notice('notice with quiet and trace off') quiet = True tracing = False - print '- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30) + print('- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30)) trace('trace with quiet on and trace off') notice('notice with quiet on and trace off') quiet = False tracing = True - print '- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30) + print('- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30)) trace('trace with quiet off and trace on') notice('notice with quiet off and trace on') quiet = True tracing = True - print '- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30) + print('- quiet:%s - trace:%s %s' % (str(quiet), str(tracing), '-' * 30)) trace('trace with quiet on and trace on') notice('notice with quiet on and trace on') default = l - print '=-' * 40 - print 'tail: %d' % (len(l.tail)) - print l - print '=-' * 40 + print('=-' * 40) + print('tail: %d' % (len(l.tail))) + print(l) + print('=-' * 40) del l diff --git a/source-builder/sb/macros.py b/source-builder/sb/macros.py index d2bdc91..2af8d36 100644 --- a/source-builder/sb/macros.py +++ b/source-builder/sb/macros.py @@ -21,6 +21,8 @@ # Macro tables. # +from __future__ import print_function + import re import os import string @@ -41,7 +43,7 @@ class macros: def __iter__(self): return self - def next(self): + def __next__(self): if self.index < len(self.keys): key = self.keys[self.index] self.index += 1 @@ -51,6 +53,19 @@ class macros: def iterkeys(self): return self.keys + def _unicode_to_str(self, us): + try: + if type(us) == unicode: + return us.encode('ascii', 'replace') + except: + pass + try: + if type(us) == bytes: + return us.encode('ascii', 'replace') + except: + pass + return us + def __init__(self, name = None, original = None, sbdir = '.'): self.files = [] self.macro_filter = re.compile(r'%{[^}]+}') @@ -124,7 +139,7 @@ class macros: return text def __iter__(self): - return macros.macro_iterator(self.keys()) + return macros.macro_iterator(list(self.keys())) def __getitem__(self, key): macro = self.get(key) @@ -133,14 +148,20 @@ class macros: return macro[2] def __setitem__(self, key, value): + key = self._unicode_to_str(key) if type(key) is not str: raise TypeError('bad key type (want str): %s' % (type(key))) + if type(value) is not tuple: + value = self._unicode_to_str(value) if type(value) is str: value = ('none', 'none', value) if type(value) is not tuple: raise TypeError('bad value type (want tuple): %s' % (type(value))) if len(value) != 3: raise TypeError('bad value tuple (len not 3): %d' % (len(value))) + value = (self._unicode_to_str(value[0]), + self._unicode_to_str(value[1]), + self._unicode_to_str(value[2])) if type(value[0]) is not str: raise TypeError('bad value tuple type field: %s' % (type(value[0]))) if type(value[1]) is not str: @@ -163,11 +184,11 @@ class macros: return self.has_key(key) def __len__(self): - return len(self.keys()) + return len(list(self.keys())) def keys(self, globals = True): if globals: - keys = self.macros['global'].keys() + keys = list(self.macros['global'].keys()) else: keys = [] for rm in self.get_read_maps(): @@ -182,7 +203,7 @@ class macros: def has_key(self, key): if type(key) is not str: raise TypeError('bad key type (want str): %s' % (type(key))) - if self.key_filter(key) not in self.keys(): + if self.key_filter(key) not in list(self.keys()): return False return True @@ -195,7 +216,7 @@ class macros: self.macros.pop(_map, None) def maps(self): - return self.macros.keys() + return list(self.macros.keys()) def map_keys(self, _map): if _map in self.macros: @@ -226,7 +247,7 @@ class macros: trace_me = False if trace_me: - print '[[[[]]]] parsing macros' + print('[[[[]]]] parsing macros') macros = { 'global': {} } map = 'global' lc = 0 @@ -238,11 +259,12 @@ class macros: #print 'l:%s' % (l[:-1]) if len(l) == 0: continue + l = self._unicode_to_str(l) l_remaining = l for c in l: if trace_me: - print ']]]]]]]] c:%s(%d) s:%s t:"%s" m:%r M:%s' % \ - (c, ord(c), state, token, macro, map) + print(']]]]]]]] c:%s(%d) s:%s t:"%s" m:%r M:%s' % \ + (c, ord(c), state, token, macro, map)) l_remaining = l_remaining[1:] if c is '#' and not state.startswith('value'): break @@ -345,7 +367,10 @@ class macros: else: raise error.internal('bad state: %s' % (state)) if state is 'macro': - macros[map][macro[0].lower()] = (macro[1], macro[2], macro[3]) + macros[map][self._unicode_to_str(macro[0].lower())] = \ + (self._unicode_to_str(macro[1]), + self._unicode_to_str(macro[2]), + self._unicode_to_str(macro[3])) macro = [] token = '' state = 'key' @@ -365,7 +390,7 @@ class macros: mc.close() self.files += [n] return - except IOError, err: + except IOError as err: pass raise error.general('opening macro file: %s' % \ (path.host(self.expand(name)))) @@ -481,22 +506,22 @@ class macros: if __name__ == "__main__": import copy import sys - m = macros(name = 'defaults.mc') + m = macros() d = copy.copy(m) m['test1'] = 'something' - if d.has_key('test1'): - print 'error: copy failed.' + if 'test1' in d: + print('error: copy failed.') sys.exit(1) m.parse("[test]\n" \ "test1: none, undefine, ''\n" \ "name: none, override, 'pink'\n") - print 'set test:', m.set_read_map('test') + print('set test:', m.set_read_map('test')) if m['name'] != 'pink': - print 'error: override failed. name is %s' % (m['name']) + print('error: override failed. name is %s' % (m['name'])) sys.exit(1) - if m.has_key('test1'): - print 'error: map undefine failed.' + if 'test1' in m: + print('error: map undefine failed.') sys.exit(1) - print 'unset test:', m.unset_read_map('test') - print m - print m.keys() + print('unset test:', m.unset_read_map('test')) + print(m) + print(list(m.keys())) diff --git a/source-builder/sb/mailer.py b/source-builder/sb/mailer.py index b6f686c..b9ef812 100644 --- a/source-builder/sb/mailer.py +++ b/source-builder/sb/mailer.py @@ -21,6 +21,8 @@ # Manage emailing results or reports. # +from __future__ import print_function + import os import smtplib import socket @@ -64,7 +66,7 @@ class mail: mrc = open(mailrc, 'r') lines = mrc.readlines() mrc.close() - except IOError, err: + except IOError as err: raise error.general('error reading: %s' % (mailrc)) for l in lines: l = _clean(l) @@ -93,9 +95,9 @@ class mail: try: s = smtplib.SMTP(self.smtp_host()) s.sendmail(from_addr, [to_addr], msg) - except smtplib.SMTPException, se: + except smtplib.SMTPException as se: raise error.general('sending mail: %s' % (str(se))) - except socket.error, se: + except socket.error as se: raise error.general('sending mail: %s' % (str(se))) if __name__ == '__main__': @@ -104,6 +106,6 @@ if __name__ == '__main__': append_options(optargs) opts = options.load(sys.argv, optargs = optargs, defaults = 'defaults.mc') m = mail(opts) - print 'From: %s' % (m.from_address()) - print 'SMTP Host: %s' % (m.smtp_host()) + print('From: %s' % (m.from_address())) + print('SMTP Host: %s' % (m.smtp_host())) m.send(m.from_address(), 'Test mailer.py', 'This is a test') diff --git a/source-builder/sb/options.py b/source-builder/sb/options.py index ecfc5b8..39e5ff4 100644 --- a/source-builder/sb/options.py +++ b/source-builder/sb/options.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2013 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -21,6 +21,8 @@ # Determine the defaults and load the specific file. # +from __future__ import print_function + import datetime import glob import pprint @@ -41,9 +43,10 @@ import version basepath = 'sb' # -# Save the host state. +# Save the host and POSIX state. # host_windows = False +host_posix = True class command_line: """Process the command line in a common way for all Tool Builder commands.""" @@ -96,7 +99,7 @@ class command_line: def __str__(self): def _dict(dd): s = '' - ddl = dd.keys() + ddl = list(dd.keys()) ddl.sort() for d in ddl: s += ' ' + d + ': ' + str(dd[d]) + '\n' @@ -189,44 +192,44 @@ class command_line: self.help() def help(self): - print '%s: [options] [args]' % (self.command_name) - print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2015 Chris Johns' - print 'Options and arguments:' - print '--force : Force the build to proceed' - print '--quiet : Quiet output (not used)' - print '--trace : Trace the execution' - print '--dry-run : Do everything but actually run the build' - print '--warn-all : Generate warnings' - print '--no-clean : Do not clean up the build tree' - print '--always-clean : Always clean the build tree, even with an error' - print '--keep-going : Do not stop on an error.' - print '--regression : Set --no-install, --keep-going and --always-clean' - print '--jobs : Run with specified number of jobs, default: num CPUs.' - print '--host : Set the host triplet' - print '--build : Set the build triplet' - print '--target : Set the target triplet' - print '--prefix path : Tools build prefix, ie where they are installed' - print '--topdir path : Top of the build tree, default is $PWD' - print '--configdir path : Path to the configuration directory, default: ./config' - print '--builddir path : Path to the build directory, default: ./build' - print '--sourcedir path : Path to the source directory, default: ./source' - print '--tmppath path : Path to the temp directory, default: ./tmp' - print '--macros file[,[file] : Macro format files to load after the defaults' - print '--log file : Log file where all build out is written too' - print '--url url[,url] : URL to look for source' - print '--no-download : Disable the source downloader' - print '--no-install : Do not install the packages to the prefix' - print '--targetcflags flags : List of C flags for the target code' - print '--targetcxxflags flags : List of C++ flags for the target code' - print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code' - print '--with-