From d7550652cb2b23b7105e2dabe1e13fb1b18d5215 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Wed, 18 Feb 2015 10:15:31 +1100 Subject: Add options support for --with/--without processing. Add --with-download support. You can now test downloading by using --dry-run and --with-download. Print the redirect if one is taken. --- source-builder/sb/download.py | 131 ++++++++++++++++++++++++++++++++---------- source-builder/sb/options.py | 49 ++++++++++++++-- 2 files changed, 144 insertions(+), 36 deletions(-) diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index 9490aa0..17d1d7e 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -36,6 +36,16 @@ import log import path import sources +def _do_download(opts): + download = True + if opts.dry_run(): + download = False + wa = opts.with_arg('download') + if wa is not None: + if wa[0] == 'with_download' and wa[1] == 'yes': + download = True + return download + def _humanize_bytes(bytes, precision = 1): abbrevs = ( (1 << 50L, 'PB'), @@ -102,7 +112,56 @@ def _hash_check(file_, absfile, macros, remove = True): log.warning('%s: no hash found' % (file_)) return not failed -def _http_parser(source, config, opts): +def _local_path(source, pathkey, config): + for p in config.define(pathkey).split(':'): + local = path.join(path.abspath(p), source['file']) + if source['local'] is None: + source['local_prefix'] = path.abspath(p) + source['local'] = local + if path.exists(local): + source['local_prefix'] = path.abspath(p) + source['local'] = local + _hash_check(source['file'], local, config.macros) + break + +def _http_parser(source, pathkey, config, opts): + # + # Hack for gitweb.cgi patch downloads. We rewrite the various fields. + # + if 'gitweb.cgi' in source['url']: + url = source['url'] + if '?' not in url: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + req = url.split('?')[1] + if len(req) == 0: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + # + # The gitweb.cgi request should have: + # p= + # a=patch + # h= + # so extract the p and h parts to make the local name. + # + p = None + a = None + h = None + for r in req.split(';'): + if '=' not in r: + raise error.general('invalid gitweb.cgi path: %s' % (url)) + rs = r.split('=') + if rs[0] == 'p': + p = rs[1].replace('.', '-') + elif rs[0] == 'a': + a = rs[1] + elif rs[0] == 'h': + h = rs[1] + if p is None or h is None: + raise error.general('gitweb.cgi path missing p or h: %s' % (url)) + source['file'] = '%s-%s.patch' % (p, h) + # + # Check local path + # + _local_path(source, pathkey, config) # # Is the file compressed ? # @@ -120,10 +179,18 @@ def _http_parser(source, config, opts): source['compressed-type'] = 'xz' source['compressed'] = '%{__xz} -dc' -def _patchworks_parser(source, config, opts): +def _patchworks_parser(source, pathkey, config, opts): + # + # Check local path + # + _local_path(source, pathkey, config) source['url'] = 'http%s' % (source['path'][2:]) -def _git_parser(source, config, opts): +def _git_parser(source, pathkey, config, opts): + # + # Check local path + # + _local_path(source, pathkey, config) # # Symlink. # @@ -137,7 +204,11 @@ def _git_parser(source, config, opts): path.join(source['local_prefix'], 'git', source['file']) source['symlink'] = source['local'] -def _cvs_parser(source, config, opts): +def _cvs_parser(source, pathkey, config, opts): + # + # Check local path + # + _local_path(source, pathkey, config) # # Symlink. # @@ -185,7 +256,11 @@ def _cvs_parser(source, config, opts): else: source['symlink'] = source['local'] -def _file_parser(source, config, opts): +def _file_parser(source, pathkey, config, opts): + # + # Check local path + # + _local_path(source, pathkey, config) # # Symlink. # @@ -217,22 +292,12 @@ def parse_url(url, pathkey, config, opts): # Get the file. Checks the local source directory first. # source['local'] = None - for p in config.define(pathkey).split(':'): - local = path.join(path.abspath(p), source['file']) - if source['local'] is None: - source['local_prefix'] = path.abspath(p) - source['local'] = local - if path.exists(local): - source['local_prefix'] = path.abspath(p) - source['local'] = local - _hash_check(source['file'], local, config.macros) - break - source['script'] = '' for p in parsers: if url.startswith(p): source['type'] = p - if parsers[p](source, config, opts): + if parsers[p](source, pathkey, config, opts): break + source['script'] = '' return source def _http_downloader(url, local, config, opts): @@ -246,7 +311,7 @@ def _http_downloader(url, local, config, opts): dst = os.path.relpath(path.host(local)) log.notice('download: %s -> %s' % (url, dst)) failed = False - if not opts.dry_run(): + if _do_download(opts): _in = None _out = None _length = None @@ -258,7 +323,10 @@ def _http_downloader(url, local, config, opts): _wipe_output = False try: try: + _in = None _in = urllib2.urlopen(url) + if url != _in.geturl(): + log.notice(' redirect: %s' % (_in.geturl())) _out = open(path.host(local), 'wb') try: _length = int(_in.info().getheader('Content-Length').strip()) @@ -300,12 +368,15 @@ def _http_downloader(url, local, config, opts): msg = 'download: %s: error' % (url) log.stderr(msg) log.notice(msg) + if _in is not None: + _in.close() if _out is not None: _out.close() raise if _out is not None: _out.close() if _in is not None: + _in.close() del _in if not failed: if not path.isfile(local): @@ -338,7 +409,7 @@ def _git_downloader(url, local, config, opts): us[0] = _as[1] + url_base if not repo.valid(): log.notice('git: clone: %s -> %s' % (us[0], rlp)) - if not opts.dry_run(): + if _do_download(opts): repo.clone(us[0], local) else: repo.clean(['-f', '-d']) @@ -350,28 +421,28 @@ def _git_downloader(url, local, config, opts): if len(_as) != 2: raise error.general('invalid git branch/checkout: %s' % (_as)) log.notice('git: checkout: %s => %s' % (us[0], _as[1])) - if not opts.dry_run(): + if _do_download(opts): repo.checkout(_as[1]) elif _as[0] == 'pull': log.notice('git: pull: %s' % (us[0])) - if not opts.dry_run(): + if _do_download(opts): repo.pull() elif _as[0] == 'submodule': if len(_as) != 2: raise error.general('invalid git submodule: %s' % (_as)) log.notice('git: submodule: %s <= %s' % (us[0], _as[1])) - if not opts.dry_run(): + if _do_download(opts): repo.submodule(_as[1]) elif _as[0] == 'fetch': log.notice('git: fetch: %s -> %s' % (us[0], rlp)) - if not opts.dry_run(): + if _do_download(opts): repo.fetch() elif _as[0] == 'reset': arg = [] if len(_as) > 1: arg = ['--%s' % (_as[1])] log.notice('git: reset: %s' % (us[0])) - if not opts.dry_run(): + if _do_download(opts): repo.reset(arg) elif _as[0] == 'protocol': pass @@ -409,20 +480,20 @@ def _cvs_downloader(url, local, config, opts): if not path.isdir(local): log.notice('Creating source directory: %s' % \ (os.path.relpath(path.host(local)))) - if not opts.dry_run(): + if _do_download(opts): path.mkdir(local) log.notice('cvs: checkout: %s -> %s' % (us[0], rlp)) - if not opts.dry_run(): + if _do_download(opts): repo.checkout(':%s' % (us[0][6:]), module, tag, date) for a in us[1:]: _as = a.split('=') if _as[0] == 'update': log.notice('cvs: update: %s' % (us[0])) - if not opts.dry_run(): + if _do_download(opts): repo.update() elif _as[0] == 'reset': log.notice('cvs: reset: %s' % (us[0])) - if not opts.dry_run(): + if _do_download(opts): repo.reset() return True @@ -445,7 +516,7 @@ def get_file(url, local, opts, config): log.notice('Creating source directory: %s' % \ (os.path.relpath(path.host(path.dirname(local))))) log.output('making dir: %s' % (path.host(path.dirname(local)))) - if not opts.dry_run(): + if _do_download(opts): path.mkdir(path.dirname(local)) if not path.exists(local) and opts.download_disabled(): raise error.general('source not found: %s' % (path.host(local))) @@ -472,5 +543,5 @@ def get_file(url, local, opts, config): if url.startswith(dl): if downloaders[dl](url, local, config, opts): return - if not opts.dry_run(): + if _do_download(opts): raise error.general('downloading %s: all paths have failed, giving up' % (url)) diff --git a/source-builder/sb/options.py b/source-builder/sb/options.py index a468694..663c7f0 100644 --- a/source-builder/sb/options.py +++ b/source-builder/sb/options.py @@ -190,7 +190,7 @@ class command_line: def help(self): print '%s: [options] [args]' % (self.command_name) - print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns' + print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2015 Chris Johns' print 'Options and arguments:' print '--force : Force the build to proceed' print '--quiet : Quiet output (not used)' @@ -427,14 +427,49 @@ class command_line: def params(self): return self.opts['params'] + def parse_args(self, arg, error = True): + for a in range(0, len(self.args)): + if self.args[a].startswith(arg): + lhs = None + rhs = None + if '=' in self.args[a]: + eqs = self.args[a].split('=') + lhs = eqs[0] + if len(eqs) > 2: + rhs = '='.join(eqs[1:]) + else: + rhs = eqs[1] + else: + lhs = self.args[a] + a += 1 + if a >= len(self.args): + return [arg, None] + rhs = self.args[a] + return [lhs, rhs] + a += 1 + return None + def get_arg(self, arg): if self.optargs is None or arg not in self.optargs: raise error.internal('bad arg: %s' % (arg)) - for a in self.args: - sa = a.split('=') - if sa[0].startswith(arg): - return sa - return None + return self.parse_args(arg) + + def with_arg(self, label): + for pre in ['with', 'without']: + arg_str = '--%s-%s' % (pre, label) + arg_label = '%s_%s' % (pre, label) + arg = self.parse_args(arg_str, error = False) + if arg is not None: + if arg[1] is None: + result = 'yes' + else: + result = arg[1] + break + if pre == 'with': + result = 'yes' + else: + result = 'no' + return [arg_label, result] def get_config_files(self, config): # @@ -570,6 +605,8 @@ def run(args): log.notice(str(_opts)) log.notice('Defaults:') log.notice(str(_opts.defaults)) + log.notice('with-opt1: %r' % (_opts.with_arg('opt1'))) + log.notice('without-opt2: %r' % (_opts.with_arg('opt2'))) except error.general, gerr: print gerr sys.exit(1) -- cgit v1.2.3