diff options
author | Chris Johns <chrisj@rtems.org> | 2016-04-15 11:44:27 +1000 |
---|---|---|
committer | Chris Johns <chrisj@rtems.org> | 2016-04-15 11:59:30 +1000 |
commit | 383f7e66906213b4f0cc5bd464789bfff731ffee (patch) | |
tree | c23bd9dfbe89627adab3004dbbb04b64c0ba5507 /source-builder | |
parent | sb: Do not filter -f options from a BSP. (diff) | |
download | rtems-source-builder-383f7e66906213b4f0cc5bd464789bfff731ffee.tar.bz2 |
sb: Add --rsb-file options to %source and %patch to set a file name.
Override the automatic file name of a downloaded file and use the file
name provided by the option. This is useful if the URL has no meanful
file that can be automatically extracted from the URL.
Diffstat (limited to 'source-builder')
-rw-r--r-- | source-builder/sb/build.py | 51 | ||||
-rw-r--r-- | source-builder/sb/download.py | 100 |
2 files changed, 103 insertions, 48 deletions
diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index 15d569d..b995e6b 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -209,8 +209,33 @@ class build: if sm is None: raise error.internal('source macro not found: %s in %s (%s)' % \ (s, name, _map)) - url = self.config.expand(sm[2]) - src = download.parse_url(url, '_sourcedir', self.config, self.opts) + opts = [] + url = [] + for sp in sm[2].split(): + if len(url) == 0 and sp[0] == '-': + opts += [sp] + else: + url += [sp] + if len(url) == 0: + raise error.general('source URL not found: %s' % (' '.join(args))) + # + # Look for --rsb-file as an option we use as a local file name. + # This can be used if a URL has no reasonable file name the + # download URL parser can figure out. + # + file_override = None + if len(opts) > 0: + for o in opts: + if o.startswith('--rsb-file'): + os_ = o.split('=') + if len(os_) != 2: + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + if os_[0] != '--rsb-file': + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + file_override = os_[1] + opts = [o for o in opts if not o.startswith('--rsb-')] + url = self.config.expand(' '.join(url)) + src = download.parse_url(url, '_sourcedir', self.config, self.opts, file_override) download.get_file(src['url'], src['local'], self.opts, self.config) if 'symlink' in src: sname = name.replace('-', '_') @@ -303,6 +328,22 @@ class build: url += [pp] if len(url) == 0: raise error.general('patch URL not found: %s' % (' '.join(args))) + # + # Look for --rsb-file as an option we use as a local file name. + # This can be used if a URL has no reasonable file name the + # download URL parser can figure out. + # + file_override = None + if len(opts) > 0: + for o in opts: + if o.startswith('--rsb-file'): + os_ = o.split('=') + if len(os_) != 2: + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + if os_[0] != '--rsb-file': + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + file_override = os_[1] + opts = [o for o in opts if not o.startswith('--rsb-')] if len(opts) == 0: opts = default_opts else: @@ -312,12 +353,10 @@ class build: # # Parse the URL first in the source builder's patch directory. # - patch = download.parse_url(url, '_patchdir', self.config, self.opts) + patch = download.parse_url(url, '_patchdir', self.config, self.opts, file_override) # - # If not in the source builder package check the source directory. + # Download the patch # - if not path.exists(patch['local']): - patch = download.parse_url(url, '_patchdir', self.config, self.opts) download.get_file(patch['url'], patch['local'], self.opts, self.config) if 'compressed' in patch: patch['script'] = patch['compressed'] + ' ' + patch['local'] diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index 1bd132e..32d801b 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -130,57 +130,62 @@ def _hash_check(file_, absfile, macros, remove = True): def _local_path(source, pathkey, config): for p in config.define(pathkey).split(':'): - local = path.join(path.abspath(p), source['file']) + local_prefix = path.abspath(p) + local = path.join(local_prefix, source['file']) if source['local'] is None: - source['local_prefix'] = path.abspath(p) + source['local_prefix'] = local_prefix source['local'] = local if path.exists(local): - source['local_prefix'] = path.abspath(p) + source['local_prefix'] = local_prefix source['local'] = local _hash_check(source['file'], local, config.macros) break def _http_parser(source, pathkey, config, opts): # - # Hack for gitweb.cgi patch downloads. We rewrite the various fields. + # If the file has not been overrided attempt to recover a possible file name. # - if 'gitweb.cgi' in source['url']: - url = source['url'] - if '?' not in url: - raise error.general('invalid gitweb.cgi request: %s' % (url)) - req = url.split('?')[1] - if len(req) == 0: - raise error.general('invalid gitweb.cgi request: %s' % (url)) + if 'file-override' not in source['options']: # - # The gitweb.cgi request should have: - # p=<what> - # a=patch - # h=<hash> - # so extract the p and h parts to make the local name. + # Hack for gitweb.cgi patch downloads. We rewrite the various fields. # - p = None - a = None - h = None - for r in req.split(';'): - if '=' not in r: - raise error.general('invalid gitweb.cgi path: %s' % (url)) - rs = r.split('=') - if rs[0] == 'p': - p = rs[1].replace('.', '-') - elif rs[0] == 'a': - a = rs[1] - elif rs[0] == 'h': - h = rs[1] - if p is None or h is None: - raise error.general('gitweb.cgi path missing p or h: %s' % (url)) - source['file'] = '%s-%s.patch' % (p, h) - # - # Check the source file name for any extra request query data and remove if - # found. Some hosts do not like file names containing them. - # - if '?' in source['file']: - qmark = source['file'].find('?') - source['file'] = source['file'][:qmark] + if 'gitweb.cgi' in source['url']: + url = source['url'] + if '?' not in url: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + req = url.split('?')[1] + if len(req) == 0: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + # + # The gitweb.cgi request should have: + # p=<what> + # a=patch + # h=<hash> + # so extract the p and h parts to make the local name. + # + p = None + a = None + h = None + for r in req.split(';'): + if '=' not in r: + raise error.general('invalid gitweb.cgi path: %s' % (url)) + rs = r.split('=') + if rs[0] == 'p': + p = rs[1].replace('.', '-') + elif rs[0] == 'a': + a = rs[1] + elif rs[0] == 'h': + h = rs[1] + if p is None or h is None: + raise error.general('gitweb.cgi path missing p or h: %s' % (url)) + source['file'] = '%s-%s.patch' % (p, h) + # + # Check the source file name for any extra request query data and remove if + # found. Some hosts do not like file names containing them. + # + if '?' in source['file']: + qmark = source['file'].find('?') + source['file'] = source['file'][:qmark] # # Check local path # @@ -296,17 +301,27 @@ parsers = { 'http': _http_parser, 'cvs': _cvs_parser, 'file': _file_parser } -def parse_url(url, pathkey, config, opts): +def parse_url(url, pathkey, config, opts, file_override = None): # # Split the source up into the parts we need. # source = {} source['url'] = url + source['options'] = [] colon = url.find(':') if url[colon + 1:colon + 3] != '//': raise error.general('malforned URL (no protocol prefix): %s' % (url)) source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:]) - source['file'] = path.basename(url) + if file_override is None: + source['file'] = path.basename(url) + else: + bad_chars = [c for c in ['/', '\\', '?', '*'] if c in file_override] + if len(bad_chars) > 0: + raise error.general('bad characters in file name: %s' % (file_override)) + + log.output('download: file-override: %s' % (file_override)) + source['file'] = file_override + source['options'] += ['file-override'] source['name'], source['ext'] = path.splitext(source['file']) if source['name'].endswith('.tar'): source['name'] = source['name'][:-4] @@ -332,7 +347,7 @@ def _http_downloader(url, local, config, opts): if url.startswith('https://api.github.com'): url = urllib_parse.urljoin(url, config.expand('tarball/%{version}')) dst = os.path.relpath(path.host(local)) - log.output('download: %s -> %s' % (url, dst)) + log.output('download: (full) %s -> %s' % (url, dst)) log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst)) failed = False if _do_download(opts): @@ -358,6 +373,7 @@ def _http_downloader(url, local, config, opts): _ssl_context = ssl._create_unverified_context() _in = urllib_request.urlopen(_req, context = _ssl_context) except: + log.output('download: no ssl context') _ssl_context = None if _ssl_context is None: _in = urllib_request.urlopen(_req) |