From f179dc6538ec11d3e95f88b940763783c9a4b2dc Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Thu, 8 Jun 2017 16:03:15 +1000 Subject: sb: Backport from master the '--rsb-file=' option. Upates #3033. --- source-builder/sb/build.py | 51 ++++++++++++++++++--- source-builder/sb/download.py | 101 ++++++++++++++++++++++++------------------ 2 files changed, 104 insertions(+), 48 deletions(-) diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index 15d569d..b995e6b 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -209,8 +209,33 @@ class build: if sm is None: raise error.internal('source macro not found: %s in %s (%s)' % \ (s, name, _map)) - url = self.config.expand(sm[2]) - src = download.parse_url(url, '_sourcedir', self.config, self.opts) + opts = [] + url = [] + for sp in sm[2].split(): + if len(url) == 0 and sp[0] == '-': + opts += [sp] + else: + url += [sp] + if len(url) == 0: + raise error.general('source URL not found: %s' % (' '.join(args))) + # + # Look for --rsb-file as an option we use as a local file name. + # This can be used if a URL has no reasonable file name the + # download URL parser can figure out. + # + file_override = None + if len(opts) > 0: + for o in opts: + if o.startswith('--rsb-file'): + os_ = o.split('=') + if len(os_) != 2: + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + if os_[0] != '--rsb-file': + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + file_override = os_[1] + opts = [o for o in opts if not o.startswith('--rsb-')] + url = self.config.expand(' '.join(url)) + src = download.parse_url(url, '_sourcedir', self.config, self.opts, file_override) download.get_file(src['url'], src['local'], self.opts, self.config) if 'symlink' in src: sname = name.replace('-', '_') @@ -303,6 +328,22 @@ class build: url += [pp] if len(url) == 0: raise error.general('patch URL not found: %s' % (' '.join(args))) + # + # Look for --rsb-file as an option we use as a local file name. + # This can be used if a URL has no reasonable file name the + # download URL parser can figure out. + # + file_override = None + if len(opts) > 0: + for o in opts: + if o.startswith('--rsb-file'): + os_ = o.split('=') + if len(os_) != 2: + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + if os_[0] != '--rsb-file': + raise error.general('invalid --rsb-file option: %s' % (' '.join(args))) + file_override = os_[1] + opts = [o for o in opts if not o.startswith('--rsb-')] if len(opts) == 0: opts = default_opts else: @@ -312,12 +353,10 @@ class build: # # Parse the URL first in the source builder's patch directory. # - patch = download.parse_url(url, '_patchdir', self.config, self.opts) + patch = download.parse_url(url, '_patchdir', self.config, self.opts, file_override) # - # If not in the source builder package check the source directory. + # Download the patch # - if not path.exists(patch['local']): - patch = download.parse_url(url, '_patchdir', self.config, self.opts) download.get_file(patch['url'], patch['local'], self.opts, self.config) if 'compressed' in patch: patch['script'] = patch['compressed'] + ' ' + patch['local'] diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index c9cb792..5744844 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -26,6 +26,7 @@ from __future__ import print_function import hashlib import os +import re import stat import sys try: @@ -130,57 +131,62 @@ def _hash_check(file_, absfile, macros, remove = True): def _local_path(source, pathkey, config): for p in config.define(pathkey).split(':'): - local = path.join(path.abspath(p), source['file']) + local_prefix = path.abspath(p) + local = path.join(local_prefix, source['file']) if source['local'] is None: - source['local_prefix'] = path.abspath(p) + source['local_prefix'] = local_prefix source['local'] = local if path.exists(local): - source['local_prefix'] = path.abspath(p) + source['local_prefix'] = local_prefix source['local'] = local _hash_check(source['file'], local, config.macros) break def _http_parser(source, pathkey, config, opts): # - # Hack for gitweb.cgi patch downloads. We rewrite the various fields. + # If the file has not been overrided attempt to recover a possible file name. # - if 'gitweb.cgi' in source['url']: - url = source['url'] - if '?' not in url: - raise error.general('invalid gitweb.cgi request: %s' % (url)) - req = url.split('?')[1] - if len(req) == 0: - raise error.general('invalid gitweb.cgi request: %s' % (url)) + if 'file-override' not in source['options']: # - # The gitweb.cgi request should have: - # p= - # a=patch - # h= - # so extract the p and h parts to make the local name. + # Hack for gitweb.cgi patch downloads. We rewrite the various fields. # - p = None - a = None - h = None - for r in req.split(';'): - if '=' not in r: - raise error.general('invalid gitweb.cgi path: %s' % (url)) - rs = r.split('=') - if rs[0] == 'p': - p = rs[1].replace('.', '-') - elif rs[0] == 'a': - a = rs[1] - elif rs[0] == 'h': - h = rs[1] - if p is None or h is None: - raise error.general('gitweb.cgi path missing p or h: %s' % (url)) - source['file'] = '%s-%s.patch' % (p, h) - # - # Check the source file name for any extra request query data and remove if - # found. Some hosts do not like file names containing them. - # - if '?' in source['file']: - qmark = source['file'].find('?') - source['file'] = source['file'][:qmark] + if 'gitweb.cgi' in source['url']: + url = source['url'] + if '?' not in url: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + req = url.split('?')[1] + if len(req) == 0: + raise error.general('invalid gitweb.cgi request: %s' % (url)) + # + # The gitweb.cgi request should have: + # p= + # a=patch + # h= + # so extract the p and h parts to make the local name. + # + p = None + a = None + h = None + for r in req.split(';'): + if '=' not in r: + raise error.general('invalid gitweb.cgi path: %s' % (url)) + rs = r.split('=') + if rs[0] == 'p': + p = rs[1].replace('.', '-') + elif rs[0] == 'a': + a = rs[1] + elif rs[0] == 'h': + h = rs[1] + if p is None or h is None: + raise error.general('gitweb.cgi path missing p or h: %s' % (url)) + source['file'] = '%s-%s.patch' % (p, h) + # + # Check the source file name for any extra request query data and remove if + # found. Some hosts do not like file names containing them. + # + if '?' in source['file']: + qmark = source['file'].find('?') + source['file'] = source['file'][:qmark] # # Check local path # @@ -301,17 +307,27 @@ def set_release_path(release_path, macros): release_path = '%{rtems_release_url}/%{rsb_version}/sources' macros.define('release_path', release_path) -def parse_url(url, pathkey, config, opts): +def parse_url(url, pathkey, config, opts, file_override = None): # # Split the source up into the parts we need. # source = {} source['url'] = url + source['options'] = [] colon = url.find(':') if url[colon + 1:colon + 3] != '//': raise error.general('malforned URL (no protocol prefix): %s' % (url)) source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:]) - source['file'] = path.basename(url) + if file_override is None: + source['file'] = path.basename(url) + else: + bad_chars = [c for c in ['/', '\\', '?', '*'] if c in file_override] + if len(bad_chars) > 0: + raise error.general('bad characters in file name: %s' % (file_override)) + + log.output('download: file-override: %s' % (file_override)) + source['file'] = file_override + source['options'] += ['file-override'] source['name'], source['ext'] = path.splitext(source['file']) if source['name'].endswith('.tar'): source['name'] = source['name'][:-4] @@ -337,7 +353,7 @@ def _http_downloader(url, local, config, opts): if url.startswith('https://api.github.com'): url = urllib_parse.urljoin(url, config.expand('tarball/%{version}')) dst = os.path.relpath(path.host(local)) - log.output('download: %s -> %s' % (url, dst)) + log.output('download: (full) %s -> %s' % (url, dst)) log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst)) failed = False if _do_download(opts): @@ -363,6 +379,7 @@ def _http_downloader(url, local, config, opts): _ssl_context = ssl._create_unverified_context() _in = urllib_request.urlopen(_req, context = _ssl_context) except: + log.output('download: no ssl context') _ssl_context = None if _ssl_context is None: _in = urllib_request.urlopen(_req) -- cgit v1.2.3