diff options
author | Chris Johns <chrisj@rtems.org> | 2016-03-17 16:47:31 +1100 |
---|---|---|
committer | Chris Johns <chrisj@rtems.org> | 2016-03-17 16:47:31 +1100 |
commit | 31f33047a278b4ade2d0fb4964ebac4e7209b105 (patch) | |
tree | 637500eaf8b663541c867ea2d1faba014b3449af | |
parent | sb: Support --dry-run --with-download for 3rd party RTEMS BSP packages. (diff) | |
download | rtems-source-builder-31f33047a278b4ade2d0fb4964ebac4e7209b105.tar.bz2 |
sb: Change urlib to urllib2 on Python2 and add user agent support.
Some sites block the standard python user agent. Change to Wget
as it seems to make the tested sites behave.
Limit the size of long URLs when logging to the user.
Closes #2656.
-rw-r--r-- | source-builder/sb/download.py | 35 |
1 files changed, 24 insertions, 11 deletions
diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index bd6e6a6..0d62e34 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -32,7 +32,7 @@ try: import urllib.request as urllib_request import urllib.parse as urllib_parse except ImportError: - import urllib as urllib_request + import urllib2 as urllib_request import urlparse as urllib_parse import cvs @@ -69,6 +69,13 @@ def _humanize_bytes(bytes, precision = 1): break return '%.*f%s' % (precision, float(bytes) / factor, suffix) +def _sensible_url(url, used = 0): + space = 150 - used - 15 + if len(url) > space: + size = (space - 5) / 2 + url = url[:size] + ' ... ' + url[-size:] + return url + def _hash_check(file_, absfile, macros, remove = True): failed = False hash = sources.get_hash(file_.lower(), macros) @@ -325,7 +332,8 @@ def _http_downloader(url, local, config, opts): if url.startswith('https://api.github.com'): url = urllib_parse.urljoin(url, config.expand('tarball/%{version}')) dst = os.path.relpath(path.host(local)) - log.notice('download: %s -> %s' % (url, dst)) + log.output('download: %s -> %s' % (url, dst)) + log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst)) failed = False if _do_download(opts): _in = None @@ -337,24 +345,29 @@ def _http_downloader(url, local, config, opts): _last_percent = 200.0 _last_msg = '' _have_status_output = False + _url = url try: try: _in = None _ssl_context = None - _urllib_url = url + # See #2656 + _req = urllib_request.Request(_url) + _req.add_header('User-Agent', 'Wget/1.16.3 (freebsd10.1)') try: import ssl _ssl_context = ssl._create_unverified_context() - _in = urllib_request.urlopen(_urllib_url, context = _ssl_context) + _in = urllib_request.urlopen(_req, context = _ssl_context) except: _ssl_context = None if _ssl_context is None: - _in = urllib_request.urlopen(_urllib_url) - if url != _in.geturl(): - log.notice(' redirect: %s' % (_in.geturl())) + _in = urllib_request.urlopen(_req) + if _url != _in.geturl(): + _url = _in.geturl() + log.output(' redirect: %s' % (_url)) + log.notice(' redirect: %s' % (_sensible_url(_url))) _out = open(path.host(local), 'wb') try: - _length = int(_in.info().getheader('Content-Length').strip()) + _length = int(_in.info()['Content-Length'].strip()) except: pass while True: @@ -379,17 +392,17 @@ def _http_downloader(url, local, config, opts): log.stdout_raw('\n\r') raise except IOError as err: - log.notice('download: %s: error: %s' % (url, str(err))) + log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err))) if path.exists(local): os.remove(path.host(local)) failed = True except ValueError as err: - log.notice('download: %s: error: %s' % (url, str(err))) + log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err))) if path.exists(local): os.remove(path.host(local)) failed = True except: - msg = 'download: %s: error' % (url) + msg = 'download: %s: error' % (_sensible_url(_url)) log.stderr(msg) log.notice(msg) if _in is not None: |