From 61d2c55de6f8f7acd3bcc7eb1f0b9c13a87bf0a4 Mon Sep 17 00:00:00 2001
From: Chris Johns <chrisj@rtems.org>
Date: Thu, 1 Feb 2018 14:48:31 +1100
Subject: sb: Backport from 4.11 the --rsb-file with releases fixes.

Close #3286
---
 source-builder/sb/download.py | 16 ++++++--
 source-builder/sb/options.py  | 89 ++++++++++++++++++++++++++-----------------
 source-builder/sb/version.py  | 24 +++++++++---
 3 files changed, 86 insertions(+), 43 deletions(-)

(limited to 'source-builder/sb')

diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py
index 32d801b..6c2593f 100644
--- a/source-builder/sb/download.py
+++ b/source-builder/sb/download.py
@@ -26,6 +26,7 @@ from __future__ import print_function
 
 import hashlib
 import os
+import re
 import stat
 import sys
 try:
@@ -301,6 +302,11 @@ parsers = { 'http': _http_parser,
             'cvs':  _cvs_parser,
             'file': _file_parser }
 
+def set_release_path(release_path, macros):
+    if release_path is None:
+        release_path = '%{rtems_release_url}/%{rsb_version}/sources'
+    macros.define('release_path', release_path)
+
 def parse_url(url, pathkey, config, opts, file_override = None):
     #
     # Split the source up into the parts we need.
@@ -318,7 +324,6 @@ def parse_url(url, pathkey, config, opts, file_override = None):
         bad_chars = [c for c in ['/', '\\', '?', '*'] if c in file_override]
         if len(bad_chars) > 0:
             raise error.general('bad characters in file name: %s' % (file_override))
-
         log.output('download: file-override: %s' % (file_override))
         source['file'] = file_override
         source['options'] += ['file-override']
@@ -591,10 +596,9 @@ def get_file(url, local, opts, config):
     #
     url_bases = opts.urls()
     try:
-        rtems_release_url_value = config.macros.expand('%{rtems_release_url}/%{rsb_version}/sources')
+        rtems_release_url_value = config.macros.expand('%{release_path}')
     except:
         rtems_release_url_value = None
-        log.output('RTEMS release URL could not be expanded')
     rtems_release_url = None
     if version.released() and rtems_release_url_value:
         rtems_release_url = rtems_release_url_value
@@ -637,6 +641,12 @@ def get_file(url, local, opts, config):
             url_file = url_path[slash + 1:]
         log.trace('url_file: %s' %(url_file))
         for base in url_bases:
+            #
+            # Hack to fix #3064 where --rsb-file is being used. This code is a
+            # mess and should be refactored.
+            #
+            if version.released() and base == rtems_release_url:
+                url_file = path.basename(local)
             if base[-1:] != '/':
                 base += '/'
             next_url = urllib_parse.urljoin(base, url_file)
diff --git a/source-builder/sb/options.py b/source-builder/sb/options.py
index 7791329..7bbdd8c 100644
--- a/source-builder/sb/options.py
+++ b/source-builder/sb/options.py
@@ -54,34 +54,35 @@ class command_line:
     def __init__(self, argv, optargs, _defaults, command_path):
         self._long_opts = {
             # key                 macro                handler            param  defs   init
-            '--prefix'         : ('_prefix',           self._lo_path,     True,  None,  False),
-            '--topdir'         : ('_topdir',           self._lo_path,     True,  None,  False),
-            '--configdir'      : ('_configdir',        self._lo_path,     True,  None,  False),
-            '--builddir'       : ('_builddir',         self._lo_path,     True,  None,  False),
-            '--sourcedir'      : ('_sourcedir',        self._lo_path,     True,  None,  False),
-            '--tmppath'        : ('_tmppath',          self._lo_path,     True,  None,  False),
-            '--jobs'           : ('_jobs',             self._lo_jobs,     True,  'max', True),
-            '--log'            : ('_logfile',          self._lo_string,   True,  None,  False),
-            '--url'            : ('_url_base',         self._lo_string,   True,  None,  False),
-            '--no-download'    : ('_disable_download', self._lo_bool,     False, '0',   True),
-            '--macros'         : ('_macros',           self._lo_string,   True,  None,  False),
-            '--targetcflags'   : ('_targetcflags',     self._lo_string,   True,  None,  False),
-            '--targetcxxflags' : ('_targetcxxflags',   self._lo_string,   True,  None,  False),
-            '--libstdcxxflags' : ('_libstdcxxflags',   self._lo_string,   True,  None,  False),
-            '--force'          : ('_force',            self._lo_bool,     False, '0',   True),
-            '--quiet'          : ('_quiet',            self._lo_bool,     False, '0',   True),
-            '--trace'          : ('_trace',            self._lo_bool,     False, '0',   True),
-            '--dry-run'        : ('_dry_run',          self._lo_bool,     False, '0',   True),
-            '--warn-all'       : ('_warn_all',         self._lo_bool,     False, '0',   True),
-            '--no-clean'       : ('_no_clean',         self._lo_bool,     False, '0',   True),
-            '--keep-going'     : ('_keep_going',       self._lo_bool,     False, '0',   True),
-            '--always-clean'   : ('_always_clean',     self._lo_bool,     False, '0',   True),
-            '--no-install'     : ('_no_install',       self._lo_bool,     False, '0',   True),
-            '--regression'     : ('_regression',       self._lo_bool,     False, '0',   True),
-            '--host'           : ('_host',             self._lo_triplets, True,  None,  False),
-            '--build'          : ('_build',            self._lo_triplets, True,  None,  False),
-            '--target'         : ('_target',           self._lo_triplets, True,  None,  False),
-            '--help'           : (None,                self._lo_help,     False, None,  False)
+            '--prefix'               : ('_prefix',           self._lo_path,     True,  None,  False),
+            '--topdir'               : ('_topdir',           self._lo_path,     True,  None,  False),
+            '--configdir'            : ('_configdir',        self._lo_path,     True,  None,  False),
+            '--builddir'             : ('_builddir',         self._lo_path,     True,  None,  False),
+            '--sourcedir'            : ('_sourcedir',        self._lo_path,     True,  None,  False),
+            '--tmppath'              : ('_tmppath',          self._lo_path,     True,  None,  False),
+            '--jobs'                 : ('_jobs',             self._lo_jobs,     True,  'max', True),
+            '--log'                  : ('_logfile',          self._lo_string,   True,  None,  False),
+            '--url'                  : ('_url_base',         self._lo_string,   True,  None,  False),
+            '--no-download'          : ('_disable_download', self._lo_bool,     False, '0',   True),
+            '--macros'               : ('_macros',           self._lo_string,   True,  None,  False),
+            '--source-only-download' : ('_source_download',  self._lo_bool,     False, '0',   True),
+            '--targetcflags'         : ('_targetcflags',     self._lo_string,   True,  None,  False),
+            '--targetcxxflags'       : ('_targetcxxflags',   self._lo_string,   True,  None,  False),
+            '--libstdcxxflags'       : ('_libstdcxxflags',   self._lo_string,   True,  None,  False),
+            '--force'                : ('_force',            self._lo_bool,     False, '0',   True),
+            '--quiet'                : ('_quiet',            self._lo_bool,     False, '0',   True),
+            '--trace'                : ('_trace',            self._lo_bool,     False, '0',   True),
+            '--dry-run'              : ('_dry_run',          self._lo_bool,     False, '0',   True),
+            '--warn-all'             : ('_warn_all',         self._lo_bool,     False, '0',   True),
+            '--no-clean'             : ('_no_clean',         self._lo_bool,     False, '0',   True),
+            '--keep-going'           : ('_keep_going',       self._lo_bool,     False, '0',   True),
+            '--always-clean'         : ('_always_clean',     self._lo_bool,     False, '0',   True),
+            '--no-install'           : ('_no_install',       self._lo_bool,     False, '0',   True),
+            '--regression'           : ('_regression',       self._lo_bool,     False, '0',   True),
+            '--host'                 : ('_host',             self._lo_triplets, True,  None,  False),
+            '--build'                : ('_build',            self._lo_triplets, True,  None,  False),
+            '--target'               : ('_target',           self._lo_triplets, True,  None,  False),
+            '--help'                 : (None,                self._lo_help,     False, None,  False)
             }
 
         self.command_path = command_path
@@ -222,6 +223,7 @@ class command_line:
         print('--targetcflags flags   : List of C flags for the target code')
         print('--targetcxxflags flags : List of C++ flags for the target code')
         print('--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code')
+        print('--source-only-download : Only download the source')
         print('--with-<label>         : Add the --with-<label> to the build')
         print('--without-<label>      : Add the --without-<label> to the build')
         print('--rtems-tools path     : Path to an install RTEMS tool set')
@@ -265,9 +267,27 @@ class command_line:
                 self.opts['params'].append(a)
             arg += 1
 
-    def post_process(self):
+    def pre_process(self):
+        arg = 0
+        while arg < len(self.args):
+            a = self.args[arg]
+            if a == '--source-only-download':
+                self.args += ['--dry-run',
+                              '--quiet',
+                              '--without-log',
+                              '--without-error-report',
+                              '--without-release-url']
+            arg += 1
+
+    def post_process(self, logfile = True):
         # Handle the log first.
-        log.default = log.log(self.logfiles())
+        logctrl = self.parse_args('--without-log')
+        if logctrl is None:
+            if logfile:
+                logfiles = self.logfiles()
+            else:
+                logfiles = None
+            log.default = log.log(streams = logfiles)
         if self.trace():
             log.tracing = True
         if self.quiet():
@@ -561,7 +581,7 @@ class command_line:
             self.args.append('--target=%s-rtems%s' % (ab[0], rtems_version))
             self.args.append('--with-rtems-bsp=%s' % (ab[1]))
 
-def load(args, optargs = None, defaults = '%{_sbdir}/defaults.mc'):
+def load(args, optargs = None, defaults = '%{_sbdir}/defaults.mc', logfile = True):
     """
     Copy the defaults, get the host specific values and merge them overriding
     any matching defaults, then create an options object to handle the command
@@ -637,13 +657,14 @@ def load(args, optargs = None, defaults = '%{_sbdir}/defaults.mc'):
     o.sb_released()
     o.sb_git()
     o.rtems_options()
+    o.pre_process()
     o.process()
-    o.post_process()
+    o.post_process(logfile)
 
     #
-    # Load the release hashes
+    # Load the release settings
     #
-    version.load_release_hashes(o.defaults)
+    version.load_release_settings(o.defaults)
 
     return o
 
diff --git a/source-builder/sb/version.py b/source-builder/sb/version.py
index e1078bb..b193a49 100644
--- a/source-builder/sb/version.py
+++ b/source-builder/sb/version.py
@@ -26,6 +26,7 @@ from __future__ import print_function
 
 import sys
 
+import download
 import error
 import git
 import path
@@ -34,7 +35,7 @@ import sources
 #
 # Default to an internal string.
 #
-_version = '4.12'
+_version = '4.10'
 _revision = 'not_released'
 _version_str = '%s.%s' % (_version, _revision)
 _released = False
@@ -55,7 +56,10 @@ def _load_released_version_config():
             except ImportError:
                 import ConfigParser as configparser
             v = configparser.SafeConfigParser()
-            v.read(path.join(ver, 'VERSION'))
+            try:
+                v.read(path.join(ver, 'VERSION'))
+            except:
+                raise error.general('Invalid VERSION file')
             return v
     return None
 
@@ -64,7 +68,10 @@ def _load_released_version():
     global _version_str
     v = _load_released_version_config()
     if v is not None:
-        _version_str = v.get('version', 'release')
+        try:
+            _version_str = v.get('version', 'release')
+        except:
+            raise error.general('Invalid VERSION file')
         _released = True
     return _released
 
@@ -94,8 +101,8 @@ def str():
             _load_git_version()
     return _version_str
 
-def load_release_hashes(macros):
-    def hash_error(msg):
+def load_release_settings(macros):
+    def setting_error(msg):
         raise error.general(msg)
 
     if released():
@@ -105,11 +112,16 @@ def load_release_hashes(macros):
                 hashes = v.items('hashes')
             except:
                 hashes = []
+            try:
+                release_path = v.get('version', 'release_path', raw = True)
+            except:
+                release_path = None
             for hash in hashes:
                 hs = hash[1].split()
                 if len(hs) != 2:
                     raise error.general('invalid release hash in VERSION')
-                sources.hash((hs[0], hash[0], hs[1]), macros, hash_error)
+                sources.hash((hs[0], hash[0], hs[1]), macros, setting_error)
+            download.set_release_path(release_path, macros)
 
 def version():
     return _version
-- 
cgit v1.2.3