python: Provide support to select a valid python version.

- Update imports after wrapping the code.
- Fix python3 issues.
- Fix config path issues for in repo and install runs.

Closes #3537

5 files changed, 98 insertions, 25 deletions

diff --git a/rtemstoolkit/configuration.py b/rtemstoolkit/configuration.py
index 10d97e5..3b03296 100644
--- a/rtemstoolkit/configuration.py
+++ b/rtemstoolkit/configuration.py
@@ -37,18 +37,20 @@ from __future__ import print_function
 import os
 import re
 
-try:
-    import configparser
-except:
-    import ConfigParser as configparser
-
 from rtemstoolkit import error
 from rtemstoolkit import path
 
 class configuration:
 
-    def __init__(self):
-        self.config = configparser.ConfigParser()
+    def __init__(self, raw = True):
+        self.raw = True
+        try:
+            import configparser
+            self.config = configparser.ConfigParser(strict = False)
+        except:
+            # python2
+            import ConfigParser as configparser
+            self.config = configparser.ConfigParser()
         self.ini = None
         self.macro_filter = re.compile('\$\{.+\}')
 
@@ -66,12 +68,15 @@ class configuration:
         for section in self.config.sections():
             s += [' [%s]' % (section)]
             for option in self.config.options(section):
-                s += ['  %s = %s' % (option, self.config.get(section, option))]
+                s += ['  %s = %s' % (option,
+                                     self.config.get(section,
+                                                     option,
+                                                     raw = self.raw))]
         return os.linesep.join(s)
 
     def get_item(self, section, label, err = True):
         try:
-            rec = self.config.get(section, label).replace(os.linesep, ' ')
+            rec = self.config.get(section, label, raw = self.raw).replace(os.linesep, ' ')
         except:
             if err:
                 raise error.general('config: no "%s" found in "%s"' % (label, section))
@@ -89,7 +94,8 @@ class configuration:
                 raise error.general('config: interpolation is ${section:value}: %s' % (m))
             try:
                 ref = self.config.get(section_value[0],
-                                      section_value[1]).replace(os.linesep, ' ')
+                                      section_value[1],
+                                      raw = self.raw).replace(os.linesep, ' ')
                 rec = rec.replace(m, ref)
             except:
                 pass
@@ -98,7 +104,7 @@ class configuration:
     def get_items(self, section, err = True, flatten = True):
         try:
             items = []
-            for name, key in self.config.items(section):
+            for name, key in self.config.items(section, raw = self.raw):
                 if flatten:
                     items += [(name, key.replace(os.linesep, ' '))]
                 else:
@@ -117,7 +123,7 @@ class configuration:
 
     def get_item_names(self, section, err = True):
         try:
-            return [item[0] for item in self.config.items(section)]
+            return [item[0] for item in self.config.items(section, raw = self.raw)]
         except:
             if err:
                 raise error.general('config: section "%s" not found' % (section))
diff --git a/rtemstoolkit/log.py b/rtemstoolkit/log.py
index 3eb2c1b..76a2767 100755
--- a/rtemstoolkit/log.py
+++ b/rtemstoolkit/log.py
@@ -78,6 +78,8 @@ def _output(text = os.linesep, log = None):
         text = os.linesep
     if type(text) is list:
         text = os.linesep.join(text) + os.linesep
+    if isinstance(text, bytes):
+        text = text.decode('utf-8', 'ignore')
     if log:
         log.output(text)
     elif default is not None:
@@ -175,6 +177,8 @@ class log:
         text = text.replace(chr(13), '').splitlines()
         self._tail(text)
         out = os.linesep.join(text) + os.linesep
+        if isinstance(out, bytes):
+            out = out.decode('utf-8', 'ignore')
         self.lock.acquire()
         try:
             for f in range(0, len(self.fhs)):
diff --git a/rtemstoolkit/python-wrapper.sh b/rtemstoolkit/python-wrapper.sh
new file mode 100644
index 0000000..028e81b
--- /dev/null
+++ b/rtemstoolkit/python-wrapper.sh
@@ -0,0 +1,54 @@
+#
+# RTEMS Tools Project (http://www.rtems.org/)
+# Copyright 2018 Chris Johns (chrisj@rtems.org)
+# All rights reserved.
+#
+# This file is part of the RTEMS Tools package in 'rtems-tools'.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# This script wraps python finding a suitable version to use.
+#
+set -e
+if test ! -f $PYTHON_CMD; then
+  echo "error: RTEMS Toolkit python command not found: $PYTHON_CMD"
+  exit 5
+fi
+for py in python3 python2 python
+do
+  set +e
+  py_cmd=$(command -v $py)
+  set -e
+  if test -n "$RTEMS_PYTHON_OVERRIDE"; then
+    if test "$RTEMS_PYTHON_OVERRIDE" != "$py"; then
+      py_cmd=""
+    fi
+  fi
+  if test -n "$py_cmd"; then
+    exec $py_cmd $PYTHON_CMD $0 $*
+  fi
+done
+echo "error: RTEMS Toolkit no valid python found"
+exit 5
diff --git a/rtemstoolkit/rtems.py b/rtemstoolkit/rtems.py
index 8aa22e5..157cb5b 100755
--- a/rtemstoolkit/rtems.py
+++ b/rtemstoolkit/rtems.py
@@ -62,15 +62,21 @@ def clean_windows_path():
         if 'msys' in cspath[0] and cspath[0].endswith('bin'):
             os.environ['PATH'] = os.pathsep.join(cspath[1:])
 
-def configuration_path():
+def configuration_path(prog = None):
     '''Return the path the configuration data path for RTEMS. The path is relative
     to the installed executable. Mangage the installed package and the in source
     tree when running from within the rtems-tools repo.
-
+    Note:
+     1. This code assumes the executable is wrapped and not using 'env'.
+     2. Ok to directly call os.path.
     '''
-    exec_name = os.path.abspath(sys.argv[0])
-    for top in [os.path.dirname(exec_name),
-                os.path.dirname(os.path.dirname(exec_name))]:
+    if prog is None:
+        exec_name = sys.argv[1]
+    else:
+        exec_name = prog
+    exec_name = os.path.abspath(exec_name)
+    for top in [os.path.dirname(os.path.dirname(exec_name)),
+                os.path.dirname(exec_name)]:
         config_path = path.join(top, 'share', 'rtems', 'config')
         if path.exists(config_path):
             break
@@ -80,21 +86,21 @@ def configuration_path():
         config_path = None
     return config_path
 
-def configuration_file(config):
+def configuration_file(config, prog = None):
     '''Return the path to a configuration file for RTEMS. The path is relative to
     the installed executable or we are testing and running from within the
     rtems-tools repo.
 
     '''
-    return path.join(configuration_path(), config)
+    return path.join(configuration_path(prog = prog), config)
 
-def bsp_configuration_file():
+def bsp_configuration_file(prog = None):
     '''Return the path to the BSP configuration file for RTEMS. The path is
     relative to the installed executable or we are testing and running from
     within the rtems-tools repo.
 
     '''
-    return configuration_file('rtems-bsps.ini')
+    return configuration_file('rtems-bsps.ini', prog = prog)
 
 class configuration:
 
@@ -227,13 +233,13 @@ class configuration:
         return ' '.join([self.config_flags('no-' + e) for e in self.excludes(arch, bsp)])
 
     def archs(self):
-        return sorted(self.archs.keys())
+        return sorted(list(self.archs.keys()))
 
     def arch_present(self, arch):
         return arch in self.archs
 
     def arch_excludes(self, arch):
-        excludes = self.archs[arch]['excludes'].keys()
+        excludes = list(self.archs[arch]['excludes'].keys())
         for exclude in self.archs[arch]['excludes']:
             if 'all' not in self.archs[arch]['excludes'][exclude]:
                 excludes.remove(exclude)
@@ -246,7 +252,7 @@ class configuration:
         return bsp in self.archs[arch]['bsps']
 
     def bsp_excludes(self, arch, bsp):
-        excludes = self.archs[arch]['excludes'].keys()
+        excludes = list(self.archs[arch]['excludes'].keys())
         for exclude in self.archs[arch]['excludes']:
             if 'all' not in self.archs[arch]['excludes'][exclude] and \
                bsp not in self.archs[arch]['excludes'][exclude]:
@@ -362,7 +368,7 @@ class configuration:
             s += os.linesep
         if architectures:
             s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
-            archs = sorted(self.archs.keys())
+            archs = sorted(list(self.archs.keys()))
             bsps = 0
             asize = 0
             for arch in archs:
diff --git a/rtemstoolkit/wscript b/rtemstoolkit/wscript
index 1b124c7..82123e5 100644
--- a/rtemstoolkit/wscript
+++ b/rtemstoolkit/wscript
@@ -155,6 +155,9 @@ def build(bld):
                   'windows.py'],
         install_from = '.',
         install_path = '${PREFIX}/share/rtems/rtemstoolkit')
+    bld.install_files('${PREFIX}/share/rtems/rtemstoolkit',
+                      'python-wrapper.sh',
+                      relative_trick = True)
 
 def rebuild(ctx):
     import waflib.Options