build: Avoid cache if CSafeLoader is available

Move the item cache handling to a ItemCache class.  Implement this class
depending on the availability of CSafeLoader.

1 files changed, 143 insertions, 96 deletions

diff --git a/wscript b/wscript
index 076c9790ac..ec5147b429 100755
--- a/wscript
+++ b/wscript
@@ -40,6 +40,7 @@ try:
 except:
     import ConfigParser as configparser
 
+from waflib.Errors import WafError
 from waflib.TaskGen import after, before_method, feature
 
 is_windows_host = os.name == "nt" or sys.platform in ["msys", "cygwin"]
@@ -51,7 +52,6 @@ version = {
 }
 default_prefix = "/opt/rtems/" + version["__RTEMS_MAJOR__"]
 compilers = ["gcc", "clang"]
-items = {}
 bsps = {}
 
 
@@ -1173,106 +1173,61 @@ class BuildItemContext(object):
         self.objects = objects
 
 
-def is_one_item_newer(ctx, path, mtime):
-    try:
-        mtime2 = os.path.getmtime(path)
-        if mtime <= mtime2:
-            return True
-        names = os.listdir(path)
-    except Exception as e:
-        ctx.fatal("Cannot access build specification directory: {}".format(e))
-    for name in names:
-        path2 = os.path.join(path, name)
-        if name.endswith(".yml") and not name.startswith("."):
-            mtime2 = os.path.getmtime(path2)
-            if mtime <= mtime2:
-                return True
-        else:
-            mode = os.lstat(path2).st_mode
-            if stat.S_ISDIR(mode) and is_one_item_newer(ctx, path2, mtime):
-                return True
-    return False
-
-
-def must_update_item_cache(ctx, path, cache_file):
-    try:
-        mtime = os.path.getmtime(cache_file)
-    except:
-        return True
-    return is_one_item_newer(ctx, path, mtime)
-
-
-def load_from_yaml(ctx, data_by_uid, base, path):
-    try:
-        names = os.listdir(path)
-    except Exception as e:
-        ctx.fatal("Cannot list build specification directory: {}".format(e))
-    for name in names:
-        path2 = os.path.join(path, name)
-        if name.endswith(".yml") and not name.startswith("."):
-            uid = "/" + os.path.relpath(path2, base).replace(".yml", "")
-            with open(path2, "r") as f:
-                data_by_uid[uid] = load(f.read(), SafeLoader)
-        else:
-            mode = os.lstat(path2).st_mode
-            if stat.S_ISDIR(mode):
-                load_from_yaml(ctx, data_by_uid, base, path2)
-
-
-def load_items_in_directory(ctx, ctors, path):
-    p = "c4che/" + re.sub(r"[^\w]", "_", path) + ".pickle"
-    try:
-        f = ctx.bldnode.make_node(p)
-    except AttributeError:
-        f = ctx.path.make_node("build/" + p)
-    f.parent.mkdir()
-    cache_file = f.abspath()
-    data_by_uid = {}
-    if must_update_item_cache(ctx, path, cache_file):
-        from waflib import Logs
-
-        Logs.warn(
-            "Regenerate build specification cache (needs a couple of seconds)..."
-        )
+ctors = {
+    "ada-test-program": AdaTestProgramItem,
+    "bsp": BSPItem,
+    "config-file": ConfigFileItem,
+    "config-header": ConfigHeaderItem,
+    "test-program": TestProgramItem,
+    "group": GroupItem,
+    "library": LibraryItem,
+    "objects": ObjectsItem,
+    "option": OptionItem,
+    "script": ScriptItem,
+    "start-file": StartFileItem,
+}
 
-        load_from_yaml(ctx, data_by_uid, path, path)
-        with open(cache_file, "wb") as f:
-            pickle.dump(data_by_uid, f)
-    else:
-        with open(cache_file, "rb") as f:
-            data_by_uid = pickle.load(f)
-    for uid, data in data_by_uid.items():
-        if data["type"] == "build":
-            items[uid] = ctors[data["build-type"]](uid, data)
+try:
+    #
+    # Try to use the system-provided yaml module with libyaml support.
+    #
+    from yaml import load, CSafeLoader as SafeLoader
 
+    class ItemCache(dict):
 
-def load_items(ctx, specs):
-    if items:
-        return
+        def __init__(self):
+            super(ItemCache, self).__init__()
+            self.spec_directories = []
 
-    ctors = {
-        "ada-test-program": AdaTestProgramItem,
-        "bsp": BSPItem,
-        "config-file": ConfigFileItem,
-        "config-header": ConfigHeaderItem,
-        "test-program": TestProgramItem,
-        "group": GroupItem,
-        "library": LibraryItem,
-        "objects": ObjectsItem,
-        "option": OptionItem,
-        "script": ScriptItem,
-        "start-file": StartFileItem,
-    }
+        def set_spec_directories(self, spec_directories):
+            self.spec_directories = spec_directories
 
-    for path in specs:
-        load_items_in_directory(ctx, ctors, path)
+        def load_from_yaml(self, base, path):
+            try:
+                names = os.listdir(path)
+            except Exception as e:
+                raise WafError(
+                    "Cannot list build specification directory: {}".format(e))
+            for name in names:
+                path2 = os.path.join(path, name)
+                if name.endswith(".yml") and not name.startswith("."):
+                    uid = "/" + os.path.relpath(path2, base).replace(
+                        ".yml", "")
+                    with open(path2, "r") as f:
+                        data = load(f.read(), SafeLoader)
+                        if data["type"] == "build":
+                            self[uid] = ctors[data["build-type"]](uid, data)
+                else:
+                    mode = os.lstat(path2).st_mode
+                    if stat.S_ISDIR(mode):
+                        self.load_from_yaml(base, path2)
 
+        def load_all(self, ctx):
+            if bool(self):
+                return
 
-try:
-    #
-    # Try to use the system-provided yaml module with libyaml support.
-    #
-    from yaml import load, CSafeLoader as SafeLoader
+            for path in self.spec_directories:
+                self.load_from_yaml(path, path)
 except ImportError:
     #
     # Fall back to the Python implementation provided by the project.  This
@@ -1286,6 +1241,96 @@ except ImportError:
     sys.path += [yaml_path]
     from yaml import load, SafeLoader
 
+    def is_one_item_newer(ctx, path, mtime):
+        try:
+            mtime2 = os.path.getmtime(path)
+            if mtime <= mtime2:
+                return True
+            names = os.listdir(path)
+        except Exception as e:
+            ctx.fatal(
+                "Cannot access build specification directory: {}".format(e))
+        for name in names:
+            path2 = os.path.join(path, name)
+            if name.endswith(".yml") and not name.startswith("."):
+                mtime2 = os.path.getmtime(path2)
+                if mtime <= mtime2:
+                    return True
+            else:
+                mode = os.lstat(path2).st_mode
+                if stat.S_ISDIR(mode) and is_one_item_newer(ctx, path2, mtime):
+                    return True
+        return False
+
+    def must_update_item_cache(ctx, path, cache_file):
+        try:
+            mtime = os.path.getmtime(cache_file)
+        except:
+            return True
+        return is_one_item_newer(ctx, path, mtime)
+
+    def load_from_yaml(ctx, data_by_uid, base, path):
+        try:
+            names = os.listdir(path)
+        except Exception as e:
+            ctx.fatal(
+                "Cannot list build specification directory: {}".format(e))
+        for name in names:
+            path2 = os.path.join(path, name)
+            if name.endswith(".yml") and not name.startswith("."):
+                uid = "/" + os.path.relpath(path2, base).replace(".yml", "")
+                with open(path2, "r") as f:
+                    data_by_uid[uid] = load(f.read(), SafeLoader)
+            else:
+                mode = os.lstat(path2).st_mode
+                if stat.S_ISDIR(mode):
+                    load_from_yaml(ctx, data_by_uid, base, path2)
+
+    def load_items_in_directory(ctx, path):
+        p = "c4che/" + re.sub(r"[^\w]", "_", path) + ".pickle"
+        try:
+            f = ctx.bldnode.make_node(p)
+        except AttributeError:
+            f = ctx.path.make_node("build/" + p)
+        f.parent.mkdir()
+        cache_file = f.abspath()
+        data_by_uid = {}
+        if must_update_item_cache(ctx, path, cache_file):
+            from waflib import Logs
+
+            Logs.warn("Regenerate the build specification cache.  "
+                      "Install the PyYAML Python package to avoid this.  "
+                      "The cache regeneration needs a couple of seconds...")
+
+            load_from_yaml(ctx, data_by_uid, path, path)
+            with open(cache_file, "wb") as f:
+                pickle.dump(data_by_uid, f)
+        else:
+            with open(cache_file, "rb") as f:
+                data_by_uid = pickle.load(f)
+        for uid, data in data_by_uid.items():
+            if data["type"] == "build":
+                items[uid] = ctors[data["build-type"]](uid, data)
+
+    class ItemCache(dict):
+
+        def __init__(self):
+            super(ItemCache, self).__init__()
+            self.spec_directories = []
+
+        def set_spec_directories(self, spec_directories):
+            self.spec_directories = spec_directories
+
+        def load_all(self, ctx):
+            if bool(self):
+                return
+
+            for path in self.spec_directories:
+                load_items_in_directory(ctx, path)
+
+
+items = ItemCache()
+
 
 def load_items_from_options(ctx):
     specs = ctx.options.rtems_specs
@@ -1293,7 +1338,8 @@ def load_items_from_options(ctx):
         specs = specs.split(",")
     else:
         specs = ["spec/build"]
-    load_items(ctx, specs)
+    items.set_spec_directories(specs)
+    items.load_all(ctx)
     return specs
 
 
@@ -1589,7 +1635,8 @@ def build(bld):
             bld,
             ["compiler", "config", "specs", "tools", "top_group"],
         )
-        load_items(bld, bld.env.SPECS)
+        items.set_spec_directories(bld.env.SPECS)
+        items.load_all(bld)
         append_variant_builds(bld)
         return
     long_command_line_workaround(bld)