libdl: Add symbol searching and loading from archives.

- Load archive symbol tables to support searching of archives
  for symbols.
- Search archive symbols and load the object file that contains
  the symbol.
- Search the global and archives until all remaining unresolved symbols
  are not found. Group the loaded object files in the pending queue.
- Run the object file and loaded dependents as a group before adding to the
  main object list.
- Remove orphaned object files after references are removed.

Updates #3686

6 files changed, 365 insertions, 29 deletions

diff --git a/cpukit/include/rtems/rtl/rtl-archive.h b/cpukit/include/rtems/rtl/rtl-archive.h
new file mode 100644
index 0000000000..1fe3aae385
--- /dev/null
+++ b/cpukit/include/rtems/rtl/rtl-archive.h
@@ -0,0 +1,196 @@
+/*
+ *  COPYRIGHT (c) 2018 Chris Johns <chrisj@rtems.org>
+ *
+ *  The license and distribution terms for this file may be
+ *  found in the file LICENSE in this distribution or at
+ *  http://www.rtems.org/license/LICENSE.
+ */
+/**
+ * @file
+ *
+ * @ingroup rtems_rtl
+ *
+ * @brief RTEMS Run-Time Linker Archive
+ *
+ * The RTL Archive module manages dependent loading of object files from
+ * archives. The archives need to have a `ranlib` generated symbol table.
+ *
+ * This module reads a configuration file called `rtl-libs.conf` from a default
+ * directory of `/etc`. The file is a line per glob'ed path to archives to
+ * search for symbols.
+ *
+ * The archive symbols are held in a per archive cache for searching.
+ *
+ * @note Errors in the reading of a config file, locating archives, reading
+ *       symbol tables and loading object files are not considered RTL error
+ *       reported to a user. The user error is undefined symbols.
+ */
+
+#if !defined (_RTEMS_RTL_ARCHIVE_H_)
+#define _RTEMS_RTL_ARCHIVE_H_
+
+#include <rtems.h>
+#include <rtems/chain.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/**
+ * Flags for archives.
+ */
+#define RTEMS_RTL_ARCHIVE_USER_LOAD (1 << 0) /**< User forced load. */
+#define RTEMS_RTL_ARCHIVE_REMOVE    (1 << 1) /**< The achive is not found. */
+#define RTEMS_RTL_ARCHIVE_LOAD      (1 << 2) /**< Load the achive. */
+
+/**
+ * Symbol search and loading results.
+ */
+typedef enum rtems_rtl_archive_search
+{
+  rtems_rtl_archive_search_not_found = 0, /**< The search failed to find the
+                                               symbol. */
+  rtems_rtl_archive_search_found = 1,     /**< The symbols was found. */
+  rtems_rtl_archive_search_loaded = 2,    /**< The symbol was found and the
+                                               object file has been loaded */
+  rtems_rtl_archive_search_error = 3,     /**< There was an error searching or
+                                               loading. */
+  rtems_rtl_archive_search_no_config = 4 /**< There is no config or it is
+                                          *   invalid. */
+} rtems_rtl_archive_search;
+
+/**
+ * RTL Archive symbols.
+ */
+typedef struct rtems_rtl_archive_symbol
+{
+  size_t      entry;  /**< Index in the symbol offset table. */
+  const char* label;  /**< The symbol's label. */
+} rtems_rtl_archive_symbol;
+
+/**
+ * RTL Archive symbols.
+ */
+typedef struct rtems_rtl_archive_symbols
+{
+  void*                     base;     /**< Base of the symbol table. */
+  size_t                    size;     /**< Size of the symbol table. */
+  size_t                    entries;  /**< Entries in the symbol table. */
+  const char*               names;    /**< Start of the symbol names. */
+  rtems_rtl_archive_symbol* symbols;  /**< Sorted symbol table. */
+} rtems_rtl_archive_symbols;
+
+/**
+ * RTL Archive data.
+ */
+typedef struct rtems_rtl_archive
+{
+  rtems_chain_node          node;     /**< Chain link. */
+  const char*               name;     /**< Archive absolute path. */
+  size_t                    size;     /**< Size of the archive. */
+  time_t                    mtime;    /**< Archive's last modified time. */
+  off_t                     enames;   /**< Extended file name offset, lazy load. */
+  rtems_rtl_archive_symbols symbols;  /**< Ranlib symbol table. */
+  size_t                    refs;     /**< Loaded object modules. */
+  uint32_t                  flags;    /**< Some flags. */
+} rtems_rtl_archive;
+
+/**
+ * RTL Archive data.
+ */
+typedef struct rtems_rtl_archives
+{
+  const char*         config_name;    /**< Config file name. */
+  time_t              config_mtime;   /**< Config last modified time. */
+  size_t              config_length;  /**< Length the config data. */
+  const char*         config;         /**< Config file contents. */
+  rtems_chain_control archives;       /**< The located archives. */
+} rtems_rtl_archives;
+
+/**
+ * Error handler call when finding an archive.
+ */
+typedef void (*rtems_rtl_archive_error)(int num, const char* text);
+
+/**
+ * Open the RTL archives support with the specified configration file.
+ *
+ * @param archives The archives data to open.
+ * @param config The path to the configuration file.
+ * @return bool Returns @true is the archives are open.
+ */
+void rtems_rtl_archives_open (rtems_rtl_archives* archives, const char* config);
+
+/**
+ * Close the RTL archives support.
+ *
+ * @param archives The archives data to close.
+ */
+void rtems_rtl_archives_close (rtems_rtl_archives* archives);
+
+/**
+ * Refresh the archives data. Check if the configuration has changes and if it
+ * has reload it. Check each path in the configuration and creates archive
+ * instances for new archives and remove archives not present any more.
+ *
+ * Refreshing is a development aid so reboots can be avoided as users trial
+ * configurations that work.
+ *
+ * @param archives The archives data to refresh.
+ * @retval false The refresh failed, an error will have been set.
+ */
+bool rtems_rtl_archives_refresh (rtems_rtl_archives* archives);
+
+/**
+ * Load an archive.
+ *
+ * @param archives The archives data to search.
+ * @param name     The archive to load.
+ * @retval true    The archive is loaded.
+ */
+bool rtems_rtl_archive_load (rtems_rtl_archives* archives, const char* name);
+
+/**
+ * Search for a symbol and load the first object file that has the symbol.
+ *
+ * @param archives The archives data to search.
+ * @param symbol   The symbol name to search for.
+ * @param load     If @true load the object file the symbol is found in
+ *                 else return the found not found status.
+ */
+rtems_rtl_archive_search rtems_rtl_archive_obj_load (rtems_rtl_archives* archives,
+                                                     const char*         symbol,
+                                                     bool                load);
+
+/**
+ * Find a module in an archive returning the offset in the archive and
+ * the size. If the name field is pointing to a string pointer and
+ * that poniter is NULL and the offset is valid the name is extracted
+ * from the archive and filled in. This is used when loading a file
+ * from the archive after a symbol is found. The file name is not know
+ * and could be extended which requires searching the extended string
+ * table in the archive.
+ *
+ * @param fd Open file handle for the archive.
+ * @param fsize Size of the archive.
+ * @paarm name Pointer to the name string.
+ * @param offset The offset of the file in the archive.
+ * @param size The size of the file in the archive.
+ * @param extended_names The offset in the archive for the extended names.
+ * @param error The error handler called on an error.
+ * @retval true The file was found in the archive.
+ * @retval false The file was not found.
+ */
+bool rtems_rtl_obj_archive_find_obj (int                     fd,
+                                     size_t                  fsize,
+                                     const char**            name,
+                                     off_t*                  offset,
+                                     size_t*                 size,
+                                     off_t*                  extended_names,
+                                     rtems_rtl_archive_error error);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif
diff --git a/cpukit/include/rtems/rtl/rtl-obj.h b/cpukit/include/rtems/rtl/rtl-obj.h
index 976d6c293f..bc503887e2 100644
--- a/cpukit/include/rtems/rtl/rtl-obj.h
+++ b/cpukit/include/rtems/rtl/rtl-obj.h
@@ -92,9 +92,8 @@ typedef struct rtems_rtl_loader_table
 #define RTEMS_RTL_OBJ_SECT_DATA  (1 << 2)  /**< Section holds program data. */
 #define RTEMS_RTL_OBJ_SECT_BSS   (1 << 3)  /**< Section holds program bss. */
 #define RTEMS_RTL_OBJ_SECT_EH    (1 << 4)  /**< Section holds exception data. */
-#define RTEMS_RTL_OBJ_SECT_REL   (1 << 5)  /**< Section holds relocation records. */
-#define RTEMS_RTL_OBJ_SECT_RELA  (1 << 6)  /**< Section holds relocation addend
-                                            *   records. */
+#define RTEMS_RTL_OBJ_SECT_REL   (1 << 5)  /**< Section holds relocation recs. */
+#define RTEMS_RTL_OBJ_SECT_RELA  (1 << 6)  /**< Section holds reloc addend recs. */
 #define RTEMS_RTL_OBJ_SECT_SYM   (1 << 7)  /**< Section holds symbols. */
 #define RTEMS_RTL_OBJ_SECT_STR   (1 << 8)  /**< Section holds strings. */
 #define RTEMS_RTL_OBJ_SECT_ALLOC (1 << 9)  /**< Section allocates runtime memory. */
@@ -168,6 +167,7 @@ typedef bool (*rtems_rtl_obj_depends_iterator) (rtems_rtl_obj* obj,
 #define RTEMS_RTL_OBJ_BASE         (1 << 2) /**< The base image. */
 #define RTEMS_RTL_OBJ_RELOC_TAG    (1 << 3) /**< Tag the object as visited when reloc
                                              *   parsing. */
+#define RTEMS_RTL_OBJ_DEP_VISITED  (1 << 4) /**< Dependency loop detection. */
 
 /**
  * RTL Object. There is one for each object module loaded plus one for the base
@@ -338,6 +338,20 @@ static inline size_t rtems_rtl_obj_align (size_t   offset,
 }
 
 /**
+ * Is the symbol in this object's files globa symbol table?
+ *
+ * @param obj The object file's descriptor to search.
+ * @param sym The symbol to check.
+ * @retval bool Returns @true if present else @false is returned.
+ */
+static inline bool rtems_rtl_obj_has_symbol (const rtems_rtl_obj*     obj,
+                                             const rtems_rtl_obj_sym* sym)
+{
+  return (sym >= obj->global_table &&
+          sym < (obj->global_table + obj->global_syms));
+}
+
+/**
  * Allocate an object structure on the heap.
  *
  * @retval NULL No memory for the object.
@@ -380,14 +394,6 @@ bool rtems_rtl_parse_name (const char*  name,
                            off_t*       ooffset);
 
 /**
- * Check of the name matches the object file's object name.
- *
- * @param obj The object file's descriptor.
- * @param name The name to match.
- */
-bool rtems_rtl_match_name (rtems_rtl_obj* obj, const char* name);
-
-/**
  * Find an object file on disk that matches the name. The object descriptor is
  * fill in with the various parts of a name. A name can have archive, object
  * file and offset components. The search path in the RTL is searched.
@@ -497,6 +503,15 @@ void rtems_rtl_obj_erase_dependents (rtems_rtl_obj* obj);
 bool rtems_rtl_obj_add_dependent (rtems_rtl_obj* obj, rtems_rtl_obj* dependent);
 
 /**
+ * Remove dependencies. This decrements the dependent object file references.
+ *
+ * @param obj The object file's descriptor.
+ * @retval true The dependencies have been removed.
+ * @retval false There is no space in the table.
+ */
+bool rtems_rtl_obj_remove_dependencies (rtems_rtl_obj* obj);
+
+/**
  * Iterate over the module dependenices.
  *
  * @param obj The object file's descriptor.
@@ -690,6 +705,13 @@ bool rtems_rtl_obj_load_sections (rtems_rtl_obj*             obj,
                                   void*                      data);
 
 /**
+ * Does the object have constructors to run?
+ *
+ * @return bool True if there are constructors to run.
+ */
+bool rtems_rtl_obj_ctors_to_run (rtems_rtl_obj* obj);
+
+/**
  * Invoke the constructors the object has. Constructors are a table of pointers
  * to "void (*)(void);" where NULL pointers are skipped. The table's size is
  * taken from the section's size. The objet ELF specific code is responisble
@@ -700,6 +722,13 @@ bool rtems_rtl_obj_load_sections (rtems_rtl_obj*             obj,
 void rtems_rtl_obj_run_ctors (rtems_rtl_obj* obj);
 
 /**
+ * Does the object have destructors to run?
+ *
+ * @return bool True if there are destructors to run.
+ */
+bool rtems_rtl_obj_dtors_to_run (rtems_rtl_obj* obj);
+
+/**
  * Invoke the destructors the object has. Destructors are a table of pointers
  * to "void (*)(void);" where NULL pointers are skipped. The table's size is
  * taken from the section's size. The objet ELF specific code is responisble
@@ -710,6 +739,13 @@ void rtems_rtl_obj_run_ctors (rtems_rtl_obj* obj);
 void rtems_rtl_obj_run_dtors (rtems_rtl_obj* obj);
 
 /**
+ * Get the object file reference count.
+ *
+ * @retval int The object file's reference count.
+ */
+size_t rtems_rtl_obj_get_reference (rtems_rtl_obj* obj);
+
+/**
  * Increment the object file reference count.
  *
  * @param obj The object file's descriptor.
@@ -724,6 +760,14 @@ void rtems_rtl_obj_inc_reference (rtems_rtl_obj* obj);
 void rtems_rtl_obj_dec_reference (rtems_rtl_obj* obj);
 
 /**
+ * Is the object file orphaned? An orphaned object file is not locked, has no
+ * users and it not being referenced.
+ *
+ * @param obj The object file's descriptor.
+ */
+bool rtems_rtl_obj_orphaned (rtems_rtl_obj* obj);
+
+/**
  * Load the object file, reading all sections into memory, symbols and
  * performing any relocation fixups.
  *
diff --git a/cpukit/include/rtems/rtl/rtl-shell.h b/cpukit/include/rtems/rtl/rtl-shell.h
new file mode 100644
index 0000000000..b0a6e8d8d4
--- /dev/null
+++ b/cpukit/include/rtems/rtl/rtl-shell.h
@@ -0,0 +1,37 @@
+/*
+ *  COPYRIGHT (c) 2012 Chris Johns <chrisj@rtems.org>
+ *
+ *  The license and distribution terms for this file may be
+ *  found in the file LICENSE in this distribution or at
+ *  http://www.rtems.org/license/LICENSE.
+ */
+/**
+ * @file
+ *
+ * @ingroup rtems_rtl
+ *
+ * @brief RTEMS Run-Time Linker ELF Shell Support.
+ */
+
+#if !defined (_RTEMS_RTL_SHELL_H_)
+#define _RTEMS_RTL_SHELL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/**
+ * The RTL single shell command contains sub-commands.
+ *
+ * @param argc The argument count.
+ * @param argv Array of argument strings.
+ * @retval 0 No error.
+ * @return int The exit code.
+ */
+int rtems_rtl_shell_command (int argc, char* argv[]);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif
diff --git a/cpukit/include/rtems/rtl/rtl-trace.h b/cpukit/include/rtems/rtl/rtl-trace.h
index 4b93c8c91c..84e81d5344 100644
--- a/cpukit/include/rtems/rtl/rtl-trace.h
+++ b/cpukit/include/rtems/rtl/rtl-trace.h
@@ -48,6 +48,8 @@ typedef uint32_t rtems_rtl_trace_mask;
 #define RTEMS_RTL_TRACE_ALLOCATOR              (1UL << 9)
 #define RTEMS_RTL_TRACE_UNRESOLVED             (1UL << 10)
 #define RTEMS_RTL_TRACE_CACHE                  (1UL << 11)
+#define RTEMS_RTL_TRACE_ARCHIVES               (1UL << 12)
+#define RTEMS_RTL_TRACE_DEPENDENCY             (1UL << 13)
 #define RTEMS_RTL_TRACE_ALL                    (0xffffffffUL & ~(RTEMS_RTL_TRACE_CACHE))
 
 /**
diff --git a/cpukit/include/rtems/rtl/rtl-unresolved.h b/cpukit/include/rtems/rtl/rtl-unresolved.h
index a69b51ac7f..a425384370 100644
--- a/cpukit/include/rtems/rtl/rtl-unresolved.h
+++ b/cpukit/include/rtems/rtl/rtl-unresolved.h
@@ -71,21 +71,30 @@ typedef uint32_t rtems_rtl_word;
 typedef enum rtems_rtl_unresolved_rtype
 {
   rtems_rtl_unresolved_empty = 0,  /**< The records is empty. Must always be 0 */
-  rtems_rtl_unresolved_name = 1,   /**< The record is a name. */
+  rtems_rtl_unresolved_symbol = 1, /**< The record is a symbol. */
   rtems_rtl_unresolved_reloc = 2   /**< The record is a relocation record. */
 } rtems_rtl_unresolved_rtype;
 
 /**
- * Unresolved externals symbol names. The names are reference counted and
- * separate from the relocation records because a number of records could
- * reference the same symbol name.
+ * Unresolved external symbol flags.
  */
-typedef struct rtems_rtl_unresolv_name
+#define RTEMS_RTL_UNRESOLV_SYM_SEARCH_ARCHIVE (1 << 0) /**< Search the archive. */
+
+/**
+ * Unresolved externals symbols. The symbols are reference counted and separate
+ * from the relocation records because a number of records could reference the
+ * same symbol.
+ *
+ * The name is extended in the allocator of the record in the unresolved data
+ * block. The 10 is a minimum that is added to by joining more than one record.
+ */
+typedef struct rtems_rtl_unresolv_symbol
 {
   uint16_t   refs;     /**< The number of references to this name. */
+  uint16_t   flags;    /**< Flags to manage the symbol. */
   uint16_t   length;   /**< The length of this name. */
-  const char name[12]; /**< The symbol name. */
-} rtems_rtl_unresolv_name;
+  const char name[10]; /**< The symbol name. */
+} rtems_rtl_unresolv_symbol;
 
 /**
  * Unresolved externals symbols require the relocation records to be held
@@ -108,8 +117,8 @@ typedef struct rtems_rtl_unresolv_rec
   rtems_rtl_unresolved_rtype type;
   union
   {
-    rtems_rtl_unresolv_name  name;   /**< The name, or */
-    rtems_rtl_unresolv_reloc reloc;  /**< the relocation record. */
+    rtems_rtl_unresolv_symbol name;   /**< The symnbol, or */
+    rtems_rtl_unresolv_reloc  reloc;  /**< the relocation record. */
   } rec;
 } rtems_rtl_unresolv_rec;
 
@@ -166,8 +175,8 @@ void rtems_rtl_unresolved_table_close (rtems_rtl_unresolved* unresolved);
 /**
  * Iterate over the table of unresolved entries.
  */
-bool rtems_rtl_unresolved_interate (rtems_rtl_unresolved_iterator iterator,
-                                    void*                         data);
+bool rtems_rtl_unresolved_iterate (rtems_rtl_unresolved_iterator iterator,
+                                   void*                         data);
 
 /**
  * Add a relocation to the list of unresolved relocations.
@@ -206,6 +215,12 @@ bool rtems_rtl_unresolved_remove (rtems_rtl_obj*        obj,
                                   const rtems_rtl_word* rel);
 
 /**
+ * Set all symbols to be archive searchable. This is done when the available
+ * archives have been refreshed and there are new archives to search for.
+ */
+void rtems_rtl_unresolved_set_archive_search (void);
+
+/**
  * Dump the RTL unresolved data.
  */
 void rtems_rtl_unresolved_dump (void);
diff --git a/cpukit/include/rtems/rtl/rtl.h b/cpukit/include/rtems/rtl/rtl.h
index 41cb010ef4..debf17cd44 100644
--- a/cpukit/include/rtems/rtl/rtl.h
+++ b/cpukit/include/rtems/rtl/rtl.h
@@ -24,6 +24,7 @@
 #include <rtems/thread.h>
 
 #include <rtems/rtl/rtl-allocator.h>
+#include <rtems/rtl/rtl-archive.h>
 #include <rtems/rtl/rtl-fwd.h>
 #include <rtems/rtl/rtl-obj.h>
 #include <rtems/rtl/rtl-obj-cache.h>
@@ -105,8 +106,10 @@ struct rtems_rtl_data
   rtems_recursive_mutex lock;           /**< The RTL lock */
   rtems_rtl_alloc_data  allocator;      /**< The allocator data. */
   rtems_chain_control   objects;        /**< List if loaded object files. */
+  rtems_chain_control   pending;        /**< Listof object files needing work. */
   const char*           paths;          /**< Search paths for archives. */
   rtems_rtl_symbols     globals;        /**< Global symbol table. */
+  rtems_rtl_archives    archives;       /**< Archive search and loader. */
   rtems_rtl_unresolved  unresolved;     /**< Unresolved symbols. */
   rtems_rtl_obj*        base;           /**< Base object file. */
   rtems_rtl_obj_cache   symbols;        /**< Symbols object file cache. */
@@ -126,8 +129,8 @@ struct rtems_rtl_data
 rtems_rtl_data* rtems_rtl_data_unprotected (void);
 
 /**
- * Get the RTL global symbol table with out locking. This call assmes the RTL
- * is locked.
+ * Get the RTL global symbol table with out locking. This call assumes
+ * the RTL is locked.
  *
  * @return rtems_rtl_symbols* The RTL global symbols after being locked.
  * @retval NULL The RTL data is not initialised.
@@ -135,7 +138,24 @@ rtems_rtl_data* rtems_rtl_data_unprotected (void);
 rtems_rtl_symbols* rtems_rtl_global_symbols (void);
 
 /**
- * Get the RTL resolved table with out locking. This call assmes the RTL
+ * Get the RTL objects table with out locking. This call assumes the RTL
+ * is locked.
+ *
+ * @return rtems_chain_control* The RTL objects chain.
+ * @retval NULL The RTL data is not initialised.
+ */
+rtems_chain_control* rtems_rtl_objects_unprotected (void);
+
+/**
+ * Get the RTL pending with out locking. This call assumes the RTL is locked.
+ *
+ * @return rtems_chain_control* The RTL pending list control.
+ * @retval NULL The RTL data is not initialised.
+ */
+rtems_chain_control* rtems_rtl_pending_unprotected (void);
+
+/**
+ * Get the RTL unresolved table with out locking. This call assumes the RTL
  * is locked.
  *
  * @return rtems_rtl_unresolv* The RTL unresolved symbols and reloc records.
@@ -144,6 +164,14 @@ rtems_rtl_symbols* rtems_rtl_global_symbols (void);
 rtems_rtl_unresolved* rtems_rtl_unresolved_unprotected (void);
 
 /**
+ * Get the RTL archives with out locking. This call assumes the RTL is locked.
+ *
+ * @return rtems_rtl_archives* The RTL acrhives.
+ * @retval NULL The RTL data is not initialised.
+ */
+rtems_rtl_archives* rtems_rtl_archives_unprotected (void);
+
+/**
  * Get the RTL symbols, strings, or relocations object file caches. This call
  * assmes the RTL is locked.
  *
@@ -276,12 +304,26 @@ rtems_rtl_obj* rtems_rtl_load_object (const char* name, int mode);
 bool rtems_rtl_unload_object (rtems_rtl_obj* obj);
 
 /**
- * Run any constructor functions the object file may contain. This call
- * assumes the linker is unlocked.
+ * Load an object file.  This is the user accessable interface to unloading an
+ * object file. See @rtems_rtl_load_object.
+ *
+ * @param name The name of the object file.
+ * @param mode The mode of the load as defined by the dlopen call.
+ * @return rtl_obj* The object file descriptor. NULL is returned if the load fails.
+ */
+rtems_rtl_obj* rtems_rtl_load (const char* name, int mode);
+
+/**
+ * Unload an object file. This is the user accessable interface to unloading an
+ * object file. See @rtems_rtl_unload_object.
+ *
+ * Assumes the RTL has been locked.
  *
- * @param obj The object file.
+ * @param obj The object file descriptor.
+ * @retval true The object file has been unloaded.
+ * @retval false The object file could not be unloaded.
  */
-void rtems_rtl_run_ctors (rtems_rtl_obj* obj);
+bool rtems_rtl_unload (rtems_rtl_obj* obj);
 
 /**
  * Get the last error message clearing it. This operation locks the run time