libmisc/shell: Add the rtrace command for buffered tracing support.

The rtrace command interfaces to the RTEMS Trace Linker's trace
buffering data allowing users to capture and report trace data.

7 files changed, 1051 insertions, 2 deletions

diff --git a/cpukit/Makefile.am b/cpukit/Makefile.am
index 10b59db111..7df116ce07 100644
--- a/cpukit/Makefile.am
+++ b/cpukit/Makefile.am
@@ -172,6 +172,11 @@ include_rtems_HEADERS += libmisc/capture/capture.h
 include_rtems_HEADERS += libmisc/capture/capture-cli.h
 include_rtems_HEADERS += libmisc/capture/captureimpl.h
 
+# tracing headers
+include_rtems_tracedir = $(include_rtemsdir)/trace
+include_rtems_trace_HEADERS =
+include_rtems_trace_HEADERS += libmisc/capture/rtems-trace-buffer-vars.h
+
 ## cpuuse
 include_rtems_HEADERS += libmisc/cpuuse/cpuuse.h
 
diff --git a/cpukit/libmisc/Makefile.am b/cpukit/libmisc/Makefile.am
index f09b69d311..ac703adc93 100644
--- a/cpukit/libmisc/Makefile.am
+++ b/cpukit/libmisc/Makefile.am
@@ -21,7 +21,8 @@ libcapture_a_SOURCES = capture/capture.c capture/capture-cli.c \
     capture/capture_user_extension.c capture/capture_buffer.c \
     capture/capture_support.c \
     capture/capture.h capture/captureimpl.h capture/capture-cli.h \
-    capture/capture_buffer.h
+    capture/capture_buffer.h \
+    capture/rtems-trace-buffer-vars.c capture/rtems-trace-buffer-vars.h
 
 ## cpuuse
 EXTRA_DIST += cpuuse/README
@@ -108,7 +109,7 @@ libshell_a_SOURCES = shell/cat_file.c shell/cmds.c shell/internal.h \
     shell/main_setenv.c shell/main_getenv.c shell/main_unsetenv.c \
     shell/main_mkrfs.c shell/main_debugrfs.c shell/main_df.c \
     shell/main_lsof.c shell/main_edit.c \
-    shell/main_blkstats.c \
+    shell/main_blkstats.c shell/main_rtrace.c \
     shell/shell-wait-for-input.c
 libshell_a_SOURCES += shell/main_cmdls.c
 libshell_a_SOURCES += shell/main_cmdchown.c
diff --git a/cpukit/libmisc/capture/rtems-trace-buffer-vars.c b/cpukit/libmisc/capture/rtems-trace-buffer-vars.c
new file mode 100644
index 0000000000..ef958c58f7
--- /dev/null
+++ b/cpukit/libmisc/capture/rtems-trace-buffer-vars.c
@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2015 Chris Johns <chrisj@rtems.org>
+ *
+ *  The license and distribution terms for this file may be
+ *  found in the file LICENSE in this distribution or at
+ *  http://www.rtems.org/license/LICENSE.
+ */
+
+#include <rtems.h>
+
+#include <rtems/trace/rtems-trace-buffer-vars.h>
+
+/**
+ * External Trace Linker and TBG data. We provide weak versions to allow us to
+ * link and be present in an application that has not been trace linked.
+ */
+
+/*
+ * Trace linker data.
+ */
+uint32_t __rtld_trace_names_size __attribute__ ((weak));
+const char const* __rtld_trace_names[1] __attribute__ ((weak));
+uint32_t __rtld_trace_enables_size __attribute__ ((weak));
+const uint32_t __rtld_trace_enables[1] __attribute__ ((weak));
+uint32_t __rtld_trace_triggers_size __attribute__ ((weak));
+const uint32_t __rtld_trace_triggers[1] __attribute__ ((weak));
+const __rtld_trace_sig __rtld_trace_signatures[1] __attribute__ ((weak));
+
+/*
+ * Trace buffer generator data.
+ */
+const bool __rtld_tbg_present __attribute__ ((weak));
+const uint32_t __rtld_tbg_mode __attribute__ ((weak));
+const uint32_t __rtld_tbg_buffer_size __attribute__ ((weak));
+uint32_t __rtld_tbg_buffer[1] __attribute__ ((weak));
+volatile uint32_t __rtld_tbg_buffer_in __attribute__ ((weak));
+volatile bool __rtld_tbg_finished __attribute__ ((weak));
+volatile bool __rtld_tbg_triggered __attribute__ ((weak));
+
+uint32_t
+rtems_trace_names_size (void)
+{
+  return __rtld_trace_names_size;
+}
+
+const char*
+rtems_trace_names (const uint32_t index)
+{
+  return __rtld_trace_names[index];
+}
+
+uint32_t
+rtems_trace_enables_size (void)
+{
+  return __rtld_trace_enables_size;
+}
+
+uint32_t
+rtems_trace_enables (const uint32_t index)
+{
+  return __rtld_trace_enables[index];
+}
+
+uint32_t
+rtems_trace_triggers_size (void)
+{
+  return __rtld_trace_triggers_size;
+}
+
+uint32_t
+rtems_trace_triggers (const uint32_t index)
+{
+  return __rtld_trace_triggers[index];
+}
+
+const rtems_trace_sig*
+rtems_trace_signatures (const uint32_t index)
+{
+  return &__rtld_trace_signatures[index];
+}
+
+bool
+rtems_trace_enable_set(const uint32_t index)
+{
+  return (__rtld_trace_enables[index / 32] & (1 << (index & (32 - 1)))) != 0 ? true : false;
+}
+
+bool
+rtems_trace_trigger_set(const uint32_t index)
+{
+  return (__rtld_trace_triggers[index / 32] & (1 << (index & (32 - 1)))) != 0 ? true : false;
+}
+
+bool
+rtems_trace_buffering_present (void)
+{
+  return __rtld_tbg_present;
+}
+
+uint32_t
+rtems_trace_buffering_mode (void)
+{
+  return __rtld_tbg_mode;
+}
+
+uint32_t
+rtems_trace_buffering_buffer_size (void)
+{
+  return __rtld_tbg_buffer_size;
+}
+
+uint32_t*
+rtems_trace_buffering_buffer (void)
+{
+  return &__rtld_tbg_buffer[0];
+}
+
+uint32_t
+rtems_trace_buffering_buffer_in (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  uint32_t                     in;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  in = __rtld_tbg_buffer_in;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+  return in;
+}
+
+bool
+rtems_trace_buffering_finished (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  bool                         finished;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  finished = __rtld_tbg_finished;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+  return finished;
+}
+
+bool
+rtems_trace_buffering_triggered (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  bool                         triggered;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  triggered = __rtld_tbg_triggered;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+  return triggered;
+}
+
+void
+rtems_trace_buffering_start (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  __rtld_tbg_triggered = false;
+  __rtld_tbg_buffer_in = 0;
+  __rtld_tbg_finished = false;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+}
+
+void
+rtems_trace_buffering_stop (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  __rtld_tbg_finished = true;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+}
+
+void
+rtems_trace_buffering_resume (void)
+{
+  rtems_interrupt_lock_context lcontext;
+  rtems_interrupt_lock_acquire(&__rtld_tbg_lock, &lcontext);
+  __rtld_tbg_finished = false;
+  rtems_interrupt_lock_release(&__rtld_tbg_lock, &lcontext);
+}
diff --git a/cpukit/libmisc/capture/rtems-trace-buffer-vars.h b/cpukit/libmisc/capture/rtems-trace-buffer-vars.h
new file mode 100644
index 0000000000..8f0ff08587
--- /dev/null
+++ b/cpukit/libmisc/capture/rtems-trace-buffer-vars.h
@@ -0,0 +1,148 @@
+/**
+ * @file
+ *
+ * @ingroup Shell
+ *
+ * @brief Access to the RTEMS Trace Buffer Generator (TBG).
+ */
+/*
+ *  Copyright (c) 2015 Chris Johns <chrisj@rtems.org>
+ *
+ *  The license and distribution terms for this file may be
+ *  found in the file LICENSE in this distribution or at
+ *  http://www.rtems.org/license/LICENSE.
+ */
+
+#if !defined (_RTEMS_TRACE_BUFFER_VARS_H_)
+#define _RTEMS_TRACE_BUFFER_VARS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/**
+ * These functions are provided as a separated interface to the Trace Buffer
+ * Generatror (TBG) data are not really designed for any real-time performance
+ * type interface.
+ *
+ * Separating the data from the codes stops the compiler incorrectly loop
+ * optimising.
+ */
+
+typedef struct
+{
+  uint32_t          size;
+  const char* const type;
+} __rtld_trace_sig_arg;
+
+  typedef struct {
+    uint32_t                    argc;
+    const __rtld_trace_sig_arg* args;
+} __rtld_trace_sig;
+
+typedef __rtld_trace_sig_arg rtems_trace_sig_arg;
+typedef __rtld_trace_sig rtems_trace_sig;
+
+/**
+ * Returns the number of trace functions.
+ */
+uint32_t  rtems_trace_names_size (void);
+
+/**
+ * Return the name given an index. No range checking.
+ */
+const char* rtems_trace_names (const uint32_t index);
+
+/**
+ * Returns the number of words in the enables array.
+ */
+uint32_t rtems_trace_enables_size (void);
+
+/**
+ * Return the enable 32bit bitmap indexed into the enables array. No range
+ * checking.
+ */
+uint32_t rtems_trace_enables (const uint32_t index);
+
+/**
+ * Returns the number of words in the triggers array.
+ */
+uint32_t rtems_trace_triggers_size (void);
+
+/**
+ * Return the trigger 32bit bitmap indexed into the triggers array. No range
+ * checking.
+ */
+uint32_t rtems_trace_triggers (const uint32_t index);
+
+/**
+ * Return the trace function signature.
+ */
+const rtems_trace_sig* rtems_trace_signatures (const uint32_t index);
+
+/**
+ * Return true is the enable bit is set for the trace function index.
+ */
+bool rtems_trace_enable_set(const uint32_t index);
+
+/**
+ * Return true is the trigger bit is set for the trace function index.
+ */
+bool rtems_trace_trigger_set(const uint32_t index);
+
+/**
+ * The application has been linked with Trace Buffering generated code.
+ */
+bool rtems_trace_buffering_present (void);
+
+/**
+ * Return the trace buffering mode flags.
+ */
+uint32_t rtems_trace_buffering_mode (void);
+
+/**
+ * Return the size of the trace buffering buffer in words.
+ */
+uint32_t rtems_trace_buffering_buffer_size (void);
+
+/**
+ * Return the base of the trace buffering buffer.
+ */
+uint32_t* rtems_trace_buffering_buffer (void);
+
+/**
+ * Return the buffer level. This is only stable if tracing has finished.
+ */
+uint32_t rtems_trace_buffering_buffer_in (void);
+
+/**
+ * The tracing has finished.
+ */
+bool rtems_trace_buffering_finished (void);
+
+/**
+ * Trace has been triggered and enable trace functions are being recorded.
+ */
+bool rtems_trace_buffering_triggered (void);
+
+/**
+ * Start tracing by clearing the triggered flag, setting to 0 and clearing the
+ * finished flag.
+ */
+void rtems_trace_buffering_start (void);
+
+/**
+ * Stop tracing by setting the finished flag.
+ */
+void rtems_trace_buffering_stop (void);
+
+/**
+ * Resume tracing by setting the finished flag.
+ */
+void rtems_trace_buffering_resume (void);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif
diff --git a/cpukit/libmisc/shell/main_rtrace.c b/cpukit/libmisc/shell/main_rtrace.c
new file mode 100644
index 0000000000..b4bb1337ae
--- /dev/null
+++ b/cpukit/libmisc/shell/main_rtrace.c
@@ -0,0 +1,698 @@
+/*
+ *  Copyright (c) 2015 Chris Johns <chrisj@rtems.org>
+ *
+ *  The license and distribution terms for this file may be
+ *  found in the file LICENSE in this distribution or at
+ *  http://www.rtems.org/license/LICENSE.
+ */
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <rtems/shell.h>
+#include <rtems/trace/rtems-trace-buffer-vars.h>
+
+/**
+ * The type of the shell handlers we have.
+ */
+typedef int (*rtems_trace_buffering_shell_handler_t) (int argc, char *argv[]);
+
+/**
+ * Table of handlers we parse to invoke the command.
+ */
+typedef struct
+{
+  const char*                           name;    /**< The sub-command's name. */
+  rtems_trace_buffering_shell_handler_t handler; /**< The sub-command's handler. */
+  const char*                           help;    /**< The sub-command's help. */
+} rtems_trace_buffering_shell_cmd_t;
+
+static int
+rtems_trace_buffering_wrong_number_of_args (void)
+{
+  printf ("error: wrong number of arguments\n");
+  return 1;
+}
+
+static int
+rtems_trace_buffering_no_trace_buffer_code (void)
+{
+  printf("No trace buffer generated code in the application; see rtems-tld\n");
+  return 1;
+}
+
+static void
+rtems_trace_buffering_banner (const char* label)
+{
+  printf("RTEMS Trace Bufferring: %s\n", label);
+}
+
+static void
+rtems_trace_buffering_print_timestamp (uint64_t uptime)
+{
+  uint32_t hours;
+  uint32_t minutes;
+  uint32_t seconds;
+  uint32_t nanosecs;
+  uint64_t up_secs;
+
+  up_secs  = uptime / 1000000000LLU;
+  minutes  = up_secs / 60;
+  hours    = minutes / 60;
+  minutes  = minutes % 60;
+  seconds  = up_secs % 60;
+  nanosecs = uptime % 1000000000;
+
+  printf ("%5" PRIu32 ":%02" PRIu32 ":%02" PRIu32".%09" PRIu32,
+          hours, minutes, seconds, nanosecs);
+}
+
+static int
+rtems_trace_buffering_shell_status (int argc, char *argv[])
+{
+  uint32_t buffer_size;
+  uint32_t buffer_in;
+  bool     finished;
+  bool     triggered;
+  uint32_t names;
+
+  if (argc != 1)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  buffer_size = rtems_trace_buffering_buffer_size ();
+  buffer_in = rtems_trace_buffering_buffer_in ();
+  finished = rtems_trace_buffering_finished ();
+  triggered = rtems_trace_buffering_triggered ();
+  names = rtems_trace_names_size ();
+
+  rtems_trace_buffering_banner ("status");
+  printf("    Running:  %s\n", finished ? "no" : "yes");
+  printf("  Triggered:  %s\n", triggered ? "yes" : "no");
+  printf("      Level: %3" PRIu32 "%%\n", (buffer_in * 100) / buffer_size);
+  printf("     Traces: %4" PRIu32 "\n", names);
+
+  return 0;
+}
+
+static int
+rtems_trace_buffering_shell_funcs (int argc, char *argv[])
+{
+  size_t traces = rtems_trace_names_size ();
+  size_t t;
+  size_t max = 0;
+
+  if (argc != 1)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  rtems_trace_buffering_banner ("trace functions");
+  printf(" Total: %4zu\n", traces);
+
+  for (t = 0; t < traces; ++t)
+  {
+    size_t l = strlen (rtems_trace_names (t));
+    if (l > max)
+      max = l;
+  }
+
+  for (t = 0; t < traces; ++t)
+  {
+    printf(" %4zu: %c%c %-*s\n", t,
+           rtems_trace_enable_set(t) ? 'E' : '-',
+           rtems_trace_trigger_set(t) ? 'T' : '-',
+           max, rtems_trace_names (t));
+  }
+
+  return 0;
+}
+
+static int
+rtems_trace_buffering_shell_start (int argc, char *argv[])
+{
+  if (argc != 1)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  rtems_trace_buffering_banner ("resume");
+
+  if (!rtems_trace_buffering_finished ())
+  {
+    printf("already running\n");
+    return 0;
+  }
+
+  rtems_trace_buffering_start ();
+
+  return 0;
+}
+
+static int
+rtems_trace_buffering_shell_stop (int argc, char *argv[])
+{
+  if (argc != 1)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  rtems_trace_buffering_banner ("stop");
+
+  if (rtems_trace_buffering_finished ())
+  {
+    printf("already stopped\n");
+    return 0;
+  }
+
+  rtems_trace_buffering_stop ();
+
+  return 0;
+}
+
+static int
+rtems_trace_buffering_shell_resume (int argc, char *argv[])
+{
+  if (argc != 1)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  rtems_trace_buffering_banner ("resume");
+
+  if (!rtems_trace_buffering_finished ())
+  {
+    printf("already running\n");
+    return 0;
+  }
+
+  rtems_trace_buffering_start ();
+
+  return 0;
+}
+
+static void rtems_trace_buffering_print_arg (const rtems_trace_sig_arg* arg,
+                                             const uint8_t*             argv)
+{
+  if (arg->size)
+  {
+    union
+    {
+      uint8_t  bytes[sizeof (uint64_t)];
+      uint16_t u16;
+      uint32_t u32;
+      uint64_t u64;
+      void*    pointer;
+    } variable;
+
+    if (arg->size <= sizeof(uint64_t))
+      memcpy (&variable.bytes[0], argv, arg->size);
+
+    printf ("(%s) ", arg->type);
+
+    if (strchr (arg->type, '*') != NULL)
+    {
+      printf ("%p", variable.pointer);
+    }
+    else
+    {
+      size_t b;
+      switch (arg->size)
+      {
+        case 2:
+          printf ("%04" PRIx16, variable.u16);
+          break;
+        case 4:
+          printf ("%08" PRIx32, variable.u32);
+          break;
+        case 8:
+          printf ("%016" PRIx64, variable.u64);
+          break;
+        default:
+          for (b = 0; b < arg->size; ++b)
+            printf ("%02" PRIx32, (uint32_t) *argv++);
+          break;
+      }
+    }
+  }
+}
+
+static int
+rtems_trace_buffering_shell_trace (int argc, char *argv[])
+{
+  uint32_t* trace_buffer;
+  uint32_t  records;
+  uint32_t  traces;
+  uint32_t  r;
+  size_t    start = 0;
+  size_t    end = 40;
+  size_t    count;
+  uint64_t  last_sample = 0;
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  trace_buffer = rtems_trace_buffering_buffer ();
+  records = rtems_trace_buffering_buffer_in ();
+  traces = rtems_trace_names_size ();
+
+  if (argc > 1)
+  {
+    if (argc > 3)
+      return rtems_trace_buffering_wrong_number_of_args ();
+
+    if (argv[1][0] == '+')
+    {
+      if (argc > 2)
+        return rtems_trace_buffering_wrong_number_of_args ();
+      end = strtoul (argv[1] + 1, 0, 0);
+      if (end == 0)
+      {
+        printf("error: invalid number of lines\n");
+        return 1;
+      }
+      ++end;
+    }
+    else
+    {
+      start = strtoul (argv[1], 0, 0);
+      if (start >= records)
+      {
+        printf ("error: start record out of range (max %" PRIu32 ")\n", records);
+        return 1;
+      }
+    }
+
+    end += start;
+
+    if (argc == 3)
+    {
+      if (argv[2][0] == '+')
+      {
+        end = strtoul (argv[2] + 1, 0, 0);
+        if (end == 0)
+        {
+          printf("error: invalid number of lines\n");
+          return 1;
+        }
+        end += start + 1;
+      }
+      else
+      {
+        end = strtoul (argv[2], 0, 0);
+        if (end < start)
+        {
+          printf ("error: end record before start\n");
+          return 1;
+        }
+        else if (end > records)
+        {
+          printf ("error: end record out of range (max %" PRIu32 ")\n", records);
+        }
+      }
+    }
+  }
+
+  rtems_trace_buffering_banner ("trace");
+
+  if (!rtems_trace_buffering_finished ())
+  {
+    printf("tracing still running\n");
+    return 0;
+  }
+
+  printf(" Trace buffer: %p\n", trace_buffer);
+  printf(" Words traced: %" PRIu32 "\n", records);
+  printf("       Traces: %" PRIu32 "\n", traces);
+
+  count = 0;
+  r = 0;
+
+  while ((r < records) && (count < end))
+  {
+    const uint32_t header = trace_buffer[r];
+    const uint32_t func_index = header & 0xffff;
+    const uint32_t len = (header >> 16) & 0x0fff;
+    const uint32_t task_id = trace_buffer[r + 1];
+    const uint32_t task_status = trace_buffer[r + 2];
+    const uint64_t when = (((uint64_t) trace_buffer[r + 4]) << 32) | trace_buffer[r + 5];
+    const uint8_t* argv = (uint8_t*) &trace_buffer[r + 6];
+    const bool     ret = (header & (1 << 30)) == 0 ? false : true;
+    const bool     irq = (header & (1 << 31)) == 0 ? false : true;
+
+    if (count > start)
+    {
+      const rtems_trace_sig* sig = rtems_trace_signatures (func_index);
+
+      rtems_trace_buffering_print_timestamp (when);
+      printf (" %10" PRIu32 " %c%08" PRIx32 " [%3" PRIu32 "/%3" PRIu32 "] %c %s",
+              (uint32_t) (when - last_sample),
+              irq ? '*' : ' ', task_id, (task_status >> 8) & 0xff, task_status & 0xff,
+              ret ? '<' : '>', rtems_trace_names (func_index));
+
+      if (sig->argc)
+      {
+        if (ret)
+        {
+          if (sig->args[0].size)
+            printf(" => ");
+          rtems_trace_buffering_print_arg (&sig->args[0], argv);
+        }
+        else
+        {
+          size_t a;
+          printf("(");
+          for (a = 1; a < sig->argc; ++a)
+          {
+            if (a > 1)
+              printf (", ");
+            rtems_trace_buffering_print_arg (&sig->args[a], argv);
+            argv += sig->args[a].size;
+          }
+          printf(")");
+        }
+      }
+
+      printf("\n");
+    }
+
+    r += ((len - 1) / sizeof (uint32_t)) + 1;
+    last_sample = when;
+    ++count;
+  }
+
+  return 0;
+}
+
+static ssize_t
+rtems_trace_buffering_file_write (int out, const void* vbuffer, ssize_t length)
+{
+  const uint8_t* buffer = vbuffer;
+  while (length)
+  {
+    ssize_t w = write (out, buffer, length);
+    if (w < 0)
+    {
+      printf ("error: write failed: %s\n", strerror(errno));
+      return false;
+    }
+    if (w == 0)
+    {
+      printf ("error: write failed: EOF\n");
+      return false;
+    }
+
+    length -= w;
+    buffer += w;
+  }
+  return true;
+}
+
+static int
+rtems_trace_buffering_shell_save (int argc, char *argv[])
+{
+  uint32_t* trace_buffer;
+  uint32_t  records;
+  uint32_t  traces;
+  uint8_t*  buffer;
+  size_t    length;
+  uint32_t  r;
+  int       out;
+  uint8_t*  buf;
+  uint8_t*  in;
+
+  if (argc != 2)
+    return rtems_trace_buffering_wrong_number_of_args ();
+
+  if (!rtems_trace_buffering_present ())
+    return rtems_trace_buffering_no_trace_buffer_code ();
+
+  rtems_trace_buffering_banner ("trace");
+
+  if (!rtems_trace_buffering_finished ())
+  {
+    printf("tracing still running\n");
+    return 0;
+  }
+
+  trace_buffer = rtems_trace_buffering_buffer ();
+  records = rtems_trace_buffering_buffer_in ();
+  traces = rtems_trace_names_size ();
+
+  printf("   Trace File: %s\n", argv[1]);
+  printf(" Trace buffer: %p\n", trace_buffer);
+  printf(" Words traced: %" PRIu32 "\n", records);
+  printf("       Traces: %" PRIu32 "\n", traces);
+
+  out = open (argv[1], O_WRONLY | O_TRUNC | O_CREAT,
+              S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+  if (out < 0)
+  {
+    printf ("error: opening file: %s: %s\n", argv[1], strerror(errno));
+    return 1;
+  }
+
+  #define SAVE_BUF_SIZE (1024)
+
+  buf = malloc(SAVE_BUF_SIZE);
+  if (!buf)
+  {
+    close (out);
+    printf ("error: no memory\n");
+  }
+
+  memset (buf, 0, SAVE_BUF_SIZE);
+
+  in = buf;
+
+  /*
+   * Header label.
+   */
+  memcpy (in, "RTEMS-TRACE", sizeof("RTEMS-TRACE"));
+  in += 12;
+
+  /*
+   * Endian detection.
+   */
+  *((uint32_t*) in) = 0x11223344;
+  in += sizeof(uint32_t);
+
+  /*
+   * Number of traces.
+   */
+  *((uint32_t*) in) = traces;
+  in += sizeof(uint32_t);
+
+  /*
+   * Write it.
+   */
+  if (!rtems_trace_buffering_file_write (out, buf, in - buf))
+  {
+    free (buf);
+    close (out);
+    return 1;
+  }
+
+  /*
+   * The trace names.
+   */
+  for (r = 0; r < traces; ++r)
+  {
+    const char* name = rtems_trace_names (r);
+    if (!rtems_trace_buffering_file_write (out, name, strlen (name) + 1))
+    {
+      free (buf);
+      close (out);
+      return 1;
+    }
+  }
+
+  /*
+   * The trace signatures.
+   */
+  for (r = 0; r < traces; ++r)
+  {
+    const rtems_trace_sig* sig = rtems_trace_signatures (r);
+    size_t                 s;
+
+    in = buf;
+
+    memcpy (in, &sig->argc, sizeof (sig->argc));
+    in += sizeof(uint32_t);
+
+    for (s = 0; s < sig->argc; ++s)
+    {
+      const rtems_trace_sig_arg* arg = &sig->args[s];
+      size_t                     arg_len = strlen (arg->type) + 1;
+
+      if ((in - buf) > SAVE_BUF_SIZE)
+      {
+        printf ("error: save temp buffer to small\n");
+        free (buf);
+        close (out);
+        return 1;
+      }
+
+      memcpy (in, &arg->size, sizeof (arg->size));
+      in += sizeof(uint32_t);
+      memcpy (in, arg->type, arg_len);
+      in += arg_len;
+    }
+
+    if (!rtems_trace_buffering_file_write (out, buf, in - buf))
+    {
+      free (buf);
+      close (out);
+      return 1;
+    }
+  }
+
+  free (buf);
+
+  buffer = (uint8_t*) trace_buffer;
+  length = records * sizeof (uint32_t);
+
+  while (length)
+  {
+    ssize_t w = write (out, buffer, length);
+    if (w < 0)
+    {
+      printf ("error: write failed: %s\n", strerror(errno));
+      close (out);
+      return 1;
+    }
+    if (w == 0)
+    {
+      printf ("error: write failed: EOF\n");
+      close (out);
+      return 1;
+    }
+
+    length -= w;
+    buffer += w;
+  }
+
+  close (out);
+
+  return 0;
+}
+
+static void
+rtems_trace_buffering_shell_usage (const char* arg)
+{
+  printf ("%s: Trace Buffer Help\n", arg);
+  printf ("  %s [-hl] <command>\n", arg);
+  printf ("   where:\n");
+  printf ("     command: The TBG subcommand. See -l for a list plus help.\n");
+  printf ("     -h:      This help\n");
+  printf ("     -l:      The command list.\n");
+}
+
+static const rtems_trace_buffering_shell_cmd_t table[] =
+{
+  {
+    "status",
+    rtems_trace_buffering_shell_status,
+    "                       : Show the current status"
+  },
+  {
+    "funcs",
+    rtems_trace_buffering_shell_funcs,
+    "                       : List the trace functions"
+  },
+  {
+    "start",
+    rtems_trace_buffering_shell_start,
+    "                       : Start or restart tracing"
+  },
+  {
+    "stop",
+    rtems_trace_buffering_shell_stop,
+    "                       : Stop tracing"
+  },
+  {
+    "resume",
+    rtems_trace_buffering_shell_resume,
+    "                       : Resume tracing."
+  },
+  {
+    "trace",
+    rtems_trace_buffering_shell_trace,
+    " [start] [end/+length] : List the current trace records"
+  },
+  {
+    "save",
+    rtems_trace_buffering_shell_save,
+    " file                  : Save the trace buffer to a file"
+  },
+};
+
+#define RTEMS_TRACE_BUFFERING_COMMANDS \
+  (sizeof (table) / sizeof (const rtems_trace_buffering_shell_cmd_t))
+
+static int
+rtems_shell_main_rtrace (int argc, char* argv[])
+{
+  int    arg;
+  size_t t;
+
+  for (arg = 1; arg < argc; arg++)
+  {
+    if (argv[arg][0] != '-')
+      break;
+
+    switch (argv[arg][1])
+    {
+      case 'h':
+        rtems_trace_buffering_shell_usage (argv[0]);
+        return 0;
+      case 'l':
+        printf ("%s: commands are:\n", argv[0]);
+        for (t = 0; t < RTEMS_TRACE_BUFFERING_COMMANDS; ++t)
+          printf ("  %-7s %s\n", table[t].name, table[t].help);
+        return 0;
+      default:
+        printf ("error: unknown option: %s\n", argv[arg]);
+        return 1;
+    }
+  }
+
+  if ((argc - arg) < 1)
+    printf ("error: you need to provide a command, try %s -h\n", argv[0]);
+  else
+  {
+    for (t = 0; t < RTEMS_TRACE_BUFFERING_COMMANDS; ++t)
+    {
+      if (strncmp (argv[arg], table[t].name, strlen (argv[arg])) == 0)
+      {
+        int r = table[t].handler (argc - arg, argv + 1);
+        return r;
+      }
+    }
+    printf ("error: command not found: %s (try -h)\n", argv[arg]);
+  }
+
+  return 1;
+}
+
+rtems_shell_cmd_t rtems_shell_RTRACE_Command = {
+  "rtrace",                      /* name */
+  "rtrace [-l]",                 /* usage */
+  "misc",                        /* topic */
+  rtems_shell_main_rtrace,       /* command */
+  NULL,                          /* alias */
+  NULL                           /* next */
+};
diff --git a/cpukit/libmisc/shell/shellconfig.h b/cpukit/libmisc/shell/shellconfig.h
index 2e52a92c2e..9023a60274 100644
--- a/cpukit/libmisc/shell/shellconfig.h
+++ b/cpukit/libmisc/shell/shellconfig.h
@@ -87,6 +87,7 @@ extern rtems_shell_cmd_t rtems_shell_PERIODUSE_Command;
 extern rtems_shell_cmd_t rtems_shell_PROFREPORT_Command;
 extern rtems_shell_cmd_t rtems_shell_WKSPACE_INFO_Command;
 extern rtems_shell_cmd_t rtems_shell_MALLOC_INFO_Command;
+extern rtems_shell_cmd_t rtems_shell_RTRACE_Command;
 #if RTEMS_NETWORKING
   extern rtems_shell_cmd_t rtems_shell_IFCONFIG_Command;
   extern rtems_shell_cmd_t rtems_shell_ROUTE_Command;
@@ -464,6 +465,15 @@ extern rtems_shell_alias_t * const rtems_shell_Initial_aliases[];
     #endif
 
     /*
+     *  Tracing family commands
+     */
+    #if (defined(CONFIGURE_SHELL_COMMANDS_ALL) && \
+         !defined(CONFIGURE_SHELL_NO_COMMAND_RTRACE)) || \
+        defined(CONFIGURE_SHELL_COMMAND_RTRACE)
+      &rtems_shell_RTRACE_Command,
+    #endif
+
+    /*
      *  Network related commands
      */
     #if RTEMS_NETWORKING
diff --git a/cpukit/preinstall.am b/cpukit/preinstall.am
index 63e39031c0..40273bb71d 100644
--- a/cpukit/preinstall.am
+++ b/cpukit/preinstall.am
@@ -423,6 +423,15 @@ $(PROJECT_INCLUDE)/rtems/captureimpl.h: libmisc/capture/captureimpl.h $(PROJECT_
 	$(INSTALL_DATA) $< $(PROJECT_INCLUDE)/rtems/captureimpl.h
 PREINSTALL_FILES += $(PROJECT_INCLUDE)/rtems/captureimpl.h
 
+$(PROJECT_INCLUDE)/rtems/trace/$(dirstamp):
+	@$(MKDIR_P) $(PROJECT_INCLUDE)/rtems/trace
+	@: > $(PROJECT_INCLUDE)/rtems/trace/$(dirstamp)
+PREINSTALL_DIRS += $(PROJECT_INCLUDE)/rtems/trace/$(dirstamp)
+
+$(PROJECT_INCLUDE)/rtems/trace/rtems-trace-buffer-vars.h: libmisc/capture/rtems-trace-buffer-vars.h $(PROJECT_INCLUDE)/rtems/trace/$(dirstamp)
+	$(INSTALL_DATA) $< $(PROJECT_INCLUDE)/rtems/trace/rtems-trace-buffer-vars.h
+PREINSTALL_FILES += $(PROJECT_INCLUDE)/rtems/trace/rtems-trace-buffer-vars.h
+
 $(PROJECT_INCLUDE)/rtems/cpuuse.h: libmisc/cpuuse/cpuuse.h $(PROJECT_INCLUDE)/rtems/$(dirstamp)
 	$(INSTALL_DATA) $< $(PROJECT_INCLUDE)/rtems/cpuuse.h
 PREINSTALL_FILES += $(PROJECT_INCLUDE)/rtems/cpuuse.h