Add low level event recording support

Add low level event recording infrastructure for system and user
defined events.  The infrastructure is able to record high frequency
events such as

 * SMP lock acquire/release,
 * interrupt entry/exit,
 * thread switches,
 * UMA zone allocate/free, and
 * Ethernet packet input/output, etc.

It allows post-mortem analysis in fatal error handlers, e.g. the last
events are in the record buffer, the newest event overwrites the oldest
event.  It is possible to detect record buffer overflows for consumers
that expect a continuous stream of events, e.g. to display the system
state in real-time.

The implementation supports high-end SMP machines (more than 1GHz
processor frequency, more than four processors).

Add a new API instead. The implementation uses per-processor data
structures and no atomic read-modify-write operations.  It is uses
per-processor ring buffers to record the events.

The CPU counter is used to get the time of events. It is combined with
periodic uptime events to synchronize it with CLOCK_REALTIME.

The existing capture engine tries to solve this problem also, but its
performance is not good enough for high-end production systems.  The
main issues are the variable-size buffers and the use of SMP locks for
synchronization.  To fix this, the API would change significantly.

Update #3665.

6 files changed, 826 insertions, 0 deletions

diff --git a/cpukit/include/rtems/confdefs.h b/cpukit/include/rtems/confdefs.h
index 45e70313fb..89b47050d6 100644
--- a/cpukit/include/rtems/confdefs.h
+++ b/cpukit/include/rtems/confdefs.h
@@ -2123,6 +2123,10 @@ extern rtems_initialization_tasks_table Initialization_tasks[];
     defined(CONFIGURE_STACK_CHECKER_ENABLED) || \
     (defined(RTEMS_NEWLIB) && !defined(CONFIGURE_DISABLE_NEWLIB_REENTRANCY))
   static const rtems_extensions_table Configuration_Initial_Extensions[] = {
+    #if CONFIGURE_RECORD_PER_PROCESSOR_ITEMS > 0 && \
+      defined(CONFIGURE_RECORD_EXTENSIONS_ENABLED)
+      RECORD_EXTENSION,
+    #endif
     #if !defined(CONFIGURE_DISABLE_NEWLIB_REENTRANCY)
       RTEMS_NEWLIB_EXTENSION,
     #endif
@@ -2965,6 +2969,33 @@ struct _reent *__getreent(void)
       _CONFIGURE_MAXIMUM_PROCESSORS,
     #endif
   };
+
+  #if CONFIGURE_RECORD_PER_PROCESSOR_ITEMS > 0
+    #include <rtems/record.h>
+
+    #if (CONFIGURE_RECORD_PER_PROCESSOR_ITEMS & (CONFIGURE_RECORD_PER_PROCESSOR_ITEMS - 1)) != 0
+      #error "CONFIGURE_RECORD_PER_PROCESSOR_ITEMS must be a power of two"
+    #endif
+
+    #if CONFIGURE_RECORD_PER_PROCESSOR_ITEMS < 16
+      #error "CONFIGURE_RECORD_PER_PROCESSOR_ITEMS must be at least 16"
+    #endif
+
+    const unsigned int _Record_Item_count = CONFIGURE_RECORD_PER_PROCESSOR_ITEMS;
+
+    struct Record_Configured_control {
+      Record_Control    Control;
+      rtems_record_item Items[ CONFIGURE_RECORD_PER_PROCESSOR_ITEMS ];
+    };
+
+    PER_CPU_DATA_ITEM( Record_Configured_control, _Record_Per_CPU );
+
+    RTEMS_SYSINIT_ITEM(
+      _Record_Initialize,
+      RTEMS_SYSINIT_RECORD,
+      RTEMS_SYSINIT_ORDER_MIDDLE
+    );
+  #endif
 #endif
 
 #if defined(RTEMS_SMP)
diff --git a/cpukit/include/rtems/record.h b/cpukit/include/rtems/record.h
new file mode 100644
index 0000000000..cfadae79d6
--- /dev/null
+++ b/cpukit/include/rtems/record.h
@@ -0,0 +1,334 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (C) 2018, 2019 embedded brains GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTEMS_RECORD_H
+#define _RTEMS_RECORD_H
+
+#include "recorddata.h"
+
+#include <rtems/score/atomic.h>
+#include <rtems/score/cpu.h>
+#include <rtems/score/percpudata.h>
+#include <rtems/score/watchdog.h>
+#include <rtems/rtems/intr.h>
+#include <rtems/rtems/tasks.h>
+#include <rtems/counter.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+struct Record_Control {
+  Atomic_Uint       head;
+  unsigned int      tail;
+  unsigned int      mask;
+  Watchdog_Control  Watchdog;
+  rtems_record_item Header[ 3 ];
+  rtems_record_item Items[ RTEMS_ZERO_LENGTH_ARRAY ];
+};
+
+typedef struct Record_Control Record_Control;
+
+typedef RTEMS_ALIGNED( CPU_CACHE_LINE_BYTES )
+  struct Record_Configured_control Record_Configured_control;
+
+typedef struct {
+  Record_Control        *control;
+  unsigned int           head;
+  uint32_t               now;
+  rtems_interrupt_level  level;
+} rtems_record_context;
+
+PER_CPU_DATA_ITEM_DECLARE( Record_Configured_control, _Record_Per_CPU );
+
+extern const unsigned int _Record_Item_count;
+
+void _Record_Initialize( void );
+
+bool _Record_Thread_create(
+  struct _Thread_Control *executing,
+  struct _Thread_Control *created
+);
+
+void _Record_Thread_start(
+  struct _Thread_Control *executing,
+  struct _Thread_Control *started
+);
+
+void _Record_Thread_restart(
+  struct _Thread_Control *executing,
+  struct _Thread_Control *restarted
+);
+
+void _Record_Thread_delete(
+  struct _Thread_Control *executing,
+  struct _Thread_Control *deleted
+);
+
+void _Record_Thread_switch(
+  struct _Thread_Control *executing,
+  struct _Thread_Control *heir
+);
+
+void _Record_Thread_begin( struct _Thread_Control *executing );
+
+void _Record_Thread_exitted( struct _Thread_Control *executing );
+
+void _Record_Thread_terminate(
+  struct _Thread_Control *executing
+);
+
+#define RECORD_EXTENSION \
+  { \
+    _Record_Thread_create, \
+    _Record_Thread_start, \
+    _Record_Thread_restart, \
+    _Record_Thread_delete, \
+    _Record_Thread_switch, \
+    _Record_Thread_begin, \
+    _Record_Thread_exitted, \
+    NULL, \
+    _Record_Thread_terminate \
+  }
+
+RTEMS_INLINE_ROUTINE unsigned int _Record_Index(
+  const Record_Control *control,
+  unsigned int          index
+)
+{
+  return index & control->mask;
+}
+
+RTEMS_INLINE_ROUTINE unsigned int _Record_Head( const Record_Control *control )
+{
+  return _Atomic_Load_uint( &control->head, ATOMIC_ORDER_RELAXED );
+}
+
+RTEMS_INLINE_ROUTINE unsigned int _Record_Tail( const Record_Control *control )
+{
+  return control->tail;
+}
+
+RTEMS_INLINE_ROUTINE bool _Record_Is_overflow(
+  const Record_Control *control,
+  unsigned int          tail,
+  unsigned int          head
+)
+{
+  return head - tail >= control->mask + 1U;
+}
+
+RTEMS_INLINE_ROUTINE unsigned int _Record_Capacity(
+  const Record_Control *control,
+  unsigned int          tail,
+  unsigned int          head
+)
+{
+  return ( tail - head - 1U ) & control->mask;
+}
+
+RTEMS_INLINE_ROUTINE rtems_counter_ticks _Record_Now( void )
+{
+  return rtems_counter_read();
+}
+
+typedef struct RTEMS_PACKED {
+  uint32_t format;
+  uint32_t magic;
+  rtems_record_item Version;
+  rtems_record_item Processor_maximum;
+  rtems_record_item Count;
+  rtems_record_item Frequency;
+} Record_Stream_header;
+
+void _Record_Stream_header_initialize( Record_Stream_header *header );
+
+/**
+ * @addtogroup RTEMSRecord
+ *
+ * @{
+ */
+
+/**
+ * @brief Prepares to add and commit record items.
+ *
+ * This function disables interrupts.
+ *
+ * @param context The record context which must be used for the following
+ *   rtems_record_add() and rtems_record_commit() calls.  The record context
+ *   may have an arbitrary content at function entry.
+ *
+ * @see rtems_record_produce().
+ */
+RTEMS_INLINE_ROUTINE void rtems_record_prepare( rtems_record_context *context )
+{
+  rtems_interrupt_level  level;
+  const Per_CPU_Control *cpu_self;
+  Record_Control        *control;
+  unsigned int           head;
+
+  rtems_interrupt_local_disable( level );
+  context->now = RTEMS_RECORD_TIME_EVENT( _Record_Now(), 0 );
+  context->level = level;
+  cpu_self = _Per_CPU_Get();
+  control = cpu_self->record;
+  context->control = control;
+  head = _Record_Head( control );
+  context->head = head;
+}
+
+/**
+ * @brief Adds a record item.
+ *
+ * @param context The record context initialized via rtems_record_prepare().
+ * @param event The record event without a time stamp for the item.
+ * @param data The record data for the item.
+ */
+RTEMS_INLINE_ROUTINE void rtems_record_add(
+  rtems_record_context *context,
+  rtems_record_event    event,
+  rtems_record_data     data
+)
+{
+  Record_Control    *control;
+  rtems_record_item *item;
+  unsigned int       head;
+
+  control = context->control;
+  head = context->head;
+  item = &control->Items[ _Record_Index( control, head ) ];
+  context->head = head + 1;
+
+  item->event = context->now | event;
+  item->data = data;
+}
+
+/**
+ * @brief Commits a set of record items.
+ *
+ * @param context The record context initialized via rtems_record_prepare().
+ */
+RTEMS_INLINE_ROUTINE void rtems_record_commit( rtems_record_context *context )
+{
+  _Atomic_Store_uint(
+    &context->control->head,
+    context->head,
+    ATOMIC_ORDER_RELEASE
+  );
+  rtems_interrupt_local_enable( context->level );
+}
+
+/**
+ * @brief Produces a record item.
+ *
+ * @param event The record event without a time stamp for the item.
+ * @param data The record data for the item.
+ */
+void rtems_record_produce( rtems_record_event event, rtems_record_data data );
+
+/**
+ * @brief Produces two record items.
+ *
+ * @param event_0 The record event without a time stamp for the first item.
+ * @param data_0 The record data for the first item.
+ * @param event_1 The record event without a time stamp for the second item.
+ * @param data_1 The record data for the second item.
+ */
+void rtems_record_produce_2(
+  rtems_record_event event_0,
+  rtems_record_data  data_0,
+  rtems_record_event event_1,
+  rtems_record_data  data_1
+);
+
+/**
+ * @brief Produces n record items.
+ *
+ * @param item The record items without a time stamps.
+ * @param n The count of record items.
+ */
+void rtems_record_produce_n(
+  const rtems_record_item *items,
+  size_t                   n
+);
+
+typedef void ( *rtems_record_drain_visitor )(
+  const rtems_record_item *items,
+  size_t                   count,
+  void                    *arg
+);
+
+/**
+ * @brief Drains the record items on all processors.
+ *
+ * Calls the visitor function for each drained item set.
+ *
+ * @param visitor The visitor function.
+ * @param arg The argument for the visitor function.
+ */
+void rtems_record_drain( rtems_record_drain_visitor visitor, void *arg );
+
+/**
+ * @brief Drains the record items on all processors an writes them to the file
+ * descriptor.
+ *
+ * @param fd The file descriptor.
+ * @param written Set to true if items were written to the file descriptor,
+ *   otherwise set to false.
+ *
+ * @retval The bytes written to the file descriptor.
+ */
+ssize_t rtems_record_writev( int fd, bool *written );
+
+/**
+ * @brief Runs a record TCP server loop.
+ *
+ * @param port The TCP port to listen in host byte order.
+ * @param period The drain period in clock ticks.
+ */
+void rtems_record_server( uint16_t port, rtems_interval period );
+
+/**
+ * @brief Starts a record TCP server task.
+ *
+ * @param priority The task priority.
+ * @param port The TCP port to listen in host byte order.
+ * @param period The drain period in clock ticks.
+ */
+rtems_status_code rtems_record_start_server(
+  rtems_task_priority priority,
+  uint16_t            port,
+  rtems_interval      period
+);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* _RTEMS_RECORD_H */
diff --git a/cpukit/include/rtems/recordclient.h b/cpukit/include/rtems/recordclient.h
new file mode 100644
index 0000000000..61ef96791c
--- /dev/null
+++ b/cpukit/include/rtems/recordclient.h
@@ -0,0 +1,139 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (C) 2018, 2019 embedded brains GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This file must be compatible to general purpose POSIX system, e.g. Linux,
+ * FreeBSD.  It may be used for utility programs.
+ */
+
+#ifndef _RTEMS_RECORDCLIENT_H
+#define _RTEMS_RECORDCLIENT_H
+
+#include "recorddata.h"
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/**
+ * @addtogroup RTEMSRecord
+ *
+ * @{
+ */
+
+#define RTEMS_RECORD_CLIENT_MAXIMUM_CPU_COUNT 32
+
+typedef enum {
+  RTEMS_RECORD_CLIENT_SUCCESS,
+  RTEMS_RECORD_CLIENT_ERROR_INVALID_MAGIC,
+  RTEMS_RECORD_CLIENT_ERROR_UNKNOWN_FORMAT,
+  RTEMS_RECORD_CLIENT_ERROR_UNSUPPORTED_VERSION,
+  RTEMS_RECORD_CLIENT_ERROR_UNSUPPORTED_CPU
+} rtems_record_client_status;
+
+typedef rtems_record_client_status ( *rtems_record_client_handler )(
+  uint32_t            seconds,
+  uint32_t            nanoseconds,
+  uint32_t            cpu,
+  rtems_record_event  event,
+  uint64_t            data,
+  void               *arg
+);
+
+typedef struct {
+  struct {
+    uint64_t bt;
+    uint32_t time_at_bt;
+    uint32_t time_last;
+    uint32_t time_accumulated;
+  } uptime;
+  uint32_t tail[ 2 ];
+  uint32_t head[ 2 ];
+  size_t index;
+} rtems_record_client_per_cpu;
+
+typedef struct rtems_record_client_context {
+  uint64_t to_bt_scaler;
+  rtems_record_client_per_cpu per_cpu[ RTEMS_RECORD_CLIENT_MAXIMUM_CPU_COUNT ];
+  uint64_t data;
+  uint32_t cpu;
+  uint32_t event;
+  uint32_t count;
+  union {
+    rtems_record_item_32 format_32;
+    rtems_record_item_64 format_64;
+  } item;
+  size_t todo;
+  void *pos;
+  rtems_record_client_status ( *consume )(
+    struct rtems_record_client_context *,
+    const void *,
+    size_t
+  );
+  rtems_record_client_handler handler;
+  void *handler_arg;
+  uint32_t header[ 2 ];
+} rtems_record_client_context;
+
+/**
+ * @brief Initializes a record client.
+ *
+ * The record client consumes a record item stream produces by the record
+ * server.
+ *
+ * @param ctx The record client context to initialize.
+ * @param handler The handler is invoked for each received record item.
+ * @param arg The handler argument.
+ */
+void rtems_record_client_init(
+  rtems_record_client_context *ctx,
+  rtems_record_client_handler  handler,
+  void                        *arg
+);
+
+/**
+ * @brief Runs the record client to consume new stream data.
+ *
+ * @param ctx The record client context.
+ * @param buf The buffer with new stream data.
+ * @param n The size of the buffer.
+ */
+rtems_record_client_status rtems_record_client_run(
+  rtems_record_client_context *ctx,
+  const void                  *buf,
+  size_t                       n
+);
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* _RTEMS_RECORDCLIENT_H */
diff --git a/cpukit/include/rtems/recorddata.h b/cpukit/include/rtems/recorddata.h
new file mode 100644
index 0000000000..5879980039
--- /dev/null
+++ b/cpukit/include/rtems/recorddata.h
@@ -0,0 +1,317 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (C) 2018, 2019 embedded brains GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This file must be compatible to general purpose POSIX system, e.g. Linux,
+ * FreeBSD.  It may be used for utility programs.
+ */
+
+#ifndef _RTEMS_RECORDDATA_H
+#define _RTEMS_RECORDDATA_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/**
+ * @defgroup RTEMSRecord Event Recording
+ *
+ * @brief Low-level event recording support.
+ *
+ * @{
+ */
+
+/**
+ * @brief The record version.
+ *
+ * The record version reflects the record event definitions.  It is reported by
+ * the RTEMS_RECORD_VERSION event.
+ */
+#define RTEMS_RECORD_THE_VERSION 1
+
+/**
+ * @brief The items are in 32-bit little-endian format.
+ */
+#define RTEMS_RECORD_FORMAT_LE_32 0x11111111
+
+/**
+ * @brief The items are in 64-bit little-endian format.
+ */
+#define RTEMS_RECORD_FORMAT_LE_64 0x22222222
+
+/**
+ * @brief The items are in 32-bit big-endian format.
+ */
+#define RTEMS_RECORD_FORMAT_BE_32 0x33333333
+
+/**
+ * @brief The items are in 64-bit big-endian format.
+ */
+#define RTEMS_RECORD_FORMAT_BE_64 0x44444444
+
+/**
+ * @brief Magic number to identify a record item stream.
+ *
+ * This is a random number.
+ */
+#define RTEMS_RECORD_MAGIC 0x82e14ec1
+
+/**
+ * @brief The record events.
+ */
+typedef enum {
+  /* There are 512 events reserved for the system */
+  RTEMS_RECORD_EMPTY,
+  RTEMS_RECORD_VERSION,
+
+  /*
+   * Keep the following system events in lexicographical order, increment
+   * RTEMS_RECORD_THE_VERSION after each change.
+   */
+  RTEMS_RECORD_ACCEPT,
+  RTEMS_RECORD_BIND,
+  RTEMS_RECORD_BUFFER,
+  RTEMS_RECORD_CHOWN,
+  RTEMS_RECORD_CLOSE,
+  RTEMS_RECORD_CONNECT,
+  RTEMS_RECORD_COUNT,
+  RTEMS_RECORD_ETHER_INPUT,
+  RTEMS_RECORD_ETHER_OUTPUT,
+  RTEMS_RECORD_FCHMOD,
+  RTEMS_RECORD_FCNTL,
+  RTEMS_RECORD_FDATASYNC,
+  RTEMS_RECORD_FREQUENCY,
+  RTEMS_RECORD_FSTAT,
+  RTEMS_RECORD_FSYNC,
+  RTEMS_RECORD_FTRUNCATE,
+  RTEMS_RECORD_GIT_HASH,
+  RTEMS_RECORD_HEAD,
+  RTEMS_RECORD_HEAP_ALLOC,
+  RTEMS_RECORD_HEAP_FREE,
+  RTEMS_RECORD_HEAP_SIZE,
+  RTEMS_RECORD_HEAP_USAGE,
+  RTEMS_RECORD_INTERUPT_BEGIN,
+  RTEMS_RECORD_INTERUPT_END,
+  RTEMS_RECORD_INTERUPT_INSTALL,
+  RTEMS_RECORD_INTERUPT_REMOVE,
+  RTEMS_RECORD_IOCTL,
+  RTEMS_RECORD_IP6_INPUT,
+  RTEMS_RECORD_IP6_OUTPUT,
+  RTEMS_RECORD_IP_INPUT,
+  RTEMS_RECORD_IP_OUTPUT,
+  RTEMS_RECORD_KEVENT,
+  RTEMS_RECORD_KQUEUE,
+  RTEMS_RECORD_LENGTH,
+  RTEMS_RECORD_LINK,
+  RTEMS_RECORD_LSEEK,
+  RTEMS_RECORD_MKNOD,
+  RTEMS_RECORD_MMAP,
+  RTEMS_RECORD_MOUNT,
+  RTEMS_RECORD_OPEN,
+  RTEMS_RECORD_OVERFLOW,
+  RTEMS_RECORD_PAGE_ALLOC,
+  RTEMS_RECORD_PAGE_FREE,
+  RTEMS_RECORD_POLL,
+  RTEMS_RECORD_PROCESSOR,
+  RTEMS_RECORD_PROCESSOR_MAXIMUM,
+  RTEMS_RECORD_READ,
+  RTEMS_RECORD_READLINK,
+  RTEMS_RECORD_READV,
+  RTEMS_RECORD_RECV,
+  RTEMS_RECORD_RECVFROM,
+  RTEMS_RECORD_RECVMSG,
+  RTEMS_RECORD_RENAME,
+  RTEMS_RECORD_RTEMS_BARRIER_CREATE,
+  RTEMS_RECORD_RTEMS_BARRIER_DELETE,
+  RTEMS_RECORD_RTEMS_BARRIER_RELEASE,
+  RTEMS_RECORD_RTEMS_BARRIER_WAIT,
+  RTEMS_RECORD_RTEMS_EVENT_RECEIVE,
+  RTEMS_RECORD_RTEMS_EVENT_SEND,
+  RTEMS_RECORD_RTEMS_EVENT_SYSTEM_RECEIVE,
+  RTEMS_RECORD_RTEMS_EVENT_SYSTEM_SEND,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_BROADCAST,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_CREATE,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_DELETE,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_FLUSH,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_RECEIVE,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_SEND,
+  RTEMS_RECORD_RTEMS_MESSAGE_QUEUE_URGENT,
+  RTEMS_RECORD_RTEMS_PARTITION_CREATE,
+  RTEMS_RECORD_RTEMS_PARTITION_DELETE,
+  RTEMS_RECORD_RTEMS_PARTITION_GET_BUFFER,
+  RTEMS_RECORD_RTEMS_PARTITION_RETURN_BUFFER,
+  RTEMS_RECORD_RTEMS_RATE_MONOTONIC_CANCEL,
+  RTEMS_RECORD_RTEMS_RATE_MONOTONIC_CREATE,
+  RTEMS_RECORD_RTEMS_RATE_MONOTONIC_DELETE,
+  RTEMS_RECORD_RTEMS_RATE_MONOTONIC_PERIOD,
+  RTEMS_RECORD_RTEMS_SEMAPHORE_CREATE,
+  RTEMS_RECORD_RTEMS_SEMAPHORE_DELETE,
+  RTEMS_RECORD_RTEMS_SEMAPHORE_FLUSH,
+  RTEMS_RECORD_RTEMS_SEMAPHORE_OBTAIN,
+  RTEMS_RECORD_RTEMS_SEMAPHORE_RELEASE,
+  RTEMS_RECORD_RTEMS_TIMER_CANCEL,
+  RTEMS_RECORD_RTEMS_TIMER_CREATE,
+  RTEMS_RECORD_RTEMS_TIMER_DELETE,
+  RTEMS_RECORD_RTEMS_TIMER_FIRE_AFTER,
+  RTEMS_RECORD_RTEMS_TIMER_FIRE_WHEN,
+  RTEMS_RECORD_RTEMS_TIMER_RESET,
+  RTEMS_RECORD_RTEMS_TIMER_SERVER_FIRE_AFTER,
+  RTEMS_RECORD_RTEMS_TIMER_SERVER_FIRE_WHEN,
+  RTEMS_RECORD_SELECT,
+  RTEMS_RECORD_SEND,
+  RTEMS_RECORD_SENDMSG,
+  RTEMS_RECORD_SENDTO,
+  RTEMS_RECORD_SOCKET,
+  RTEMS_RECORD_STATVFS,
+  RTEMS_RECORD_SYMLINK,
+  RTEMS_RECORD_TAIL,
+  RTEMS_RECORD_TCP_INPUT,
+  RTEMS_RECORD_TCP_OUTPUT,
+  RTEMS_RECORD_THREAD_BEGIN,
+  RTEMS_RECORD_THREAD_CREATE,
+  RTEMS_RECORD_THREAD_DELETE,
+  RTEMS_RECORD_THREAD_EXIT,
+  RTEMS_RECORD_THREAD_EXITTED,
+  RTEMS_RECORD_THREAD_ID,
+  RTEMS_RECORD_THREAD_PRIO_CURRENT_HIGH,
+  RTEMS_RECORD_THREAD_PRIO_CURRENT_LOW,
+  RTEMS_RECORD_THREAD_PRIO_REAL_HIGH,
+  RTEMS_RECORD_THREAD_PRIO_REAL_LOW,
+  RTEMS_RECORD_THREAD_QUEUE_ENQUEUE,
+  RTEMS_RECORD_THREAD_QUEUE_ENQUEUE_STICKY,
+  RTEMS_RECORD_THREAD_QUEUE_EXTRACT,
+  RTEMS_RECORD_THREAD_QUEUE_SURRENDER,
+  RTEMS_RECORD_THREAD_QUEUE_SURRENDER_STICKY,
+  RTEMS_RECORD_THREAD_RESTART,
+  RTEMS_RECORD_THREAD_STACK_CURRENT,
+  RTEMS_RECORD_THREAD_STACK_SIZE,
+  RTEMS_RECORD_THREAD_STACK_USAGE,
+  RTEMS_RECORD_THREAD_START,
+  RTEMS_RECORD_THREAD_STATE_CLEAR,
+  RTEMS_RECORD_THREAD_STATE_SET,
+  RTEMS_RECORD_THREAD_SWITCH_IN,
+  RTEMS_RECORD_THREAD_SWITCH_OUT,
+  RTEMS_RECORD_THREAD_TERMINATE,
+  RTEMS_RECORD_UDP_INPUT,
+  RTEMS_RECORD_UDP_OUTPUT,
+  RTEMS_RECORD_UMA_ALLOC_PTR,
+  RTEMS_RECORD_UMA_ALLOC_ZONE,
+  RTEMS_RECORD_UMA_FREE_PTR,
+  RTEMS_RECORD_UMA_FREE_ZONE,
+  RTEMS_RECORD_UNLINK,
+  RTEMS_RECORD_UNMOUNT,
+  RTEMS_RECORD_UPTIME_HIGH,
+  RTEMS_RECORD_UPTIME_LOW,
+  RTEMS_RECORD_WORKSPACE_ALLOC,
+  RTEMS_RECORD_WORKSPACE_FREE,
+  RTEMS_RECORD_WORKSPACE_SIZE,
+  RTEMS_RECORD_WORKSPACE_USAGE,
+  RTEMS_RECORD_WRITE,
+  RTEMS_RECORD_WRITEV,
+
+  /* There are 512 events reserved for the user */
+  RTEMS_RECORD_USER = 512,
+
+  RTEMS_RECORD_LAST = 1023
+} rtems_record_event;
+
+/**
+ * @brief Bits in the record item event member reserved for the actual event.
+ */
+#define RTEMS_RECORD_EVENT_BITS 10
+
+/**
+ * @brief Bits in the record item event member reserved for the time of the
+ * event.
+ */
+#define RTEMS_RECORD_TIME_BITS 22
+
+/**
+ * @brief Builds a time event for the specified time stamp and event.
+ *
+ * The events are stored in the record item with a time stamp.  There are 22
+ * bits allocated to the time stamp and 10 bits allocated to the event.  The 22
+ * bits are enough to get reliable time stamps on a system with a 4GHz CPU
+ * counter and a 1000Hz clock tick.
+ */
+#define RTEMS_RECORD_TIME_EVENT( time, event ) \
+  ( ( ( time ) << RTEMS_RECORD_EVENT_BITS ) | ( event ) )
+
+/**
+ * @brief Gets the time of a time event.
+ */
+#define RTEMS_RECORD_GET_TIME( time_event ) \
+  ( ( time_event ) >> RTEMS_RECORD_EVENT_BITS )
+
+/**
+ * @brief Gets the event of a time event.
+ */
+#define RTEMS_RECORD_GET_EVENT( time_event ) \
+  ( ( time_event ) & ( ( 1U << RTEMS_RECORD_EVENT_BITS ) - 1U ) )
+
+/**
+ * @brief The record data integer type.
+ *
+ * It is big enough to store 32-bit integers and pointers.
+ */
+typedef unsigned long rtems_record_data;
+
+/**
+ * @brief The native record item.
+ */
+typedef struct __attribute__((__packed__)) {
+  uint32_t          event;
+  rtems_record_data data;
+} rtems_record_item;
+
+/**
+ * @brief The 32-bit format record item.
+ */
+typedef struct {
+  uint32_t event;
+  uint32_t data;
+} rtems_record_item_32;
+
+/**
+ * @brief The 64-bit format record item.
+ */
+typedef struct __attribute__((__packed__)) {
+  uint32_t event;
+  uint64_t data;
+} rtems_record_item_64;
+
+const char *rtems_record_event_text( rtems_record_event event );
+
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* _RTEMS_RECORDDATA_H */
diff --git a/cpukit/include/rtems/score/percpu.h b/cpukit/include/rtems/score/percpu.h
index 712d1cde36..22c5c30a00 100644
--- a/cpukit/include/rtems/score/percpu.h
+++ b/cpukit/include/rtems/score/percpu.h
@@ -69,6 +69,8 @@ extern "C" {
 
 #if !defined( ASM )
 
+struct Record_Control;
+
 struct _Thread_Control;
 
 struct Scheduler_Context;
@@ -513,6 +515,8 @@ typedef struct Per_CPU_Control {
     bool boot;
   #endif
 
+  struct Record_Control *record;
+
   Per_CPU_Stats Stats;
 } Per_CPU_Control;
 
diff --git a/cpukit/include/rtems/sysinit.h b/cpukit/include/rtems/sysinit.h
index 2c718af8de..60fd9ddf41 100644
--- a/cpukit/include/rtems/sysinit.h
+++ b/cpukit/include/rtems/sysinit.h
@@ -33,6 +33,7 @@ extern "C" {
 #define RTEMS_SYSINIT_MP_EARLY                   000500
 #define RTEMS_SYSINIT_DATA_STRUCTURES            000600
 #define RTEMS_SYSINIT_MP                         000700
+#define RTEMS_SYSINIT_RECORD                     000800
 #define RTEMS_SYSINIT_USER_EXTENSIONS            000900
 #define RTEMS_SYSINIT_CLASSIC_TASKS              000a00
 #define RTEMS_SYSINIT_CLASSIC_TIMER              000b00