smp: Provide cache optimized Per_CPU_Control

Delete _Per_CPU_Information_p.

4 files changed, 42 insertions, 21 deletions

diff --git a/cpukit/sapi/include/confdefs.h b/cpukit/sapi/include/confdefs.h
index c969fa78de..ed1385a3c9 100644
--- a/cpukit/sapi/include/confdefs.h
+++ b/cpukit/sapi/include/confdefs.h
@@ -2396,8 +2396,7 @@ const rtems_libio_helper rtems_fs_init_helper =
   * Instantiate the Per CPU information based upon the user configuration.
   */
  #if defined(CONFIGURE_INIT)
-   Per_CPU_Control _Per_CPU_Information[CONFIGURE_SMP_MAXIMUM_PROCESSORS];
-   Per_CPU_Control *_Per_CPU_Information_p[CONFIGURE_SMP_MAXIMUM_PROCESSORS];
+   Per_CPU_Control_envelope _Per_CPU_Information[CONFIGURE_SMP_MAXIMUM_PROCESSORS];
  #endif
 
 #endif
diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 11ba5f6681..14eed5a57f 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -33,6 +33,18 @@
 extern "C" {
 #endif
 
+#if defined( RTEMS_SMP )
+  /*
+   * This ensures that on SMP configurations the individual per-CPU controls
+   * are on different cache lines to prevent false sharing.  This define can be
+   * used in assembler code to easily get the per-CPU control for a particular
+   * processor.
+   */
+  #define PER_CPU_CONTROL_SIZE_LOG2 7
+
+  #define PER_CPU_CONTROL_SIZE ( 1 << PER_CPU_CONTROL_SIZE_LOG2 )
+#endif
+
 #if !defined( ASM )
 
 #ifndef __THREAD_CONTROL_DEFINED__
@@ -184,32 +196,47 @@ typedef struct {
   #endif
 } Per_CPU_Control;
 
+#if defined( RTEMS_SMP )
+typedef struct {
+  Per_CPU_Control per_cpu;
+  char unused_space_for_cache_line_alignment
+    [ PER_CPU_CONTROL_SIZE - sizeof( Per_CPU_Control ) ];
+} Per_CPU_Control_envelope;
+#else
+typedef struct {
+  Per_CPU_Control per_cpu;
+} Per_CPU_Control_envelope;
+#endif
+
 /**
  *  @brief Set of Per CPU Core Information
  *
  *  This is an array of per CPU core information.
  */
-extern Per_CPU_Control _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
+extern Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
 
 #if defined( RTEMS_SMP )
 static inline Per_CPU_Control *_Per_CPU_Get( void )
 {
   _Assert_Thread_dispatching_repressed();
 
-  return &_Per_CPU_Information[ _SMP_Get_current_processor() ];
+  return &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu;
 }
 #else
-#define _Per_CPU_Get() ( &_Per_CPU_Information[ 0 ] )
+#define _Per_CPU_Get() ( &_Per_CPU_Information[ 0 ].per_cpu )
 #endif
 
 static inline Per_CPU_Control *_Per_CPU_Get_by_index( uint32_t index )
 {
-  return &_Per_CPU_Information[ index ];
+  return &_Per_CPU_Information[ index ].per_cpu;
 }
 
 static inline uint32_t _Per_CPU_Get_index( const Per_CPU_Control *per_cpu )
 {
-  return ( uint32_t ) ( per_cpu - &_Per_CPU_Information[ 0 ] );
+  const Per_CPU_Control_envelope *per_cpu_envelope =
+    ( const Per_CPU_Control_envelope * ) per_cpu;
+
+  return ( uint32_t ) ( per_cpu_envelope - &_Per_CPU_Information[ 0 ] );
 }
 
 #if defined( RTEMS_SMP )
@@ -220,15 +247,6 @@ static inline void _Per_CPU_Send_interrupt( const Per_CPU_Control *per_cpu )
 }
 
 /**
- *  @brief Set of Pointers to Per CPU Core Information
- *
- *  This is an array of pointers to each CPU's per CPU data structure.
- *  It should be simpler to retrieve this pointer in assembly language
- *  that to calculate the array offset.
- */
-extern Per_CPU_Control *_Per_CPU_Information_p[];
-
-/**
  *  @brief Initialize SMP Handler
  *
  *  This method initialize the SMP Handler.
diff --git a/cpukit/score/src/percpu.c b/cpukit/score/src/percpu.c
index 5e1a917e1b..b041b45197 100644
--- a/cpukit/score/src/percpu.c
+++ b/cpukit/score/src/percpu.c
@@ -39,13 +39,10 @@
     /*
      *  Initialize per cpu pointer table
      */
-    _Per_CPU_Information_p[0] = _Per_CPU_Get_by_index( 0 );
     for ( cpu = 1 ; cpu < max_cpus; ++cpu ) {
 
       Per_CPU_Control *p = _Per_CPU_Get_by_index( cpu );
 
-      _Per_CPU_Information_p[cpu] = p;
-
 #if CPU_ALLOCATE_INTERRUPT_STACK == TRUE
       {
         size_t size = rtems_configuration_get_interrupt_stack_size();
@@ -101,5 +98,5 @@
    * statically allocated per cpu structure.  And the fields are initialized
    * as individual elements just like it has always been done.
    */
-  Per_CPU_Control _Per_CPU_Information[1];
+  Per_CPU_Control_envelope _Per_CPU_Information[1];
 #endif
diff --git a/cpukit/score/src/percpuasm.c b/cpukit/score/src/percpuasm.c
index a79936bab4..60ca48dc5a 100644
--- a/cpukit/score/src/percpuasm.c
+++ b/cpukit/score/src/percpuasm.c
@@ -26,13 +26,20 @@ RTEMS_STATIC_ASSERT(
   CPU_SIZEOF_POINTER
 );
 
-#ifdef __SIZEOF_POINTER__
+#if defined( __SIZEOF_POINTER__ )
   RTEMS_STATIC_ASSERT(
     CPU_SIZEOF_POINTER == __SIZEOF_POINTER__,
     __SIZEOF_POINTER__
   );
 #endif
 
+#if defined( RTEMS_SMP )
+  RTEMS_STATIC_ASSERT(
+    sizeof( Per_CPU_Control_envelope ) == PER_CPU_CONTROL_SIZE,
+    PER_CPU_CONTROL_SIZE
+  );
+#endif
+
 RTEMS_STATIC_ASSERT(
   offsetof(Per_CPU_Control, isr_nest_level) == PER_CPU_ISR_NEST_LEVEL,
   PER_CPU_ISR_NEST_LEVEL