score: Per-CPU thread dispatch disable level

Use a per-CPU thread dispatch disable level.  So instead of one global
thread dispatch disable level we have now one instance per processor.
This is a major performance improvement for SMP.  On non-SMP
configurations this may simplifiy the interrupt entry/exit code.

The giant lock is still present, but it is now decoupled from the thread
dispatching in _Thread_Dispatch(), _Thread_Handler(),
_Thread_Restart_self() and the interrupt entry/exit.   Access to the
giant lock is now available via _Giant_Acquire() and _Giant_Release().
The giant lock is still implicitly acquired via
_Thread_Dispatch_decrement_disable_level().

The giant lock is only acquired for high-level operations in interrupt
handlers (e.g. release of a semaphore, sending of an event).

As a side-effect this change fixes the lost thread dispatch necessary
indication bug in _Thread_Dispatch().

A per-CPU thread dispatch disable level greatly simplifies the SMP
support for the interrupt entry/exit code since no spin locks have to be
acquired in this area.  It is only necessary to get the current
processor index and use this to calculate the address of the own per-CPU
control.  This reduces the interrupt latency considerably.

All elements for the interrupt entry/exit code are now part of the
Per_CPU_Control structure: thread dispatch disable level, ISR nest level
and thread dispatch necessary.  Nothing else is required (except CPU
port specific stuff like on SPARC).

1 files changed, 20 insertions, 3 deletions

diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 7a4c70e646..8acd867fcb 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -166,6 +166,12 @@ typedef struct {
    */
   uint32_t isr_nest_level;
 
+  /**
+   * @brief The thread dispatch critical section nesting counter which is used
+   * to prevent context switches at inopportune moments.
+   */
+  volatile uint32_t thread_dispatch_disable_level;
+
   /** This is set to true when this CPU needs to run the dispatcher. */
   volatile bool dispatch_necessary;
 
@@ -267,9 +273,14 @@ extern Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
 #if defined( RTEMS_SMP )
 static inline Per_CPU_Control *_Per_CPU_Get( void )
 {
-  _Assert_Thread_dispatching_repressed();
+  Per_CPU_Control *per_cpu =
+    &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu;
 
-  return &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu;
+  _Assert(
+    per_cpu->thread_dispatch_disable_level != 0 || _ISR_Get_level() != 0
+  );
+
+  return per_cpu;
 }
 #else
 #define _Per_CPU_Get() ( &_Per_CPU_Information[ 0 ].per_cpu )
@@ -325,6 +336,8 @@ void _Per_CPU_Wait_for_state(
  * On a non SMP system, the _SMP_Get_current_processor() is defined to 0.
  * Thus when built for non-SMP, there should be no performance penalty.
  */
+#define _Thread_Dispatch_disable_level \
+  _Per_CPU_Get()->thread_dispatch_disable_level
 #define _Thread_Heir \
   _Per_CPU_Get()->heir
 #define _Thread_Executing \
@@ -373,9 +386,13 @@ void _Per_CPU_Wait_for_state(
  */
 #define PER_CPU_ISR_NEST_LEVEL \
   PER_CPU_END_STACK
-#define PER_CPU_DISPATCH_NEEDED \
+#define PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL \
   PER_CPU_ISR_NEST_LEVEL + 4
+#define PER_CPU_DISPATCH_NEEDED \
+  PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL + 4
 
+#define THREAD_DISPATCH_DISABLE_LEVEL \
+  (SYM(_Per_CPU_Information) + PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL)
 #define ISR_NEST_LEVEL \
   (SYM(_Per_CPU_Information) + PER_CPU_ISR_NEST_LEVEL)
 #define DISPATCH_NEEDED \