score: Per-CPU thread dispatch disable level

Use a per-CPU thread dispatch disable level.  So instead of one global
thread dispatch disable level we have now one instance per processor.
This is a major performance improvement for SMP.  On non-SMP
configurations this may simplifiy the interrupt entry/exit code.

The giant lock is still present, but it is now decoupled from the thread
dispatching in _Thread_Dispatch(), _Thread_Handler(),
_Thread_Restart_self() and the interrupt entry/exit.   Access to the
giant lock is now available via _Giant_Acquire() and _Giant_Release().
The giant lock is still implicitly acquired via
_Thread_Dispatch_decrement_disable_level().

The giant lock is only acquired for high-level operations in interrupt
handlers (e.g. release of a semaphore, sending of an event).

As a side-effect this change fixes the lost thread dispatch necessary
indication bug in _Thread_Dispatch().

A per-CPU thread dispatch disable level greatly simplifies the SMP
support for the interrupt entry/exit code since no spin locks have to be
acquired in this area.  It is only necessary to get the current
processor index and use this to calculate the address of the own per-CPU
control.  This reduces the interrupt latency considerably.

All elements for the interrupt entry/exit code are now part of the
Per_CPU_Control structure: thread dispatch disable level, ISR nest level
and thread dispatch necessary.  Nothing else is required (except CPU
port specific stuff like on SPARC).

5 files changed, 92 insertions, 72 deletions

diff --git a/cpukit/score/include/rtems/score/isr.h b/cpukit/score/include/rtems/score/isr.h
index 8d57db25e8..4d4a5f3c49 100644
--- a/cpukit/score/include/rtems/score/isr.h
+++ b/cpukit/score/include/rtems/score/isr.h
@@ -96,28 +96,6 @@ SCORE_EXTERN ISR_Handler_entry *_ISR_Vector_table;
  */
 void _ISR_Handler_initialization ( void );
 
-#if defined( RTEMS_SMP )
-/**
- *  @brief Enter SMP interrupt code.
- *
- *  This method is used to enter the SMP interrupt section.
- *
- *  @retval This method returns the isr level.
- */
-int _ISR_SMP_Enter(void);
-
-/**
- *  @brief Exit SMP interrupt code.
- *
- *  This method is used to exit the SMP interrupt.
- *
- *  @retval This method returns 0 on a simple return and returns 1 on a
- *  dispatching return.
- */
-int _ISR_SMP_Exit(void);
-
-#endif /* defined( RTEMS_SMP ) */
-
 /**
  *  @brief Install interrupt handler vector.
  *
diff --git a/cpukit/score/include/rtems/score/objectimpl.h b/cpukit/score/include/rtems/score/objectimpl.h
index c7ed94c5f4..c6c870fcd9 100644
--- a/cpukit/score/include/rtems/score/objectimpl.h
+++ b/cpukit/score/include/rtems/score/objectimpl.h
@@ -880,6 +880,23 @@ RTEMS_INLINE_ROUTINE void _Objects_Put_without_thread_dispatch(
 }
 
 /**
+ * @brief Puts back an object obtained with _Objects_Get().
+ *
+ * The thread dispatch disable level will remain unchanged.
+ *
+ * On SMP configurations the Giant lock will be released.
+ */
+RTEMS_INLINE_ROUTINE void _Objects_Put_and_keep_thread_dispatch_disabled(
+  Objects_Control *the_object
+)
+{
+  (void) the_object;
+#if defined(RTEMS_SMP)
+  _Giant_Release();
+#endif
+}
+
+/**
  * @brief Puts back an object obtained with _Objects_Get_isr_disable().
  */
 RTEMS_INLINE_ROUTINE void _Objects_Put_for_get_isr_disable(
diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 7a4c70e646..8acd867fcb 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -166,6 +166,12 @@ typedef struct {
    */
   uint32_t isr_nest_level;
 
+  /**
+   * @brief The thread dispatch critical section nesting counter which is used
+   * to prevent context switches at inopportune moments.
+   */
+  volatile uint32_t thread_dispatch_disable_level;
+
   /** This is set to true when this CPU needs to run the dispatcher. */
   volatile bool dispatch_necessary;
 
@@ -267,9 +273,14 @@ extern Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
 #if defined( RTEMS_SMP )
 static inline Per_CPU_Control *_Per_CPU_Get( void )
 {
-  _Assert_Thread_dispatching_repressed();
+  Per_CPU_Control *per_cpu =
+    &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu;
 
-  return &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu;
+  _Assert(
+    per_cpu->thread_dispatch_disable_level != 0 || _ISR_Get_level() != 0
+  );
+
+  return per_cpu;
 }
 #else
 #define _Per_CPU_Get() ( &_Per_CPU_Information[ 0 ].per_cpu )
@@ -325,6 +336,8 @@ void _Per_CPU_Wait_for_state(
  * On a non SMP system, the _SMP_Get_current_processor() is defined to 0.
  * Thus when built for non-SMP, there should be no performance penalty.
  */
+#define _Thread_Dispatch_disable_level \
+  _Per_CPU_Get()->thread_dispatch_disable_level
 #define _Thread_Heir \
   _Per_CPU_Get()->heir
 #define _Thread_Executing \
@@ -373,9 +386,13 @@ void _Per_CPU_Wait_for_state(
  */
 #define PER_CPU_ISR_NEST_LEVEL \
   PER_CPU_END_STACK
-#define PER_CPU_DISPATCH_NEEDED \
+#define PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL \
   PER_CPU_ISR_NEST_LEVEL + 4
+#define PER_CPU_DISPATCH_NEEDED \
+  PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL + 4
 
+#define THREAD_DISPATCH_DISABLE_LEVEL \
+  (SYM(_Per_CPU_Information) + PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL)
 #define ISR_NEST_LEVEL \
   (SYM(_Per_CPU_Information) + PER_CPU_ISR_NEST_LEVEL)
 #define DISPATCH_NEEDED \
diff --git a/cpukit/score/include/rtems/score/threaddispatch.h b/cpukit/score/include/rtems/score/threaddispatch.h
index 7e7afb9161..9cd12873fb 100644
--- a/cpukit/score/include/rtems/score/threaddispatch.h
+++ b/cpukit/score/include/rtems/score/threaddispatch.h
@@ -14,7 +14,7 @@
 #ifndef _RTEMS_SCORE_THREADDISPATCH_H
 #define _RTEMS_SCORE_THREADDISPATCH_H
 
-#include <rtems/score/cpu.h>
+#include <rtems/score/percpu.h>
 #include <rtems/score/smplock.h>
 
 #ifdef __cplusplus
@@ -40,13 +40,6 @@ extern "C" {
  */
 
 /**
- *  The following declares the dispatch critical section nesting
- *  counter which is used to prevent context switches at inopportune
- *  moments.
- */
-SCORE_EXTERN volatile uint32_t   _Thread_Dispatch_disable_level;
-
-/**
  * @brief Indicates if the executing thread is inside a thread dispatch
  * critical section.
  *
@@ -56,36 +49,64 @@ SCORE_EXTERN volatile uint32_t   _Thread_Dispatch_disable_level;
  */
 RTEMS_INLINE_ROUTINE bool _Thread_Dispatch_is_enabled(void)
 {
-  return _Thread_Dispatch_disable_level == 0;
-}
+  bool enabled;
 
 #if defined(RTEMS_SMP)
-  typedef struct {
-    SMP_lock_Control lock;
-    uint32_t owner_cpu;
-    uint32_t nest_level;
-  } Thread_Dispatch_disable_level_lock_control;
+  ISR_Level level;
 
-  /**
-   * The following declares the smp spinlock to be used to control
-   * the dispatch critical section accesses across cpus.
-   */
-  SCORE_EXTERN Thread_Dispatch_disable_level_lock_control
-    _Thread_Dispatch_disable_level_lock;
+  _ISR_Disable( level );
+#endif
+
+  enabled = _Thread_Dispatch_disable_level == 0;
+
+#if defined(RTEMS_SMP)
+  _ISR_Enable( level );
+#endif
+
+  return enabled;
+}
+
+/**
+ * @briefs Gets thread dispatch disable level.
+ *
+ * @return The value of the thread dispatch level.
+ */
+RTEMS_INLINE_ROUTINE uint32_t _Thread_Dispatch_get_disable_level(void)
+{
+  return _Thread_Dispatch_disable_level;
+}
+
+/**
+ * @brief Thread dispatch initialization.
+ *
+ * This routine initializes the thread dispatching subsystem.
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Dispatch_initialization( void )
+{
+  _Thread_Dispatch_disable_level = 1;
+}
 
+#if defined(RTEMS_SMP)
   /**
-   *  @brief Initializes the thread dispatching subsystem.
+   * @brief Acquires the giant lock.
+   *
+   * The giant lock is a recursive SMP lock protecting nearly all operating
+   * system services.
+   *
+   * This lock is implicitly acquired in
+   * _Thread_Dispatch_increment_disable_level().
    *
-   *  This routine initializes the thread dispatching subsystem.
+   * Thread dispatching must be disabled before this lock can be acquired.
    */
-  void _Thread_Dispatch_initialization(void);
+  void _Giant_Acquire( void );
 
   /**
-   *  @brief Returns value of the the thread dispatch level.
+   * @brief Releases the giant lock.
    *
-   * This routine returns value of the the thread dispatch level.
+   * This lock is implicitly released in
+   * _Thread_Dispatch_decrement_disable_level().
    */
-  uint32_t _Thread_Dispatch_get_disable_level(void);
+  void _Giant_Release( void );
 
   /**
    *  @brief Sets thread dispatch level to the value passed in.
@@ -110,16 +131,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Dispatch_is_enabled(void)
   uint32_t _Thread_Dispatch_decrement_disable_level(void);
 #else /* RTEMS_SMP */
   /**
-   * @brief Get thread dispatch disable level.
-   *
-   * This routine returns value of the the thread dispatch level.
-   */
-  RTEMS_INLINE_ROUTINE uint32_t _Thread_Dispatch_get_disable_level(void)
-  {
-    return _Thread_Dispatch_disable_level;
-  }
-
-  /**
    * @brief Set thread dispatch disable level.
    *
    * This routine sets thread dispatch level to the
@@ -160,16 +171,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Dispatch_is_enabled(void)
 
     return level;
   }
-
-  /**
-   * @brief Thread dispatch initialization.
-   *
-   * This routine initializes the thread dispatching subsystem.
-   */
-  RTEMS_INLINE_ROUTINE void _Thread_Dispatch_initialization( void )
-  {
-    _Thread_Dispatch_set_disable_level( 1 );
-  }
 #endif /* RTEMS_SMP */
 
 /**
diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h
index 128820b1e9..717f00b4e8 100644
--- a/cpukit/score/include/rtems/score/threadimpl.h
+++ b/cpukit/score/include/rtems/score/threadimpl.h
@@ -527,6 +527,13 @@ RTEMS_INLINE_ROUTINE void _Thread_Unblock (
 
 RTEMS_INLINE_ROUTINE void _Thread_Restart_self( void )
 {
+#if defined(RTEMS_SMP)
+  ISR_Level level;
+
+  _Per_CPU_ISR_disable_and_acquire( _Per_CPU_Get(), level );
+  ( void ) level;
+#endif
+
 #if ( CPU_HARDWARE_FP == TRUE ) || ( CPU_SOFTWARE_FP == TRUE )
   if ( _Thread_Executing->fp_context != NULL )
     _Context_Restore_fp( &_Thread_Executing->fp_context );