score: Add and use _Thread_Do_dispatch()

The _Thread_Dispatch() function is quite complex and the time to set up
and tear down the stack frame is significant.  Split this function into
two parts.  The complex part is now in _Thread_Do_dispatch().  Call
_Thread_Do_dispatch() in _Thread_Enable_dispatch() only if necessary.
This increases the average case performance.

Simplify _Thread_Handler() for SMP configurations.

Update #2273.

3 files changed, 120 insertions, 85 deletions

diff --git a/cpukit/score/include/rtems/score/threaddispatch.h b/cpukit/score/include/rtems/score/threaddispatch.h
index aec436fb45..80de19d53b 100644
--- a/cpukit/score/include/rtems/score/threaddispatch.h
+++ b/cpukit/score/include/rtems/score/threaddispatch.h
@@ -193,26 +193,40 @@ RTEMS_INLINE_ROUTINE void _Thread_Dispatch_initialization( void )
 #endif /* RTEMS_SMP */
 
 /**
- *  @brief Dispatch thread.
+ * @brief Performs a thread dispatch if necessary.
  *
- *  This routine is responsible for transferring control of the
- *  processor from the executing thread to the heir thread. Once the
- *  heir is running an attempt is made to dispatch any ASRs.
- *  As part of this process, it is responsible for the following actions:
- *     + saving the context of the executing thread
- *     + restoring the context of the heir thread
- *     + dispatching any signals for the resulting executing thread
-
- *  ALTERNATE ENTRY POINTS:
- *    void _Thread_Enable_dispatch();
+ * This routine is responsible for transferring control of the processor from
+ * the executing thread to the heir thread.  Once the heir is running an
+ * attempt is made to run the pending post-switch thread actions.
  *
- *  - INTERRUPT LATENCY:
- *    + dispatch thread
- *    + no dispatch thread
+ * As part of this process, it is responsible for the following actions
+ *   - update timing information of the executing thread,
+ *   - save the context of the executing thread,
+ *   - invokation of the thread switch user extensions,
+ *   - restore the context of the heir thread, and
+ *   - run of pending post-switch thread actions of the resulting executing
+ *     thread.
+ *
+ * On entry the thread dispatch level must be equal to zero.
  */
 void _Thread_Dispatch( void );
 
 /**
+ * @brief Performs a thread dispatch on the current processor.
+ *
+ * On entry the thread dispatch disable level must be equal to one and
+ * interrupts must be disabled.
+ *
+ * This function assumes that a thread dispatch is necessary.
+ *
+ * @param[in] cpu_self The current processor.
+ * @param[in] level The previous interrupt level.
+ *
+ * @see _Thread_Dispatch().
+ */
+void _Thread_Do_dispatch( Per_CPU_Control *cpu_self, ISR_Level level );
+
+/**
  * This routine prevents dispatching.
  */
 
@@ -228,8 +242,33 @@ RTEMS_INLINE_ROUTINE void _Thread_Disable_dispatch( void )
 
 RTEMS_INLINE_ROUTINE void _Thread_Enable_dispatch_body( void )
 {
-  if ( _Thread_Dispatch_decrement_disable_level() == 0 )
-    _Thread_Dispatch();
+  Per_CPU_Control *cpu_self;
+  uint32_t         disable_level;
+
+  cpu_self = _Per_CPU_Get();
+
+#if defined( RTEMS_SMP )
+  _Giant_Release( cpu_self );
+#endif
+
+  disable_level = cpu_self->thread_dispatch_disable_level;
+
+  if ( disable_level == 1 ) {
+    ISR_Level level;
+
+    _ISR_Disable_without_giant( level );
+
+    if ( cpu_self->dispatch_necessary ) {
+      _Thread_Do_dispatch( cpu_self, level );
+    } else {
+      cpu_self->thread_dispatch_disable_level = 0;
+      _Profiling_Thread_dispatch_enable( cpu_self, 0 );
+    }
+
+    _ISR_Enable_without_giant( level );
+  } else {
+    cpu_self->thread_dispatch_disable_level = disable_level - 1;
+  }
 }
 
 /**
diff --git a/cpukit/score/src/threaddispatch.c b/cpukit/score/src/threaddispatch.c
index cc023fc57f..f20f427bd6 100644
--- a/cpukit/score/src/threaddispatch.c
+++ b/cpukit/score/src/threaddispatch.c
@@ -60,40 +60,15 @@ static void _Thread_Run_post_switch_actions( Thread_Control *executing )
   _Thread_Action_release_and_ISR_enable( cpu_self, level );
 }
 
-void _Thread_Dispatch( void )
+void _Thread_Do_dispatch( Per_CPU_Control *cpu_self, ISR_Level level )
 {
-  Per_CPU_Control  *cpu_self;
-  Thread_Control   *executing;
-  ISR_Level         level;
-
-#if defined( RTEMS_SMP )
-  /*
-   * On SMP the complete context switch must be atomic with respect to one
-   * processor.  See also _Thread_Handler() since _Context_switch() may branch
-   * to this function.
-   */
-  _ISR_Disable_without_giant( level );
-#endif
+  Thread_Control *executing;
 
-  cpu_self = _Per_CPU_Get();
-  _Assert( cpu_self->thread_dispatch_disable_level == 0 );
-  _Profiling_Thread_dispatch_disable( cpu_self, 0 );
-  cpu_self->thread_dispatch_disable_level = 1;
+  _Assert( cpu_self->thread_dispatch_disable_level == 1 );
 
-  /*
-   *  Now determine if we need to perform a dispatch on the current CPU.
-   */
   executing = cpu_self->executing;
 
-#if !defined( RTEMS_SMP )
-  _ISR_Disable( level );
-#endif
-
-#if defined( RTEMS_SMP )
-  if ( cpu_self->dispatch_necessary ) {
-#else
-  while ( cpu_self->dispatch_necessary ) {
-#endif
+  do {
     Thread_Control *heir = _Thread_Get_heir_and_make_it_executing( cpu_self );
 
     /*
@@ -115,6 +90,11 @@ void _Thread_Dispatch( void )
     if ( heir->budget_algorithm == THREAD_CPU_BUDGET_ALGORITHM_RESET_TIMESLICE )
       heir->cpu_time_budget = rtems_configuration_get_ticks_per_timeslice();
 
+    /*
+     * On SMP the complete context switch must be atomic with respect to one
+     * processor.  See also _Thread_Handler() since _Context_switch() may branch
+     * to this function.
+     */
 #if !defined( RTEMS_SMP )
     _ISR_Enable( level );
 #endif
@@ -158,7 +138,13 @@ void _Thread_Dispatch( void )
 #if !defined( RTEMS_SMP )
     _ISR_Disable( level );
 #endif
-  }
+  } while (
+#if defined( RTEMS_SMP )
+    false
+#else
+    cpu_self->dispatch_necessary
+#endif
+  );
 
 post_switch:
   _Assert( cpu_self->thread_dispatch_disable_level == 1 );
@@ -169,3 +155,21 @@ post_switch:
 
   _Thread_Run_post_switch_actions( executing );
 }
+
+void _Thread_Dispatch( void )
+{
+  ISR_Level        level;
+  Per_CPU_Control *cpu_self;
+
+  _ISR_Disable_without_giant( level );
+
+  cpu_self = _Per_CPU_Get();
+
+  if ( cpu_self->dispatch_necessary ) {
+    _Profiling_Thread_dispatch_disable( cpu_self, 0 );
+    cpu_self->thread_dispatch_disable_level = 1;
+    _Thread_Do_dispatch( cpu_self, level );
+  } else {
+    _ISR_Enable_without_giant( level );
+  }
+}
diff --git a/cpukit/score/src/threadhandler.c b/cpukit/score/src/threadhandler.c
index db7302807c..fd828a2325 100644
--- a/cpukit/score/src/threadhandler.c
+++ b/cpukit/score/src/threadhandler.c
@@ -26,9 +26,9 @@
 
 void _Thread_Handler( void )
 {
-  Thread_Control *executing = _Thread_Executing;
-  ISR_Level       level;
-
+  Thread_Control  *executing = _Thread_Executing;
+  ISR_Level        level;
+  Per_CPU_Control *cpu_self;
 
   /*
    * Some CPUs need to tinker with the call frame or registers when the
@@ -37,16 +37,21 @@ void _Thread_Handler( void )
    */
   _Context_Initialization_at_thread_begin();
 
-  #if !defined(RTEMS_SMP)
-    /*
-     * have to put level into a register for those cpu's that use
-     * inline asm here
-     */
-    level = executing->Start.isr_level;
-    _ISR_Set_level( level );
+  #if defined(RTEMS_SMP)
+    /* On SMP we enter _Thread_Handler() with interrupts disabled */
+    _Assert( _ISR_Get_level() != 0 );
+
+    _Thread_Debug_set_real_processor( executing, _Per_CPU_Get() );
   #endif
 
   /*
+   * have to put level into a register for those cpu's that use
+   * inline asm here
+   */
+  level = executing->Start.isr_level;
+  _ISR_Set_level( level );
+
+  /*
    * Initialize the floating point context because we do not come
    * through _Thread_Dispatch on our first invocation. So the normal
    * code path for performing the FP context switch is not hit.
@@ -61,37 +66,24 @@ void _Thread_Handler( void )
   _User_extensions_Thread_begin( executing );
 
   /*
+   * Do not use the level of the thread control block, since it has a
+   * different format.
+   */
+  _ISR_Disable_without_giant( level );
+
+  /*
    *  At this point, the dispatch disable level BETTER be 1.
    */
-  #if defined(RTEMS_SMP)
-    {
-      /*
-       * On SMP we enter _Thread_Handler() with interrupts disabled and
-       * _Thread_Dispatch() obtained the per-CPU lock for us.  We have to
-       * release it here and set the desired interrupt level of the thread.
-       */
-      Per_CPU_Control *cpu_self = _Per_CPU_Get();
-
-      _Assert( cpu_self->thread_dispatch_disable_level == 1 );
-      _Assert( _ISR_Get_level() != 0 );
-
-      _Thread_Debug_set_real_processor( executing, cpu_self );
-
-      cpu_self->thread_dispatch_disable_level = 0;
-      _Profiling_Thread_dispatch_enable( cpu_self, 0 );
-
-      level = executing->Start.isr_level;
-      _ISR_Set_level( level);
-
-      /*
-       * The thread dispatch level changed from one to zero.  Make sure we lose
-       * no thread dispatch necessary update.
-       */
-      _Thread_Dispatch();
-    }
-  #else
-    _Thread_Enable_dispatch();
-  #endif
+  cpu_self = _Per_CPU_Get();
+  _Assert( cpu_self->thread_dispatch_disable_level == 1 );
+
+  /*
+   * Make sure we lose no thread dispatch necessary update and execute the
+   * post-switch actions.  As a side-effect change the thread dispatch level
+   * from one to zero.  Do not use _Thread_Enable_dispatch() since there is no
+   * valid thread dispatch necessary indicator in this context.
+   */
+  _Thread_Do_dispatch( cpu_self, level );
 
   /*
    *  RTEMS supports multiple APIs and each API can define a different