SMP: Fix and optimize thread dispatching

According to the C11 and C++11 memory models only a read-modify-write
operation guarantees that we read the last value written in modification
order.  Avoid the sequential consistent thread fence and instead use the
inter-processor interrupt to set the thread dispatch necessary
indicator.

3 files changed, 29 insertions, 41 deletions

diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index f1dad90ccc..806c290b7c 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -279,9 +279,11 @@ typedef struct Per_CPU_Control {
    * @brief This is the heir thread for this processor.
    *
    * This field is not protected by a lock.  The only writer after multitasking
-   * start is the scheduler owning this processor.  This processor will set the
-   * dispatch necessary indicator to false, before it reads the heir.  This
-   * field is used in combination with the dispatch necessary indicator.
+   * start is the scheduler owning this processor.  It is assumed that stores
+   * to pointers are atomic on all supported SMP architectures.  The CPU port
+   * specific code (inter-processor interrupt handling and
+   * _CPU_SMP_Send_interrupt()) must guarantee that this processor observes the
+   * last value written.
    *
    * A thread can be a heir on at most one processor in the system.
    *
@@ -290,16 +292,15 @@ typedef struct Per_CPU_Control {
   struct _Thread_Control *heir;
 
   /**
-   * @brief This is set to true when this processor needs to run the
+   * @brief This is set to true when this processor needs to run the thread
    * dispatcher.
    *
    * It is volatile since interrupts may alter this flag.
    *
-   * This field is not protected by a lock.  There are two writers after
-   * multitasking start.  The scheduler owning this processor sets this
-   * indicator to true, after it updated the heir field.  This processor sets
-   * this indicator to false, before it reads the heir.  This field is used in
-   * combination with the heir field.
+   * This field is not protected by a lock and must be accessed only by this
+   * processor.  Code (e.g. scheduler and post-switch action requests) running
+   * on another processors must use an inter-processor interrupt to set the
+   * thread dispatch necessary indicator to true.
    *
    * @see _Thread_Get_heir_and_make_it_executing().
    */
diff --git a/cpukit/score/include/rtems/score/smpimpl.h b/cpukit/score/include/rtems/score/smpimpl.h
index 97c78b02b0..3167e82a82 100644
--- a/cpukit/score/include/rtems/score/smpimpl.h
+++ b/cpukit/score/include/rtems/score/smpimpl.h
@@ -146,6 +146,12 @@ static inline void _SMP_Inter_processor_interrupt_handler( void )
 {
   Per_CPU_Control *cpu_self = _Per_CPU_Get();
 
+  /*
+   * In the common case the inter-processor interrupt is issued to carry out a
+   * thread dispatch.
+   */
+  cpu_self->dispatch_necessary = true;
+
   if ( _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) != 0 ) {
     unsigned long message = _Atomic_Exchange_ulong(
       &cpu_self->message,
diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h
index 68c26c388f..7412bd9633 100644
--- a/cpukit/score/include/rtems/score/threadimpl.h
+++ b/cpukit/score/include/rtems/score/threadimpl.h
@@ -11,7 +11,7 @@
  *  COPYRIGHT (c) 1989-2008.
  *  On-Line Applications Research Corporation (OAR).
  *
- *  Copyright (c) 2014 embedded brains GmbH.
+ *  Copyright (c) 2014-2015 embedded brains GmbH.
  *
  *  The license and distribution terms for this file may be
  *  found in the file LICENSE in this distribution or at
@@ -793,7 +793,8 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Internal_allocate( void )
 /**
  * @brief Gets the heir of the processor and makes it executing.
  *
- * The thread dispatch necessary indicator is cleared as a side-effect.
+ * Must be called with interrupts disabled.  The thread dispatch necessary
+ * indicator is cleared as a side-effect.
  *
  * @return The heir thread.
  *
@@ -806,18 +807,8 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Get_heir_and_make_it_executing(
 {
   Thread_Control *heir;
 
-  cpu_self->dispatch_necessary = false;
-
-#if defined( RTEMS_SMP )
-  /*
-   * It is critical that we first update the dispatch necessary and then the
-   * read the heir so that we don't miss an update by
-   * _Thread_Dispatch_update_heir().
-   */
-  _Atomic_Fence( ATOMIC_ORDER_SEQ_CST );
-#endif
-
   heir = cpu_self->heir;
+  cpu_self->dispatch_necessary = false;
   cpu_self->executing = heir;
 
   return heir;
@@ -832,23 +823,10 @@ RTEMS_INLINE_ROUTINE void _Thread_Dispatch_update_heir(
 {
   cpu_for_heir->heir = heir;
 
-  /*
-   * It is critical that we first update the heir and then the dispatch
-   * necessary so that _Thread_Get_heir_and_make_it_executing() cannot miss an
-   * update.
-   */
-  _Atomic_Fence( ATOMIC_ORDER_SEQ_CST );
-
-  /*
-   * Only update the dispatch necessary indicator if not already set to
-   * avoid superfluous inter-processor interrupts.
-   */
-  if ( !cpu_for_heir->dispatch_necessary ) {
-    cpu_for_heir->dispatch_necessary = true;
-
-    if ( cpu_for_heir != cpu_self ) {
-      _Per_CPU_Send_interrupt( cpu_for_heir );
-    }
+  if ( cpu_for_heir == cpu_self ) {
+    cpu_self->dispatch_necessary = true;
+  } else {
+    _Per_CPU_Send_interrupt( cpu_for_heir );
   }
 }
 #endif
@@ -930,12 +908,15 @@ RTEMS_INLINE_ROUTINE void _Thread_Add_post_switch_action(
   ISR_Level        level;
 
   cpu_of_thread = _Thread_Action_ISR_disable_and_acquire( thread, &level );
-  cpu_of_thread->dispatch_necessary = true;
 
 #if defined(RTEMS_SMP)
-  if ( _Per_CPU_Get() != cpu_of_thread ) {
+  if ( _Per_CPU_Get() == cpu_of_thread ) {
+    cpu_of_thread->dispatch_necessary = true;
+  } else {
     _Per_CPU_Send_interrupt( cpu_of_thread );
   }
+#else
+  cpu_of_thread->dispatch_necessary = true;
 #endif
 
   _Chain_Append_if_is_off_chain_unprotected(