score: Fix SMP message handling

According to the C11 standard only atomic read-modify-write operations
guarantee that the last value written in modification order is read, see
"7.17.3 Order and consistency".  Thus we must use a read-modify-write in
_SMP_Inter_processor_interrupt_handler() to make sure we read an
up-to-date message.

7 files changed, 37 insertions, 34 deletions

diff --git a/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c b/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
index 7e939ff162..a3a95f4ea2 100644
--- a/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
+++ b/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
@@ -62,7 +62,6 @@ void _CPU_SMP_Prepare_start_multitasking( void )
 
 void _CPU_SMP_Send_interrupt( uint32_t target_processor_index )
 {
-  _ARM_Data_memory_barrier();
   arm_gic_irq_generate_software_irq(
     ARM_GIC_IRQ_SGI_0,
     ARM_GIC_IRQ_SOFTWARE_IRQ_TO_ALL_IN_LIST,
diff --git a/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c b/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
index 844f2b66b5..7bc7ec3089 100644
--- a/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
+++ b/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
@@ -267,10 +267,5 @@ void _CPU_SMP_Prepare_start_multitasking(void)
 
 void _CPU_SMP_Send_interrupt(uint32_t target_processor_index)
 {
-#ifdef __PPC_CPU_E6500__
-  ppc_light_weight_synchronize();
-#else
-  ppc_synchronize_data();
-#endif
   qoriq.pic.ipidr [IPI_INDEX].reg = 1U << target_processor_index;
 }
diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 55f799ed1c..8d47aaca4f 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -619,11 +619,6 @@ static inline bool _Per_CPU_Is_boot_processor(
 
 #if defined( RTEMS_SMP )
 
-static inline void _Per_CPU_Send_interrupt( const Per_CPU_Control *cpu )
-{
-  _CPU_SMP_Send_interrupt( _Per_CPU_Get_index( cpu ) );
-}
-
 /**
  *  @brief Allocate and Initialize Per CPU Structures
  *
diff --git a/cpukit/score/include/rtems/score/smpimpl.h b/cpukit/score/include/rtems/score/smpimpl.h
index d1b7214bbb..7f2de5ff70 100644
--- a/cpukit/score/include/rtems/score/smpimpl.h
+++ b/cpukit/score/include/rtems/score/smpimpl.h
@@ -167,8 +167,10 @@ void _SMP_Multicast_actions_process( void );
  */
 static inline long unsigned _SMP_Inter_processor_interrupt_handler( void )
 {
-  Per_CPU_Control *cpu_self = _Per_CPU_Get();
-  unsigned long message = 0;
+  Per_CPU_Control *cpu_self;
+  unsigned long    message;
+
+  cpu_self = _Per_CPU_Get();
 
   /*
    * In the common case the inter-processor interrupt is issued to carry out a
@@ -176,13 +178,13 @@ static inline long unsigned _SMP_Inter_processor_interrupt_handler( void )
    */
   cpu_self->dispatch_necessary = true;
 
-  if ( _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) != 0 ) {
-    message = _Atomic_Exchange_ulong(
-      &cpu_self->message,
-      0UL,
-      ATOMIC_ORDER_RELAXED
-    );
+  message = _Atomic_Exchange_ulong(
+    &cpu_self->message,
+    0,
+    ATOMIC_ORDER_ACQUIRE
+  );
 
+  if ( message != 0 ) {
     if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) {
       _SMP_Fatal( SMP_FATAL_SHUTDOWN_RESPONSE );
       /* does not continue past here */
diff --git a/cpukit/score/include/rtems/score/threaddispatch.h b/cpukit/score/include/rtems/score/threaddispatch.h
index 573d1be2b3..4cb223b147 100644
--- a/cpukit/score/include/rtems/score/threaddispatch.h
+++ b/cpukit/score/include/rtems/score/threaddispatch.h
@@ -226,6 +226,30 @@ RTEMS_INLINE_ROUTINE void _Thread_Dispatch_unnest( Per_CPU_Control *cpu_self )
   --cpu_self->thread_dispatch_disable_level;
 }
 
+/**
+ * @brief Requests a thread dispatch on the target processor.
+ *
+ * @param[in] cpu_self The current processor.
+ * @param[in] cpu_target The target processor to request a thread dispatch.
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Dispatch_request(
+  Per_CPU_Control *cpu_self,
+  Per_CPU_Control *cpu_target
+)
+{
+#if defined( RTEMS_SMP )
+  if ( cpu_self == cpu_target ) {
+    cpu_self->dispatch_necessary = true;
+  } else {
+    _Atomic_Fetch_or_ulong( &cpu_target->message, 0, ATOMIC_ORDER_RELEASE );
+    _CPU_SMP_Send_interrupt( _Per_CPU_Get_index( cpu_target ) );
+  }
+#else
+ cpu_self->dispatch_necessary = true;
+ (void) cpu_target;
+#endif
+}
+
 /** @} */
 
 #ifdef __cplusplus
diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h
index 6bb78158e3..c878266c58 100644
--- a/cpukit/score/include/rtems/score/threadimpl.h
+++ b/cpukit/score/include/rtems/score/threadimpl.h
@@ -844,11 +844,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Dispatch_update_heir(
 
   cpu_for_heir->heir = heir;
 
-  if ( cpu_for_heir == cpu_self ) {
-    cpu_self->dispatch_necessary = true;
-  } else {
-    _Per_CPU_Send_interrupt( cpu_for_heir );
-  }
+  _Thread_Dispatch_request( cpu_self, cpu_for_heir );
 }
 #endif
 
@@ -885,15 +881,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Add_post_switch_action(
 
   action->handler = handler;
 
-#if defined(RTEMS_SMP)
-  if ( _Per_CPU_Get() == cpu_of_thread ) {
-    cpu_of_thread->dispatch_necessary = true;
-  } else {
-    _Per_CPU_Send_interrupt( cpu_of_thread );
-  }
-#else
-  cpu_of_thread->dispatch_necessary = true;
-#endif
+  _Thread_Dispatch_request( _Per_CPU_Get(), cpu_of_thread );
 
   _Chain_Append_if_is_off_chain_unprotected(
     &the_thread->Post_switch_actions.Chain,
diff --git a/cpukit/score/src/smp.c b/cpukit/score/src/smp.c
index e26de42e68..d6aad85e76 100644
--- a/cpukit/score/src/smp.c
+++ b/cpukit/score/src/smp.c
@@ -167,7 +167,7 @@ void _SMP_Send_message( uint32_t cpu_index, unsigned long message )
 {
   Per_CPU_Control *cpu = _Per_CPU_Get_by_index( cpu_index );
 
-  _Atomic_Fetch_or_ulong( &cpu->message, message, ATOMIC_ORDER_RELAXED );
+  _Atomic_Fetch_or_ulong( &cpu->message, message, ATOMIC_ORDER_RELEASE );
 
   _CPU_SMP_Send_interrupt( cpu_index );
 }