SMP: Fix and optimize thread dispatching

According to the C11 and C++11 memory models only a read-modify-write
operation guarantees that we read the last value written in modification
order.  Avoid the sequential consistent thread fence and instead use the
inter-processor interrupt to set the thread dispatch necessary
indicator.

5 files changed, 33 insertions, 29 deletions

diff --git a/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c b/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
index f2c0201c2c..7e939ff162 100644
--- a/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
+++ b/c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 embedded brains GmbH.  All rights reserved.
+ * Copyright (c) 2013-2015 embedded brains GmbH.  All rights reserved.
  *
  *  embedded brains GmbH
  *  Dornierstr. 4
@@ -62,6 +62,7 @@ void _CPU_SMP_Prepare_start_multitasking( void )
 
 void _CPU_SMP_Send_interrupt( uint32_t target_processor_index )
 {
+  _ARM_Data_memory_barrier();
   arm_gic_irq_generate_software_irq(
     ARM_GIC_IRQ_SGI_0,
     ARM_GIC_IRQ_SOFTWARE_IRQ_TO_ALL_IN_LIST,
diff --git a/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c b/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
index 0b0743b35e..8952d3e809 100644
--- a/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
+++ b/c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c
@@ -235,5 +235,10 @@ void _CPU_SMP_Prepare_start_multitasking(void)
 
 void _CPU_SMP_Send_interrupt(uint32_t target_processor_index)
 {
+#ifdef __PPC_CPU_E6500__
+  ppc_light_weight_synchronize();
+#else
+  ppc_synchronize_data();
+#endif
   qoriq.pic.ipidr [IPI_INDEX].reg = 1U << target_processor_index;
 }
diff --git a/c/src/lib/libbsp/sparc/shared/irq_asm.S b/c/src/lib/libbsp/sparc/shared/irq_asm.S
index f7222e7623..9d8600e0fe 100644
--- a/c/src/lib/libbsp/sparc/shared/irq_asm.S
+++ b/c/src/lib/libbsp/sparc/shared/irq_asm.S
@@ -236,24 +236,18 @@ check_is_executing:
         beq     try_update_is_executing
          mov    1, %g1
 
-        ! Check if a thread dispatch is necessary
-        ldub    [%g6 + PER_CPU_DISPATCH_NEEDED], %g1
-        cmp     %g1, 0
-        beq     check_is_executing
-         nop
-
-        ! We have a new heir
-
-        ! Clear the thread dispatch necessary flag
-        stub    %g0, [%g6 + PER_CPU_DISPATCH_NEEDED]
-
-        ! Here we assume a strong memory order, otherwise a memory barrier must
-        ! be inserted here
+        ! We may have a new heir
 
         ! Read the executing and heir
         ld      [%g6 + PER_CPU_OFFSET_EXECUTING], %g1
         ld      [%g6 + PER_CPU_OFFSET_HEIR], %g2
 
+        ! Update the executing only if necessary to avoid cache line
+        ! monopolization.
+        cmp     %g1, %g2
+        beq     try_update_is_executing
+         mov    1, %g1
+
         ! Calculate the heir context pointer
         sub     %o1, %g1, %g1
         add     %g1, %g2, %o1
diff --git a/c/src/lib/libcpu/powerpc/new-exceptions/cpu_asm.S b/c/src/lib/libcpu/powerpc/new-exceptions/cpu_asm.S
index 5d8c70d290..8bfef20043 100644
--- a/c/src/lib/libcpu/powerpc/new-exceptions/cpu_asm.S
+++ b/c/src/lib/libcpu/powerpc/new-exceptions/cpu_asm.S
@@ -413,12 +413,12 @@ check_is_executing:
 	addi	r6, r5, PPC_CONTEXT_OFFSET_IS_EXECUTING
 	lwarx	r7, r0, r6
 	cmpwi	r7, 0
-	bne	check_thread_dispatch_necessary
+	bne	get_potential_new_heir
 
 	/* Try to update the is executing indicator of the heir context */
 	li	r7, 1
 	stwcx.	r7, r0, r6
-	bne	check_thread_dispatch_necessary
+	bne	get_potential_new_heir
 	isync
 #endif
 
@@ -536,26 +536,23 @@ PROC (_CPU_Context_restore):
 	b	restore_context
 
 #ifdef RTEMS_SMP
-check_thread_dispatch_necessary:
+get_potential_new_heir:
 
 	GET_SELF_CPU_CONTROL	r6
 
-	/* Check if a thread dispatch is necessary */
-	lbz	r7, PER_CPU_DISPATCH_NEEDED(r6)
-	cmpwi	r7, 0
-	beq	check_is_executing
-
-	/* We have a new heir */
-
-	/* Clear the thread dispatch necessary flag */
-	li	r7, 0
-	stb	r7, PER_CPU_DISPATCH_NEEDED(r6)
-	msync
+	/* We may have a new heir */
 
 	/* Read the executing and heir */
 	lwz	r7, PER_CPU_OFFSET_EXECUTING(r6)
 	lwz	r8, PER_CPU_OFFSET_HEIR(r6)
 
+	/*
+	 * Update the executing only if necessary to avoid cache line
+	 * monopolization.
+	 */
+	cmpw	r7, r8
+	beq	check_is_executing
+
 	/* Calculate the heir context pointer */
 	sub	r7, r4, r7
 	add	r4, r8, r7
diff --git a/c/src/lib/libcpu/powerpc/shared/include/powerpc-utility.h b/c/src/lib/libcpu/powerpc/shared/include/powerpc-utility.h
index 331aa58057..b8dc5f40ce 100644
--- a/c/src/lib/libcpu/powerpc/shared/include/powerpc-utility.h
+++ b/c/src/lib/libcpu/powerpc/shared/include/powerpc-utility.h
@@ -8,7 +8,7 @@
  */
 
 /*
- * Copyright (c) 2008-2014 embedded brains GmbH.
+ * Copyright (c) 2008-2015 embedded brains GmbH.
  *
  *  embedded brains GmbH
  *  Dornierstr. 4
@@ -206,6 +206,13 @@ static inline void ppc_synchronize_data(void)
   __asm__ volatile ("sync");
 }
 
+static inline void ppc_light_weight_synchronize(void)
+{
+  RTEMS_COMPILER_MEMORY_BARRIER();
+
+  __asm__ volatile ("lwsync");
+}
+
 static inline void ppc_synchronize_instructions(void)
 {
   RTEMS_COMPILER_MEMORY_BARRIER();