score: Implement forced thread migration

The current implementation of task migration in RTEMS has some
implications with respect to the interrupt latency. It is crucial to
preserve the system invariant that a task can execute on at most one
processor in the system at a time. This is accomplished with a boolean
indicator in the task context. The processor architecture specific
low-level task context switch code will mark that a task context is no
longer executing and waits that the heir context stopped execution
before it restores the heir context and resumes execution of the heir
task. So there is one point in time in which a processor is without a
task. This is essential to avoid cyclic dependencies in case multiple
tasks migrate at once. Otherwise some supervising entity is necessary to
prevent life-locks. Such a global supervisor would lead to scalability
problems so this approach is not used. Currently the thread dispatch is
performed with interrupts disabled. So in case the heir task is
currently executing on another processor then this prolongs the time of
disabled interrupts since one processor has to wait for another
processor to make progress.

It is difficult to avoid this issue with the interrupt latency since
interrupts normally store the context of the interrupted task on its
stack. In case a task is marked as not executing we must not use its
task stack to store such an interrupt context. We cannot use the heir
stack before it stopped execution on another processor. So if we enable
interrupts during this transition we have to provide an alternative task
independent stack for this time frame. This issue needs further
investigation.

11 files changed, 171 insertions, 8 deletions

diff --git a/cpukit/score/cpu/arm/cpu.c b/cpukit/score/cpu/arm/cpu.c
index 24a9249300..91109e4e1d 100644
--- a/cpukit/score/cpu/arm/cpu.c
+++ b/cpukit/score/cpu/arm/cpu.c
@@ -50,6 +50,14 @@
   );
 #endif
 
+#ifdef RTEMS_SMP
+  RTEMS_STATIC_ASSERT(
+    offsetof( Context_Control, is_executing )
+      == ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET,
+    ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET
+  );
+#endif
+
 RTEMS_STATIC_ASSERT(
   sizeof( CPU_Exception_frame ) == ARM_EXCEPTION_FRAME_SIZE,
   ARM_EXCEPTION_FRAME_SIZE
@@ -93,6 +101,10 @@ void _CPU_Context_Initialize(
   the_context->thread_id = (uint32_t) tls_area;
 #endif
 
+#ifdef RTEMS_SMP
+  the_context->is_executing = false;
+#endif
+
   if ( tls_area != NULL ) {
     _TLS_TCB_at_area_begin_initialize( tls_area );
   }
diff --git a/cpukit/score/cpu/arm/cpu_asm.S b/cpukit/score/cpu/arm/cpu_asm.S
index bae7207f2c..f2c4afe39e 100644
--- a/cpukit/score/cpu/arm/cpu_asm.S
+++ b/cpukit/score/cpu/arm/cpu_asm.S
@@ -67,12 +67,32 @@ DEFINE_FUNCTION_ARM(_CPU_Context_switch)
 	str	r3, [r0, #ARM_CONTEXT_CONTROL_THREAD_ID_OFFSET]
 #endif
 
+#ifdef RTEMS_SMP
+	/* Indicate that this context is no longer executing */
+	dmb
+	mov	r3, #0
+	strb	r3, [r0, #ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET]
+#endif
+
 /* Start restoring context */
 _restore:
 #ifdef ARM_MULTILIB_HAS_LOAD_STORE_EXCLUSIVE
 	clrex
 #endif
 
+#ifdef RTEMS_SMP
+	/* Wait for context to stop execution if necessary */
+1:
+	ldrb	r3, [r1, #ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET]
+	cmp	r3, #0
+	bne	1b
+
+	/* Indicate that this context is executing */
+	dmb
+	mov	r3, #1
+	strb	r3, [r1, #ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET]
+#endif
+
 #ifdef ARM_MULTILIB_HAS_THREAD_ID_REGISTER
 	ldr	r3, [r1, #ARM_CONTEXT_CONTROL_THREAD_ID_OFFSET]
 	mcr	p15, 0, r3, c13, c0, 3
diff --git a/cpukit/score/cpu/arm/rtems/score/cpu.h b/cpukit/score/cpu/arm/rtems/score/cpu.h
index cb9dc7c409..dc57a78a67 100644
--- a/cpukit/score/cpu/arm/rtems/score/cpu.h
+++ b/cpukit/score/cpu/arm/rtems/score/cpu.h
@@ -216,6 +216,14 @@
   #define ARM_CONTEXT_CONTROL_D8_OFFSET 48
 #endif
 
+#ifdef RTEMS_SMP
+  #ifdef ARM_MULTILIB_VFP_D32
+    #define ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET 112
+  #else
+    #define ARM_CONTEXT_CONTROL_IS_EXECUTING_OFFSET 48
+  #endif
+#endif
+
 #define ARM_EXCEPTION_FRAME_SIZE 76
 
 #define ARM_EXCEPTION_FRAME_REGISTER_SP_OFFSET 52
@@ -280,6 +288,9 @@ typedef struct {
   uint64_t register_d14;
   uint64_t register_d15;
 #endif
+#ifdef RTEMS_SMP
+  volatile bool is_executing;
+#endif
 } Context_Control;
 
 typedef struct {
@@ -410,6 +421,11 @@ void _CPU_Context_Initialize(
 #define _CPU_Context_Get_SP( _context ) \
   (_context)->register_sp
 
+#ifdef RTEMS_SMP
+  #define _CPU_Context_Get_is_executing( _context ) \
+    (_context)->is_executing
+#endif
+
 #define _CPU_Context_Restart_self( _the_context ) \
    _CPU_Context_restore( (_the_context) );
 
diff --git a/cpukit/score/cpu/i386/cpu.c b/cpukit/score/cpu/i386/cpu.c
index ba7501a246..38b84e6750 100644
--- a/cpukit/score/cpu/i386/cpu.c
+++ b/cpukit/score/cpu/i386/cpu.c
@@ -26,6 +26,24 @@
 #include <rtems/bspIo.h>
 #include <rtems/score/thread.h>
 
+#define I386_ASSERT_OFFSET(field, off) \
+  RTEMS_STATIC_ASSERT( \
+    offsetof(Context_Control, field) \
+      == I386_CONTEXT_CONTROL_ ## off ## _OFFSET, \
+    Context_Control_ ## field \
+  )
+
+I386_ASSERT_OFFSET(eflags, EFLAGS);
+I386_ASSERT_OFFSET(esp, ESP);
+I386_ASSERT_OFFSET(ebp, EBP);
+I386_ASSERT_OFFSET(ebx, EBX);
+I386_ASSERT_OFFSET(esi, ESI);
+I386_ASSERT_OFFSET(edi, EDI);
+
+#ifdef RTEMS_SMP
+  I386_ASSERT_OFFSET(is_executing, IS_EXECUTING);
+#endif
+
 void _CPU_Initialize(void)
 {
 #if CPU_HARDWARE_FP
diff --git a/cpukit/score/cpu/i386/cpu_asm.S b/cpukit/score/cpu/i386/cpu_asm.S
index 73a4c143b4..cc08312f33 100644
--- a/cpukit/score/cpu/i386/cpu_asm.S
+++ b/cpukit/score/cpu/i386/cpu_asm.S
@@ -26,13 +26,12 @@
  * Format of i386 Register structure
  */
 
-.set REG_EFLAGS,  0
-.set REG_ESP,     REG_EFLAGS + 4
-.set REG_EBP,     REG_ESP + 4
-.set REG_EBX,     REG_EBP + 4
-.set REG_ESI,     REG_EBX + 4
-.set REG_EDI,     REG_ESI + 4
-.set SIZE_REGS,   REG_EDI + 4
+.set REG_EFLAGS,  I386_CONTEXT_CONTROL_EFLAGS_OFFSET
+.set REG_ESP,     I386_CONTEXT_CONTROL_ESP_OFFSET
+.set REG_EBP,     I386_CONTEXT_CONTROL_EBP_OFFSET
+.set REG_EBX,     I386_CONTEXT_CONTROL_EBX_OFFSET
+.set REG_ESI,     I386_CONTEXT_CONTROL_ESI_OFFSET
+.set REG_EDI,     I386_CONTEXT_CONTROL_EDI_OFFSET
 
         BEGIN_CODE
 
@@ -58,9 +57,25 @@ SYM (_CPU_Context_switch):
         movl      esi,REG_ESI(eax)         /* save source register */
         movl      edi,REG_EDI(eax)         /* save destination register */
 
+#ifdef RTEMS_SMP
+        /* Indicate that this context is no longer executing */
+        movb      $0, I386_CONTEXT_CONTROL_IS_EXECUTING_OFFSET(eax)
+#endif
+
         movl      HEIRCONTEXT_ARG(esp),eax /* eax = heir threads context */
 
 restore:
+#ifdef RTEMS_SMP
+        /* Wait for context to stop execution if necessary */
+1:
+        movb      I386_CONTEXT_CONTROL_IS_EXECUTING_OFFSET(eax), bl
+        testb     bl, bl
+        jne       1b
+
+        /* Indicate that this context is executing */
+        movb      $1, I386_CONTEXT_CONTROL_IS_EXECUTING_OFFSET(eax)
+#endif
+
         pushl     REG_EFLAGS(eax)          /* push eflags */
         popf                               /* restore eflags */
         movl      REG_ESP(eax),esp         /* restore stack pointer */
diff --git a/cpukit/score/cpu/i386/rtems/score/cpu.h b/cpukit/score/cpu/i386/rtems/score/cpu.h
index a9957cbe62..ba731b0666 100644
--- a/cpukit/score/cpu/i386/rtems/score/cpu.h
+++ b/cpukit/score/cpu/i386/rtems/score/cpu.h
@@ -128,6 +128,17 @@ extern "C" {
 
 #define CPU_PER_CPU_CONTROL_SIZE 0
 
+#define I386_CONTEXT_CONTROL_EFLAGS_OFFSET 0
+#define I386_CONTEXT_CONTROL_ESP_OFFSET 4
+#define I386_CONTEXT_CONTROL_EBP_OFFSET 8
+#define I386_CONTEXT_CONTROL_EBX_OFFSET 12
+#define I386_CONTEXT_CONTROL_ESI_OFFSET 16
+#define I386_CONTEXT_CONTROL_EDI_OFFSET 20
+
+#ifdef RTEMS_SMP
+  #define I386_CONTEXT_CONTROL_IS_EXECUTING_OFFSET 24
+#endif
+
 /* structures */
 
 #ifndef ASM
@@ -147,11 +158,19 @@ typedef struct {
   uint32_t    ebx;      /* extended bx register                      */
   uint32_t    esi;      /* extended source index register            */
   uint32_t    edi;      /* extended destination index flags register */
+#ifdef RTEMS_SMP
+  volatile bool is_executing;
+#endif
 }   Context_Control;
 
 #define _CPU_Context_Get_SP( _context ) \
   (_context)->esp
 
+#ifdef RTEMS_SMP
+  #define _CPU_Context_Get_is_executing( _context ) \
+    (_context)->is_executing
+#endif
+
 /*
  *  FP context save area for the i387 numeric coprocessors.
  */
@@ -435,6 +454,13 @@ uint32_t   _CPU_ISR_Get_level( void );
  */
 
 
+#ifdef RTEMS_SMP
+  #define _I386_Context_Initialize_is_executing( _the_context ) \
+    (_the_context)->is_executing = false
+#else
+  #define _I386_Context_Initialize_is_executing( _the_context )
+#endif
+
 #define _CPU_Context_Initialize( _the_context, _stack_base, _size, \
                                    _isr, _entry_point, _is_fp, _tls_area ) \
   do { \
@@ -449,6 +475,7 @@ uint32_t   _CPU_ISR_Get_level( void );
     *((proc_ptr *)(_stack)) = (_entry_point); \
     (_the_context)->ebp     = (void *) 0; \
     (_the_context)->esp     = (void *) _stack; \
+    _I386_Context_Initialize_is_executing( _the_context ); \
   } while (0)
 
 #define _CPU_Context_Restart_self( _the_context ) \
diff --git a/cpukit/score/cpu/no_cpu/rtems/score/cpu.h b/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
index fbf207ad80..739a6a8a42 100644
--- a/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
+++ b/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
@@ -574,6 +574,18 @@ typedef struct {
      * is the stack pointer.
      */
     uint32_t   stack_pointer;
+
+#ifdef RTEMS_SMP
+    /**
+     * @brief On SMP configurations the thread context must contain a boolean
+     * indicator if this context is executing on a processor.
+     *
+     * This field must be updated during a context switch.  The context switch
+     * to the heir must wait until the heir context indicates that it is no
+     * longer executing on a processor.
+     */
+    volatile bool is_executing;
+#endif
 } Context_Control;
 
 /**
@@ -1582,6 +1594,12 @@ register struct Per_CPU_Control *_CPU_Per_CPU_current asm( "rX" );
   {
     __asm__ volatile ( "" : : : "memory" );
   }
+
+  /**
+   * @brief Macro to return the is executing field of the thread context.
+   */
+  #define _CPU_Context_Get_is_executing( _context ) \
+    ( ( _context )->is_executing )
 #endif
 
 #ifdef __cplusplus
diff --git a/cpukit/score/cpu/powerpc/cpu.c b/cpukit/score/cpu/powerpc/cpu.c
index 3c699f9366..53b4eaa247 100644
--- a/cpukit/score/cpu/powerpc/cpu.c
+++ b/cpukit/score/cpu/powerpc/cpu.c
@@ -53,6 +53,10 @@ PPC_ASSERT_OFFSET(gpr30, GPR30);
 PPC_ASSERT_OFFSET(gpr31, GPR31);
 PPC_ASSERT_OFFSET(gpr2, GPR2);
 
+#ifdef RTEMS_SMP
+  PPC_ASSERT_OFFSET(is_executing, IS_EXECUTING);
+#endif
+
 RTEMS_STATIC_ASSERT(
   sizeof(Context_Control) % PPC_DEFAULT_CACHE_LINE_SIZE == 0,
   ppc_context_size
diff --git a/cpukit/score/cpu/powerpc/rtems/score/cpu.h b/cpukit/score/cpu/powerpc/rtems/score/cpu.h
index 3a51b3112b..18a6770788 100644
--- a/cpukit/score/cpu/powerpc/rtems/score/cpu.h
+++ b/cpukit/score/cpu/powerpc/rtems/score/cpu.h
@@ -302,6 +302,9 @@ typedef struct {
   PPC_GPR_TYPE gpr30;
   PPC_GPR_TYPE gpr31;
   uint32_t gpr2;
+  #ifdef RTEMS_SMP
+    volatile bool is_executing;
+  #endif
   #ifdef __ALTIVEC__
     /*
      * 12 non-volatile vector registers, cache-aligned area for vscr/vrsave
@@ -327,7 +330,7 @@ typedef struct {
   ];
 } Context_Control;
 
-static inline ppc_context *ppc_get_context( Context_Control *context )
+static inline ppc_context *ppc_get_context( const Context_Control *context )
 {
   uintptr_t clsz = PPC_DEFAULT_CACHE_LINE_SIZE;
   uintptr_t mask = clsz - 1;
@@ -338,6 +341,11 @@ static inline ppc_context *ppc_get_context( Context_Control *context )
 
 #define _CPU_Context_Get_SP( _context ) \
   ppc_get_context(_context)->gpr1
+
+#ifdef RTEMS_SMP
+  #define _CPU_Context_Get_is_executing( _context ) \
+    ppc_get_context(_context)->is_executing
+#endif
 #endif /* ASM */
 
 #define PPC_CONTEXT_OFFSET_GPR1 32
@@ -368,6 +376,10 @@ static inline ppc_context *ppc_get_context( Context_Control *context )
 #define PPC_CONTEXT_OFFSET_GPR31 PPC_CONTEXT_GPR_OFFSET( 31 )
 #define PPC_CONTEXT_OFFSET_GPR2 PPC_CONTEXT_GPR_OFFSET( 32 )
 
+#ifdef RTEMS_SMP
+  #define PPC_CONTEXT_OFFSET_IS_EXECUTING (PPC_CONTEXT_GPR_OFFSET( 32 ) + 4)
+#endif
+
 #ifndef ASM
 typedef struct {
     /* The ABIs (PowerOpen/SVR4/EABI) only require saving f14-f31 over
diff --git a/cpukit/score/cpu/sparc/cpu.c b/cpukit/score/cpu/sparc/cpu.c
index 6c124db4d2..d05c511162 100644
--- a/cpukit/score/cpu/sparc/cpu.c
+++ b/cpukit/score/cpu/sparc/cpu.c
@@ -67,6 +67,10 @@ SPARC_ASSERT_OFFSET(o7, O7);
 SPARC_ASSERT_OFFSET(psr, PSR);
 SPARC_ASSERT_OFFSET(isr_dispatch_disable, ISR_DISPATCH_DISABLE_STACK);
 
+#if defined(RTEMS_SMP)
+SPARC_ASSERT_OFFSET(is_executing, SPARC_CONTEXT_CONTROL_IS_EXECUTING);
+#endif
+
 /*
  *  This initializes the set of opcodes placed in each trap
  *  table entry.  The routine which installs a handler is responsible
@@ -326,6 +330,10 @@ void _CPU_Context_Initialize(
    */
     the_context->isr_dispatch_disable = 0;
 
+#if defined(RTEMS_SMP)
+  the_context->is_executing = false;
+#endif
+
   if ( tls_area != NULL ) {
     void *tcb = _TLS_TCB_after_TLS_block_initialize( tls_area );
 
diff --git a/cpukit/score/cpu/sparc/rtems/score/cpu.h b/cpukit/score/cpu/sparc/rtems/score/cpu.h
index 50da44cf4b..7bcdbd9b37 100644
--- a/cpukit/score/cpu/sparc/rtems/score/cpu.h
+++ b/cpukit/score/cpu/sparc/rtems/score/cpu.h
@@ -473,6 +473,10 @@ typedef struct {
    * SPARC CPU models at high interrupt rates.
    */
   uint32_t   isr_dispatch_disable;
+
+#if defined(RTEMS_SMP)
+  volatile bool is_executing;
+#endif
 } Context_Control;
 
 /**
@@ -483,6 +487,11 @@ typedef struct {
 #define _CPU_Context_Get_SP( _context ) \
   (_context)->o6_sp
 
+#ifdef RTEMS_SMP
+  #define _CPU_Context_Get_is_executing( _context ) \
+    (_context)->is_executing
+#endif
+
 #endif /* ASM */
 
 /*
@@ -538,6 +547,10 @@ typedef struct {
 /** This macro defines an offset into the context for use in assembly. */
 #define ISR_DISPATCH_DISABLE_STACK_OFFSET 0x54
 
+#if defined(RTEMS_SMP)
+  #define SPARC_CONTEXT_CONTROL_IS_EXECUTING_OFFSET 0x58
+#endif
+
 /** This defines the size of the context area for use in assembly. */
 #define CONTEXT_CONTROL_SIZE 0x68