score: Implement forced thread migration

The current implementation of task migration in RTEMS has some
implications with respect to the interrupt latency. It is crucial to
preserve the system invariant that a task can execute on at most one
processor in the system at a time. This is accomplished with a boolean
indicator in the task context. The processor architecture specific
low-level task context switch code will mark that a task context is no
longer executing and waits that the heir context stopped execution
before it restores the heir context and resumes execution of the heir
task. So there is one point in time in which a processor is without a
task. This is essential to avoid cyclic dependencies in case multiple
tasks migrate at once. Otherwise some supervising entity is necessary to
prevent life-locks. Such a global supervisor would lead to scalability
problems so this approach is not used. Currently the thread dispatch is
performed with interrupts disabled. So in case the heir task is
currently executing on another processor then this prolongs the time of
disabled interrupts since one processor has to wait for another
processor to make progress.

It is difficult to avoid this issue with the interrupt latency since
interrupts normally store the context of the interrupted task on its
stack. In case a task is marked as not executing we must not use its
task stack to store such an interrupt context. We cannot use the heir
stack before it stopped execution on another processor. So if we enable
interrupts during this transition we have to provide an alternative task
independent stack for this time frame. This issue needs further
investigation.

1 files changed, 46 insertions, 5 deletions

diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 7f063eafb8..e232674c30 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -56,6 +56,8 @@ extern "C" {
 typedef struct Thread_Control_struct Thread_Control;
 #endif
 
+struct Scheduler_Context;
+
 /**
  *  @defgroup PerCPU RTEMS Per CPU Information
  *
@@ -268,13 +270,46 @@ typedef struct Per_CPU_Control {
    */
   volatile uint32_t thread_dispatch_disable_level;
 
-  /** This is set to true when this CPU needs to run the dispatcher. */
+  /**
+   * @brief This is set to true when this processor needs to run the
+   * dispatcher.
+   *
+   * It is volatile since interrupts may alter this flag.
+   *
+   * This field is not protected by a lock.  There are two writers after
+   * multitasking start.  The scheduler owning this processor sets this
+   * indicator to true, after it updated the heir field.  This processor sets
+   * this indicator to false, before it reads the heir.  This field is used in
+   * combination with the heir field.
+   *
+   * @see _Thread_Get_heir_and_make_it_executing().
+   */
   volatile bool dispatch_necessary;
 
-  /** This is the thread executing on this CPU. */
+  /**
+   * @brief This is the thread executing on this processor.
+   *
+   * This field is not protected by a lock.  The only writer is this processor.
+   *
+   * On SMP configurations a thread may be registered as executing on more than
+   * one processor in case a thread migration is in progress.  On SMP
+   * configurations use _Thread_Is_executing_on_a_processor() to figure out if
+   * a thread context is executing on a processor.
+   */
   Thread_Control *executing;
 
-  /** This is the heir thread for this this CPU. */
+  /**
+   * @brief This is the heir thread for this processor.
+   *
+   * This field is not protected by a lock.  The only writer after multitasking
+   * start is the scheduler owning this processor.  This processor will set the
+   * dispatch necessary indicator to false, before it reads the heir.  This
+   * field is used in combination with the dispatch necessary indicator.
+   *
+   * A thread can be a heir on at most one processor in the system.
+   *
+   * @see _Thread_Get_heir_and_make_it_executing().
+   */
   Thread_Control *heir;
 
   /** This is the time of the last context switch on this CPU. */
@@ -282,11 +317,12 @@ typedef struct Per_CPU_Control {
 
   #if defined( RTEMS_SMP )
     /**
-     * @brief This lock protects the dispatch_necessary, executing, heir and
-     * message fields.
+     * @brief This lock protects some parts of the low-level thread dispatching.
      *
      * We must use a ticket lock here since we cannot transport a local context
      * through the context switch.
+     *
+     * @see _Thread_Dispatch().
      */
     SMP_ticket_lock_Control Lock;
 
@@ -310,6 +346,11 @@ typedef struct Per_CPU_Control {
     Atomic_Ulong message;
 
     /**
+     * @brief The scheduler context of the scheduler owning this processor.
+     */
+    const struct Scheduler_Context *scheduler_context;
+
+    /**
      * @brief Indicates the current state of the CPU.
      *
      * This field is protected by the _Per_CPU_State_lock lock.