score: Fix scheduler helping protocol

Account for priority changes of threads executing in a foreign
partition.  Exchange idle threads in case a victim node uses an idle
thread and the new scheduled node needs an idle thread.

3 files changed, 202 insertions, 104 deletions

diff --git a/cpukit/score/include/rtems/score/mrspimpl.h b/cpukit/score/include/rtems/score/mrspimpl.h
index c1e05e4c96..c40f41f716 100644
--- a/cpukit/score/include/rtems/score/mrspimpl.h
+++ b/cpukit/score/include/rtems/score/mrspimpl.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 embedded brains GmbH.  All rights reserved.
+ * Copyright (c) 2014-2015 embedded brains GmbH.  All rights reserved.
  *
  *  embedded brains GmbH
  *  Dornierstr. 4
@@ -230,7 +230,7 @@ RTEMS_INLINE_ROUTINE MRSP_Status _MRSP_Obtain(
 )
 {
   MRSP_Status status;
-  const Scheduler_Control *scheduler = _Scheduler_Get( executing );
+  const Scheduler_Control *scheduler = _Scheduler_Get_own( executing );
   uint32_t scheduler_index = _Scheduler_Get_index( scheduler );
   Priority_Control initial_priority = executing->current_priority;
   Priority_Control ceiling_priority =
diff --git a/cpukit/score/include/rtems/score/schedulerimpl.h b/cpukit/score/include/rtems/score/schedulerimpl.h
index d11b2c55c8..212bace075 100644
--- a/cpukit/score/include/rtems/score/schedulerimpl.h
+++ b/cpukit/score/include/rtems/score/schedulerimpl.h
@@ -10,7 +10,7 @@
 /*
  *  Copyright (C) 2010 Gedare Bloom.
  *  Copyright (C) 2011 On-Line Applications Research Corporation (OAR).
- *  Copyright (c) 2014 embedded brains GmbH
+ *  Copyright (c) 2014-2015 embedded brains GmbH
  *
  *  The license and distribution terms for this file may be
  *  found in the file LICENSE in this distribution or at
@@ -950,6 +950,26 @@ void _Scheduler_Thread_change_resource_root(
   Thread_Control *root
 );
 
+RTEMS_INLINE_ROUTINE void _Scheduler_Set_idle_thread(
+  Scheduler_Node *node,
+  Thread_Control *idle
+)
+{
+  _Assert(
+    node->help_state == SCHEDULER_HELP_ACTIVE_OWNER
+      || node->help_state == SCHEDULER_HELP_ACTIVE_RIVAL
+  );
+  _Assert( _Scheduler_Node_get_idle( node ) == NULL );
+  _Assert(
+    _Scheduler_Node_get_owner( node ) == _Scheduler_Node_get_user( node )
+  );
+
+  _Scheduler_Thread_set_node( idle, node );
+
+  _Scheduler_Node_set_user( node, idle );
+  node->idle = idle;
+}
+
 /**
  * @brief Use an idle thread for this scheduler node.
  *
@@ -970,45 +990,44 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Scheduler_Use_idle_thread(
 {
   Thread_Control *idle = ( *get_idle_thread )( context );
 
-  _Assert(
-    node->help_state == SCHEDULER_HELP_ACTIVE_OWNER
-      || node->help_state == SCHEDULER_HELP_ACTIVE_RIVAL
-  );
-  _Assert( _Scheduler_Node_get_idle( node ) == NULL );
-  _Assert(
-    _Scheduler_Node_get_owner( node ) == _Scheduler_Node_get_user( node )
-  );
-
-  _Scheduler_Thread_set_node( idle, node );
-
-  _Scheduler_Node_set_user( node, idle );
-  node->idle = idle;
+  _Scheduler_Set_idle_thread( node, idle );
 
   return idle;
 }
 
+typedef enum {
+  SCHEDULER_TRY_TO_SCHEDULE_DO_SCHEDULE,
+  SCHEDULER_TRY_TO_SCHEDULE_DO_IDLE_EXCHANGE,
+  SCHEDULER_TRY_TO_SCHEDULE_DO_BLOCK
+} Scheduler_Try_to_schedule_action;
+
 /**
  * @brief Try to schedule this scheduler node.
  *
  * @param[in] context The scheduler instance context.
  * @param[in] node The node which wants to get scheduled.
+ * @param[in] idle A potential idle thread used by a potential victim node.
  * @param[in] get_idle_thread Function to get an idle thread.
  *
  * @retval true This node can be scheduled.
  * @retval false Otherwise.
  */
-RTEMS_INLINE_ROUTINE bool _Scheduler_Try_to_schedule_node(
+RTEMS_INLINE_ROUTINE Scheduler_Try_to_schedule_action
+_Scheduler_Try_to_schedule_node(
   Scheduler_Context         *context,
   Scheduler_Node            *node,
+  Thread_Control            *idle,
   Scheduler_Get_idle_thread  get_idle_thread
 )
 {
-  bool schedule;
+  Scheduler_Try_to_schedule_action action;
   Thread_Control *owner;
   Thread_Control *user;
 
+  action = SCHEDULER_TRY_TO_SCHEDULE_DO_SCHEDULE;
+
   if ( node->help_state == SCHEDULER_HELP_YOURSELF ) {
-    return true;
+    return action;
   }
 
   owner = _Scheduler_Node_get_owner( node );
@@ -1018,32 +1037,33 @@ RTEMS_INLINE_ROUTINE bool _Scheduler_Try_to_schedule_node(
     if ( user->Scheduler.state == THREAD_SCHEDULER_READY ) {
       _Scheduler_Thread_set_scheduler_and_node( user, node, owner );
     } else if ( owner->Scheduler.state == THREAD_SCHEDULER_BLOCKED ) {
-      _Scheduler_Use_idle_thread( context, node, get_idle_thread );
+      if ( idle != NULL ) {
+        action = SCHEDULER_TRY_TO_SCHEDULE_DO_IDLE_EXCHANGE;
+      } else {
+        _Scheduler_Use_idle_thread( context, node, get_idle_thread );
+      }
     } else {
       _Scheduler_Node_set_user( node, owner );
     }
-
-    schedule = true;
   } else if ( node->help_state == SCHEDULER_HELP_ACTIVE_OWNER ) {
     if ( user->Scheduler.state == THREAD_SCHEDULER_READY ) {
       _Scheduler_Thread_set_scheduler_and_node( user, node, owner );
+    } else if ( idle != NULL ) {
+      action = SCHEDULER_TRY_TO_SCHEDULE_DO_IDLE_EXCHANGE;
     } else {
       _Scheduler_Use_idle_thread( context, node, get_idle_thread );
     }
-
-    schedule = true;
   } else {
     _Assert( node->help_state == SCHEDULER_HELP_PASSIVE );
 
     if ( user->Scheduler.state == THREAD_SCHEDULER_READY ) {
       _Scheduler_Thread_set_scheduler_and_node( user, node, owner );
-      schedule = true;
     } else {
-      schedule = false;
+      action = SCHEDULER_TRY_TO_SCHEDULE_DO_BLOCK;
     }
   }
 
-  return schedule;
+  return action;
 }
 
 /**
@@ -1078,6 +1098,20 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Scheduler_Release_idle_thread(
   return idle;
 }
 
+RTEMS_INLINE_ROUTINE void _Scheduler_Exchange_idle_thread(
+  Scheduler_Node *needs_idle,
+  Scheduler_Node *uses_idle,
+  Thread_Control *idle
+)
+{
+  uses_idle->idle = NULL;
+  _Scheduler_Node_set_user(
+    uses_idle,
+    _Scheduler_Node_get_owner( uses_idle )
+  );
+  _Scheduler_Set_idle_thread( needs_idle, idle );
+}
+
 /**
  * @brief Block this scheduler node.
  *
diff --git a/cpukit/score/include/rtems/score/schedulersmpimpl.h b/cpukit/score/include/rtems/score/schedulersmpimpl.h
index e41c7372c0..a395f2c0ba 100644
--- a/cpukit/score/include/rtems/score/schedulersmpimpl.h
+++ b/cpukit/score/include/rtems/score/schedulersmpimpl.h
@@ -7,7 +7,7 @@
  */
 
 /*
- * Copyright (c) 2013-2014 embedded brains GmbH.  All rights reserved.
+ * Copyright (c) 2013-2015 embedded brains GmbH.  All rights reserved.
  *
  *  embedded brains GmbH
  *  Dornierstr. 4
@@ -532,41 +532,76 @@ static inline Thread_Control *_Scheduler_SMP_Enqueue_to_scheduled(
   Scheduler_SMP_Allocate_processor  allocate_processor
 )
 {
-  Thread_Control *user = _Scheduler_Node_get_user( node );
-  Thread_Control *lowest_scheduled_user =
-    _Scheduler_Node_get_user( lowest_scheduled );
   Thread_Control *needs_help;
-  Thread_Control *idle;
+  Scheduler_Try_to_schedule_action action;
 
-  _Scheduler_SMP_Node_change_state(
-    _Scheduler_SMP_Node_downcast( lowest_scheduled ),
-    SCHEDULER_SMP_NODE_READY
-  );
-  _Scheduler_Thread_change_state(
-    lowest_scheduled_user,
-    THREAD_SCHEDULER_READY
-  );
-
-  _Scheduler_Thread_set_node( user, node );
-
-  _Scheduler_SMP_Allocate_processor(
+  action = _Scheduler_Try_to_schedule_node(
     context,
     node,
-    lowest_scheduled,
-    allocate_processor
+    _Scheduler_Node_get_idle( lowest_scheduled ),
+    _Scheduler_SMP_Get_idle_thread
   );
 
-  ( *insert_scheduled )( context, node );
-  ( *move_from_scheduled_to_ready )( context, lowest_scheduled );
+  if ( action == SCHEDULER_TRY_TO_SCHEDULE_DO_SCHEDULE ) {
+    Thread_Control *lowest_scheduled_user =
+      _Scheduler_Node_get_user( lowest_scheduled );
+    Thread_Control *idle;
 
-  idle = _Scheduler_Release_idle_thread(
-    context,
-    lowest_scheduled,
-    _Scheduler_SMP_Release_idle_thread
-  );
-  if ( idle == NULL ) {
-    needs_help = lowest_scheduled_user;
+    _Scheduler_SMP_Node_change_state(
+      _Scheduler_SMP_Node_downcast( lowest_scheduled ),
+      SCHEDULER_SMP_NODE_READY
+    );
+    _Scheduler_Thread_change_state(
+      lowest_scheduled_user,
+      THREAD_SCHEDULER_READY
+    );
+
+    _Scheduler_SMP_Allocate_processor(
+      context,
+      node,
+      lowest_scheduled,
+      allocate_processor
+    );
+
+    ( *insert_scheduled )( context, node );
+    ( *move_from_scheduled_to_ready )( context, lowest_scheduled );
+
+    idle = _Scheduler_Release_idle_thread(
+      context,
+      lowest_scheduled,
+      _Scheduler_SMP_Release_idle_thread
+    );
+    if ( idle == NULL ) {
+      needs_help = lowest_scheduled_user;
+    } else {
+      needs_help = NULL;
+    }
+  } else if ( action == SCHEDULER_TRY_TO_SCHEDULE_DO_IDLE_EXCHANGE ) {
+    _Scheduler_SMP_Node_change_state(
+      _Scheduler_SMP_Node_downcast( lowest_scheduled ),
+      SCHEDULER_SMP_NODE_READY
+    );
+    _Scheduler_SMP_Node_change_state(
+      _Scheduler_SMP_Node_downcast( node ),
+      SCHEDULER_SMP_NODE_SCHEDULED
+    );
+
+    ( *insert_scheduled )( context, node );
+    ( *move_from_scheduled_to_ready )( context, lowest_scheduled );
+
+    _Scheduler_Exchange_idle_thread(
+      node,
+      lowest_scheduled,
+      _Scheduler_Node_get_idle( lowest_scheduled )
+    );
+
+    needs_help = NULL;
   } else {
+    _Assert( action == SCHEDULER_TRY_TO_SCHEDULE_DO_BLOCK );
+    _Scheduler_SMP_Node_change_state(
+      _Scheduler_SMP_Node_downcast( node ),
+      SCHEDULER_SMP_NODE_BLOCKED
+    );
     needs_help = NULL;
   }
 
@@ -660,7 +695,7 @@ static inline Thread_Control *_Scheduler_SMP_Enqueue_scheduled_ordered(
 {
   Thread_Control *needs_help;
 
-  while ( true ) {
+  do {
     Scheduler_Node *highest_ready = ( *get_highest_ready )( context, node );
 
     /*
@@ -671,55 +706,80 @@ static inline Thread_Control *_Scheduler_SMP_Enqueue_scheduled_ordered(
       ( *insert_scheduled )( context, node );
 
       needs_help = NULL;
+    } else {
+      Scheduler_Try_to_schedule_action action;
 
-      break;
-    } else if (
-      _Scheduler_Try_to_schedule_node(
+      action = _Scheduler_Try_to_schedule_node(
         context,
         highest_ready,
+        _Scheduler_Node_get_idle( node ),
         _Scheduler_SMP_Get_idle_thread
-      )
-    ) {
-      Thread_Control *user = _Scheduler_Node_get_user( node );
-      Thread_Control *idle;
-
-      _Scheduler_SMP_Node_change_state(
-        _Scheduler_SMP_Node_downcast( node ),
-        SCHEDULER_SMP_NODE_READY
       );
-      _Scheduler_Thread_change_state( user, THREAD_SCHEDULER_READY );
 
-      _Scheduler_SMP_Allocate_processor(
-        context,
-        highest_ready,
-        node,
-        allocate_processor
-      );
+      if ( action == SCHEDULER_TRY_TO_SCHEDULE_DO_SCHEDULE ) {
+        Thread_Control *user = _Scheduler_Node_get_user( node );
+        Thread_Control *idle;
 
-      ( *insert_ready )( context, node );
-      ( *move_from_ready_to_scheduled )( context, highest_ready );
+        _Scheduler_SMP_Node_change_state(
+          _Scheduler_SMP_Node_downcast( node ),
+          SCHEDULER_SMP_NODE_READY
+        );
+        _Scheduler_Thread_change_state( user, THREAD_SCHEDULER_READY );
+
+        _Scheduler_SMP_Allocate_processor(
+          context,
+          highest_ready,
+          node,
+          allocate_processor
+        );
+
+        ( *insert_ready )( context, node );
+        ( *move_from_ready_to_scheduled )( context, highest_ready );
+
+        idle = _Scheduler_Release_idle_thread(
+          context,
+          node,
+          _Scheduler_SMP_Release_idle_thread
+        );
+        if ( idle == NULL ) {
+          needs_help = user;
+        } else {
+          needs_help = NULL;
+        }
+      } else if ( action == SCHEDULER_TRY_TO_SCHEDULE_DO_IDLE_EXCHANGE ) {
+        _Scheduler_SMP_Node_change_state(
+          _Scheduler_SMP_Node_downcast( node ),
+          SCHEDULER_SMP_NODE_READY
+        );
+        _Scheduler_SMP_Node_change_state(
+          _Scheduler_SMP_Node_downcast( highest_ready ),
+          SCHEDULER_SMP_NODE_SCHEDULED
+        );
+
+        ( *insert_ready )( context, node );
+        ( *move_from_ready_to_scheduled )( context, highest_ready );
+
+        _Scheduler_Exchange_idle_thread(
+          highest_ready,
+          node,
+          _Scheduler_Node_get_idle( node )
+        );
 
-      idle = _Scheduler_Release_idle_thread(
-        context,
-        node,
-        _Scheduler_SMP_Release_idle_thread
-      );
-      if ( idle == NULL ) {
-        needs_help = user;
-      } else {
         needs_help = NULL;
-      }
+      } else {
+        _Assert( action == SCHEDULER_TRY_TO_SCHEDULE_DO_BLOCK );
 
-      break;
-    } else {
-      _Scheduler_SMP_Node_change_state(
-        _Scheduler_SMP_Node_downcast( highest_ready ),
-        SCHEDULER_SMP_NODE_BLOCKED
-      );
+        _Scheduler_SMP_Node_change_state(
+          _Scheduler_SMP_Node_downcast( highest_ready ),
+          SCHEDULER_SMP_NODE_BLOCKED
+        );
 
-      ( *extract_from_ready )( context, highest_ready );
+        ( *extract_from_ready )( context, highest_ready );
+
+        continue;
+      }
     }
-  }
+  } while ( false );
 
   return needs_help;
 }
@@ -740,16 +800,18 @@ static inline void _Scheduler_SMP_Schedule_highest_ready(
   Scheduler_SMP_Allocate_processor  allocate_processor
 )
 {
-  while ( true ) {
+  do {
     Scheduler_Node *highest_ready = ( *get_highest_ready )( context, victim );
+    Scheduler_Try_to_schedule_action action;
 
-    if (
-      _Scheduler_Try_to_schedule_node(
-        context,
-        highest_ready,
-        _Scheduler_SMP_Get_idle_thread
-      )
-    ) {
+    action = _Scheduler_Try_to_schedule_node(
+      context,
+      highest_ready,
+      NULL,
+      _Scheduler_SMP_Get_idle_thread
+    );
+
+    if ( action == SCHEDULER_TRY_TO_SCHEDULE_DO_SCHEDULE ) {
       _Scheduler_SMP_Allocate_processor(
         context,
         highest_ready,
@@ -758,17 +820,19 @@ static inline void _Scheduler_SMP_Schedule_highest_ready(
       );
 
       ( *move_from_ready_to_scheduled )( context, highest_ready );
-
-      break;
     } else {
+      _Assert( action == SCHEDULER_TRY_TO_SCHEDULE_DO_BLOCK );
+
       _Scheduler_SMP_Node_change_state(
         _Scheduler_SMP_Node_downcast( highest_ready ),
         SCHEDULER_SMP_NODE_BLOCKED
       );
 
       ( *extract_from_ready )( context, highest_ready );
+
+      continue;
     }
-  }
+  } while ( false );
 }
 
 /**