score: _Chain_Insert_ordered_unprotected()

Change the chain order relation to use a directly specified left hand
side value.  This is similar to _RBTree_Insert_inline() and helps the
compiler to better optimize the code.

8 files changed, 137 insertions, 65 deletions

diff --git a/cpukit/score/include/rtems/score/chainimpl.h b/cpukit/score/include/rtems/score/chainimpl.h
index 21c0e6b026..c94c051198 100644
--- a/cpukit/score/include/rtems/score/chainimpl.h
+++ b/cpukit/score/include/rtems/score/chainimpl.h
@@ -830,14 +830,14 @@ RTEMS_INLINE_ROUTINE bool _Chain_Get_with_empty_check_unprotected(
 /**
  * @brief Chain node order.
  *
- * @param[in] left The left node.
- * @param[in] right The right node.
+ * @param[in] left The left hand side.
+ * @param[in] right The right hand side.
  *
  * @retval true According to the order the left node precedes the right node.
  * @retval false Otherwise.
  */
 typedef bool ( *Chain_Node_order )(
-  const Chain_Node *left,
+  const void       *left,
   const Chain_Node *right
 );
 
@@ -848,20 +848,25 @@ typedef bool ( *Chain_Node_order )(
  * relation holds for all nodes from the head up to the inserted node.  Nodes
  * after the inserted node are not moved.
  *
- * @param[in,out] chain The chain.
- * @param[in,out] to_insert The node to insert.
+ * @param[in] the_chain The chain.
+ * @param[in] to_insert The node to insert.
+ * @param[in] left The left hand side passed to the order relation.  It must
+ *   correspond to the node to insert.  The separate left hand side parameter
+ *   may help the compiler to generate better code if it is stored in a local
+ *   variable.
  * @param[in] order The order relation.
  */
 RTEMS_INLINE_ROUTINE void _Chain_Insert_ordered_unprotected(
-  Chain_Control *chain,
-  Chain_Node *to_insert,
-  Chain_Node_order order
+  Chain_Control    *the_chain,
+  Chain_Node       *to_insert,
+  const void       *left,
+  Chain_Node_order  order
 )
 {
-  const Chain_Node *tail = _Chain_Immutable_tail( chain );
-  Chain_Node *next = _Chain_First( chain );
+  const Chain_Node *tail = _Chain_Immutable_tail( the_chain );
+  Chain_Node *next = _Chain_First( the_chain );
 
-  while ( next != tail && !( *order )( to_insert, next ) ) {
+  while ( next != tail && !( *order )( left, next ) ) {
     next = _Chain_Next( next );
   }
 
diff --git a/cpukit/score/include/rtems/score/schedulerprioritysmpimpl.h b/cpukit/score/include/rtems/score/schedulerprioritysmpimpl.h
index f37414c7a8..073a7ade06 100644
--- a/cpukit/score/include/rtems/score/schedulerprioritysmpimpl.h
+++ b/cpukit/score/include/rtems/score/schedulerprioritysmpimpl.h
@@ -88,19 +88,23 @@ static inline void _Scheduler_priority_SMP_Move_from_ready_to_scheduled(
   Scheduler_Node    *ready_to_scheduled
 )
 {
-  Scheduler_priority_SMP_Context *self =
-    _Scheduler_priority_SMP_Get_self( context );
-  Scheduler_priority_SMP_Node *node =
-    _Scheduler_priority_SMP_Node_downcast( ready_to_scheduled );
+  Scheduler_priority_SMP_Context *self;
+  Scheduler_priority_SMP_Node    *node;
+  Priority_Control                priority;
+
+  self = _Scheduler_priority_SMP_Get_self( context );
+  node = _Scheduler_priority_SMP_Node_downcast( ready_to_scheduled );
 
   _Scheduler_priority_Ready_queue_extract(
     &node->Base.Base.Node.Chain,
     &node->Ready_queue,
     &self->Bit_map
   );
+  priority = node->Base.priority;
   _Chain_Insert_ordered_unprotected(
     &self->Base.Scheduled,
     &node->Base.Base.Node.Chain,
+    &priority,
     _Scheduler_SMP_Insert_priority_fifo_order
   );
 }
diff --git a/cpukit/score/include/rtems/score/schedulersimpleimpl.h b/cpukit/score/include/rtems/score/schedulersimpleimpl.h
index c94f9b3bdb..ec74cdc586 100644
--- a/cpukit/score/include/rtems/score/schedulersimpleimpl.h
+++ b/cpukit/score/include/rtems/score/schedulersimpleimpl.h
@@ -39,49 +39,63 @@ RTEMS_INLINE_ROUTINE Scheduler_simple_Context *
 }
 
 RTEMS_INLINE_ROUTINE bool _Scheduler_simple_Insert_priority_lifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
-  const Thread_Control *thread_to_insert = (const Thread_Control *) to_insert;
-  const Thread_Control *thread_next = (const Thread_Control *) next;
+  const Priority_Control *priority_to_insert;
+  const Thread_Control   *thread_next;
 
-  return _Thread_Get_priority( thread_to_insert )
-    <= _Thread_Get_priority( thread_next );
+  priority_to_insert = (const Priority_Control *) to_insert;
+  thread_next = (const Thread_Control *) next;
+
+  return *priority_to_insert <= _Thread_Get_priority( thread_next );
 }
 
 RTEMS_INLINE_ROUTINE bool _Scheduler_simple_Insert_priority_fifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
-  const Thread_Control *thread_to_insert = (const Thread_Control *) to_insert;
-  const Thread_Control *thread_next = (const Thread_Control *) next;
+  const Priority_Control *priority_to_insert;
+  const Thread_Control   *thread_next;
+
+  priority_to_insert = (const Priority_Control *) to_insert;
+  thread_next = (const Thread_Control *) next;
 
-  return _Thread_Get_priority( thread_to_insert )
-    < _Thread_Get_priority( thread_next );
+  return *priority_to_insert < _Thread_Get_priority( thread_next );
 }
 
 RTEMS_INLINE_ROUTINE void _Scheduler_simple_Insert_priority_lifo(
-  Chain_Control *chain,
+  Chain_Control  *chain,
   Thread_Control *to_insert
 )
 {
+  Priority_Control priority_to_insert;
+
+  priority_to_insert = _Thread_Get_priority( to_insert );
+
   _Chain_Insert_ordered_unprotected(
     chain,
     &to_insert->Object.Node,
+    &priority_to_insert,
     _Scheduler_simple_Insert_priority_lifo_order
   );
 }
 
 RTEMS_INLINE_ROUTINE void _Scheduler_simple_Insert_priority_fifo(
-  Chain_Control *chain,
+  Chain_Control  *chain,
   Thread_Control *to_insert
 )
 {
+  Priority_Control priority_to_insert;
+
+  priority_to_insert = _Thread_Get_priority( to_insert );
+
   _Chain_Insert_ordered_unprotected(
     chain,
     &to_insert->Object.Node,
+    &priority_to_insert,
     _Scheduler_simple_Insert_priority_fifo_order
   );
 }
diff --git a/cpukit/score/include/rtems/score/schedulersmpimpl.h b/cpukit/score/include/rtems/score/schedulersmpimpl.h
index 3194ee39a0..896b1306ab 100644
--- a/cpukit/score/include/rtems/score/schedulersmpimpl.h
+++ b/cpukit/score/include/rtems/score/schedulersmpimpl.h
@@ -352,29 +352,31 @@ static inline void _Scheduler_SMP_Do_nothing_register_idle(
 }
 
 static inline bool _Scheduler_SMP_Insert_priority_lifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
-  const Scheduler_SMP_Node *node_to_insert =
-    (const Scheduler_SMP_Node *) to_insert;
-  const Scheduler_SMP_Node *node_next =
-    (const Scheduler_SMP_Node *) next;
+  const Priority_Control   *priority_to_insert;
+  const Scheduler_SMP_Node *node_next;
 
-  return node_to_insert->priority <= node_next->priority;
+  priority_to_insert = (const Priority_Control *) to_insert;
+  node_next = (const Scheduler_SMP_Node *) next;
+
+  return *priority_to_insert <= node_next->priority;
 }
 
 static inline bool _Scheduler_SMP_Insert_priority_fifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
-  const Scheduler_SMP_Node *node_to_insert =
-    (const Scheduler_SMP_Node *) to_insert;
-  const Scheduler_SMP_Node *node_next =
-    (const Scheduler_SMP_Node *) next;
+  const Priority_Control   *priority_to_insert;
+  const Scheduler_SMP_Node *node_next;
+
+  priority_to_insert = (const Priority_Control *) to_insert;
+  node_next = (const Scheduler_SMP_Node *) next;
 
-  return node_to_insert->priority < node_next->priority;
+  return *priority_to_insert < node_next->priority;
 }
 
 static inline Scheduler_SMP_Context *_Scheduler_SMP_Get_self(
@@ -719,12 +721,14 @@ static inline bool _Scheduler_SMP_Enqueue_ordered(
   Scheduler_SMP_Allocate_processor    allocate_processor
 )
 {
-  bool            needs_help;
-  Scheduler_Node *lowest_scheduled;
+  bool              needs_help;
+  Scheduler_Node   *lowest_scheduled;
+  Priority_Control  node_priority;
 
   lowest_scheduled = ( *get_lowest_scheduled )( context, node );
+  node_priority = _Scheduler_SMP_Node_priority( node );
 
-  if ( ( *order )( &node->Node.Chain, &lowest_scheduled->Node.Chain ) ) {
+  if ( ( *order )( &node_priority, &lowest_scheduled->Node.Chain ) ) {
     _Scheduler_SMP_Enqueue_to_scheduled(
       context,
       node,
@@ -776,8 +780,10 @@ static inline bool _Scheduler_SMP_Enqueue_scheduled_ordered(
   while ( true ) {
     Scheduler_Node                   *highest_ready;
     Scheduler_Try_to_schedule_action  action;
+    Priority_Control                  node_priority;
 
     highest_ready = ( *get_highest_ready )( context, node );
+    node_priority = _Scheduler_SMP_Node_priority( node );
 
     /*
      * The node has been extracted from the scheduled chain.  We have to place
@@ -785,7 +791,7 @@ static inline bool _Scheduler_SMP_Enqueue_scheduled_ordered(
      */
     if (
       node->sticky_level > 0
-        && ( *order )( &node->Node.Chain, &highest_ready->Node.Chain )
+        && ( *order )( &node_priority, &highest_ready->Node.Chain )
     ) {
       ( *insert_scheduled )( context, node );
 
@@ -1165,11 +1171,16 @@ static inline void _Scheduler_SMP_Insert_scheduled_lifo(
   Scheduler_Node    *node_to_insert
 )
 {
-  Scheduler_SMP_Context *self = _Scheduler_SMP_Get_self( context );
+  Scheduler_SMP_Context *self;
+  Priority_Control       priority_to_insert;
+
+  self = _Scheduler_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( node_to_insert );
 
   _Chain_Insert_ordered_unprotected(
     &self->Scheduled,
     &node_to_insert->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_lifo_order
   );
 }
@@ -1179,11 +1190,16 @@ static inline void _Scheduler_SMP_Insert_scheduled_fifo(
   Scheduler_Node    *node_to_insert
 )
 {
-  Scheduler_SMP_Context *self = _Scheduler_SMP_Get_self( context );
+  Scheduler_SMP_Context *self;
+  Priority_Control       priority_to_insert;
+
+  self = _Scheduler_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( node_to_insert );
 
   _Chain_Insert_ordered_unprotected(
     &self->Scheduled,
     &node_to_insert->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_fifo_order
   );
 }
@@ -1214,7 +1230,11 @@ static inline bool _Scheduler_SMP_Ask_for_help(
     node_state = _Scheduler_SMP_Node_state( node );
 
     if ( node_state == SCHEDULER_SMP_NODE_BLOCKED ) {
-      if ( ( *order )( &node->Node.Chain, &lowest_scheduled->Node.Chain ) ) {
+      Priority_Control node_priority;
+
+      node_priority = _Scheduler_SMP_Node_priority( node );
+
+      if ( ( *order )( &node_priority, &lowest_scheduled->Node.Chain ) ) {
         _Thread_Scheduler_cancel_need_for_help(
           thread,
           _Thread_Get_CPU( thread )
diff --git a/cpukit/score/src/coremsginsert.c b/cpukit/score/src/coremsginsert.c
index 4613f50436..a9aa1e584e 100644
--- a/cpukit/score/src/coremsginsert.c
+++ b/cpukit/score/src/coremsginsert.c
@@ -22,17 +22,17 @@
 
 #if defined(RTEMS_SCORE_COREMSG_ENABLE_MESSAGE_PRIORITY)
 static bool _CORE_message_queue_Order(
-  const Chain_Node *left,
+  const void       *left,
   const Chain_Node *right
 )
 {
-   const CORE_message_queue_Buffer_control *left_message;
+   const int                               *left_priority;
    const CORE_message_queue_Buffer_control *right_message;
 
-   left_message = (const CORE_message_queue_Buffer_control *) left;
+   left_priority = (const int *) left;
    right_message = (const CORE_message_queue_Buffer_control *) right;
 
-   return _CORE_message_queue_Get_message_priority( left_message ) <
+   return *left_priority <
      _CORE_message_queue_Get_message_priority( right_message );
 }
 #endif
@@ -66,9 +66,13 @@ void _CORE_message_queue_Insert_message(
     _Chain_Append_unprotected( pending_messages, &the_message->Node );
 #if defined(RTEMS_SCORE_COREMSG_ENABLE_MESSAGE_PRIORITY)
   } else  if ( submit_type != CORE_MESSAGE_QUEUE_URGENT_REQUEST ) {
+    int priority;
+
+    priority = _CORE_message_queue_Get_message_priority( the_message );
     _Chain_Insert_ordered_unprotected(
       pending_messages,
       &the_message->Node,
+      &priority,
       _CORE_message_queue_Order
     );
 #endif
diff --git a/cpukit/score/src/schedulerpriorityaffinitysmp.c b/cpukit/score/src/schedulerpriorityaffinitysmp.c
index 6caf00c3a0..a948eef0fc 100644
--- a/cpukit/score/src/schedulerpriorityaffinitysmp.c
+++ b/cpukit/score/src/schedulerpriorityaffinitysmp.c
@@ -40,7 +40,7 @@
  */
 
 static bool _Scheduler_priority_affinity_SMP_Insert_priority_lifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
@@ -49,7 +49,7 @@ static bool _Scheduler_priority_affinity_SMP_Insert_priority_lifo_order(
 }
 
 static bool _Scheduler_priority_affinity_SMP_Insert_priority_fifo_order(
-  const Chain_Node *to_insert,
+  const void       *to_insert,
   const Chain_Node *next
 )
 {
@@ -283,6 +283,8 @@ static void _Scheduler_priority_affinity_SMP_Check_for_migrations(
   self = _Scheduler_priority_SMP_Get_self( context );
 
   while (1) {
+    Priority_Control lowest_scheduled_priority;
+
     if ( _Priority_bit_map_Is_empty( &self->Bit_map ) ) {
       /* Nothing to do */
       break;
@@ -310,9 +312,12 @@ static void _Scheduler_priority_affinity_SMP_Check_for_migrations(
     if ( lowest_scheduled == NULL )
       break;
 
+    lowest_scheduled_priority =
+      _Scheduler_SMP_Node_priority( lowest_scheduled );
+
     if (
       _Scheduler_SMP_Insert_priority_lifo_order(
-        &lowest_scheduled->Node.Chain,
+        &lowest_scheduled_priority,
         &highest_ready->Node.Chain
       )
     ) {
diff --git a/cpukit/score/src/schedulersimplesmp.c b/cpukit/score/src/schedulersimplesmp.c
index 4be43abff1..df08a19eab 100644
--- a/cpukit/score/src/schedulersimplesmp.c
+++ b/cpukit/score/src/schedulersimplesmp.c
@@ -98,13 +98,17 @@ static void _Scheduler_simple_SMP_Move_from_scheduled_to_ready(
   Scheduler_Node    *scheduled_to_ready
 )
 {
-  Scheduler_simple_SMP_Context *self =
-    _Scheduler_simple_SMP_Get_self( context );
+  Scheduler_simple_SMP_Context *self;
+  Priority_Control              priority_to_insert;
+
+  self = _Scheduler_simple_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( scheduled_to_ready );
 
   _Chain_Extract_unprotected( &scheduled_to_ready->Node.Chain );
   _Chain_Insert_ordered_unprotected(
     &self->Ready,
     &scheduled_to_ready->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_lifo_order
   );
 }
@@ -114,13 +118,17 @@ static void _Scheduler_simple_SMP_Move_from_ready_to_scheduled(
   Scheduler_Node    *ready_to_scheduled
 )
 {
-  Scheduler_simple_SMP_Context *self =
-    _Scheduler_simple_SMP_Get_self( context );
+  Scheduler_simple_SMP_Context *self;
+  Priority_Control              priority_to_insert;
+
+  self = _Scheduler_simple_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( ready_to_scheduled );
 
   _Chain_Extract_unprotected( &ready_to_scheduled->Node.Chain );
   _Chain_Insert_ordered_unprotected(
     &self->Base.Scheduled,
     &ready_to_scheduled->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_fifo_order
   );
 }
@@ -130,12 +138,16 @@ static void _Scheduler_simple_SMP_Insert_ready_lifo(
   Scheduler_Node    *node_to_insert
 )
 {
-  Scheduler_simple_SMP_Context *self =
-    _Scheduler_simple_SMP_Get_self( context );
+  Scheduler_simple_SMP_Context *self;
+  Priority_Control              priority_to_insert;
+
+  self = _Scheduler_simple_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( node_to_insert );
 
   _Chain_Insert_ordered_unprotected(
     &self->Ready,
     &node_to_insert->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_lifo_order
   );
 }
@@ -145,12 +157,16 @@ static void _Scheduler_simple_SMP_Insert_ready_fifo(
   Scheduler_Node    *node_to_insert
 )
 {
-  Scheduler_simple_SMP_Context *self =
-    _Scheduler_simple_SMP_Get_self( context );
+  Scheduler_simple_SMP_Context *self;
+  Priority_Control              priority_to_insert;
+
+  self = _Scheduler_simple_SMP_Get_self( context );
+  priority_to_insert = _Scheduler_SMP_Node_priority( node_to_insert );
 
   _Chain_Insert_ordered_unprotected(
     &self->Ready,
     &node_to_insert->Node.Chain,
+    &priority_to_insert,
     _Scheduler_SMP_Insert_priority_fifo_order
   );
 }
diff --git a/cpukit/score/src/schedulerstrongapa.c b/cpukit/score/src/schedulerstrongapa.c
index d5bfed74f4..57ffb61367 100644
--- a/cpukit/score/src/schedulerstrongapa.c
+++ b/cpukit/score/src/schedulerstrongapa.c
@@ -64,19 +64,23 @@ static void _Scheduler_strong_APA_Move_from_ready_to_scheduled(
   Scheduler_Node    *ready_to_scheduled
 )
 {
-  Scheduler_strong_APA_Context *self =
-    _Scheduler_strong_APA_Get_self( context );
-  Scheduler_strong_APA_Node *node =
-    _Scheduler_strong_APA_Node_downcast( ready_to_scheduled );
+  Scheduler_strong_APA_Context *self;
+  Scheduler_strong_APA_Node    *node;
+  Priority_Control              priority;
+
+  self = _Scheduler_strong_APA_Get_self( context );
+  node = _Scheduler_strong_APA_Node_downcast( ready_to_scheduled );
 
   _Scheduler_priority_Ready_queue_extract(
     &node->Base.Base.Node.Chain,
     &node->Ready_queue,
     &self->Bit_map
   );
+  priority = node->Base.priority;
   _Chain_Insert_ordered_unprotected(
     &self->Base.Scheduled,
     &node->Base.Base.Node.Chain,
+    &priority,
     _Scheduler_SMP_Insert_priority_fifo_order
   );
 }