score: Optimize scheduler priority updates

Thread priority changes may append or prepend the thread to its priority
group on the scheduler ready queue.  Previously, a separate priority
value and a prepend-it flag in the scheduler node were used to propagate
a priority change to the scheduler.

Now, use an append-it bit in the priority control and reduce the plain
priority value to 63 bits.

This change leads to a significant code size reduction (about 25%) of
the SMP schedulers.  The negligible increase of the standard priority
scheduler is due to some additional shift operations
(SCHEDULER_PRIORITY_MAP() and SCHEDULER_PRIORITY_UNMAP()).

Before:

   text filename
    136 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleblock.o
    464 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimplechangepriority.o
     24 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimple.o
    108 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleschedule.o
    292 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleunblock.o
    264 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleyield.o

   text filename
    280 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityblock.o
    488 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerprioritychangepriority.o
    200 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriority.o
    164 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityschedule.o
    328 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityunblock.o
    200 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityyield.o

   text filename
  24112 arm-rtems5/c/imx7/cpukit/score/src/libscore_a-scheduleredfsmp.o

   text filename
  37204 sparc-rtems5/c/gr740/cpukit/score/src/libscore_a-scheduleredfsmp.o

   text filename
  42236 powerpc-rtems5/c/qoriq_e6500_32/cpukit/score/src/libscore_a-scheduleredfsmp.o

After:

   text filename
    136 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleblock.o
    272 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimplechangepriority.o
     24 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimple.o
    108 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleschedule.o
    292 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleunblock.o
    264 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulersimpleyield.o

   text filename
    280 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityblock.o
    488 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerprioritychangepriority.o
    208 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriority.o
    164 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityschedule.o
    332 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityunblock.o
    200 sparc-rtems5/c/erc32/cpukit/score/src/libscore_a-schedulerpriorityyield.o

   text filename
  18860 arm-rtems5/c/imx7/cpukit/score/src/libscore_a-scheduleredfsmp.o

   text filename
  28520 sparc-rtems5/c/gr740/cpukit/score/src/libscore_a-scheduleredfsmp.o

   text filename
  32664 powerpc-rtems5/c/qoriq_e6500_32/cpukit/score/src/libscore_a-scheduleredfsmp.o

1 files changed, 43 insertions, 107 deletions

diff --git a/cpukit/score/src/schedulerpriorityaffinitysmp.c b/cpukit/score/src/schedulerpriorityaffinitysmp.c
index 72b4ffb600..4808c84c3f 100644
--- a/cpukit/score/src/schedulerpriorityaffinitysmp.c
+++ b/cpukit/score/src/schedulerpriorityaffinitysmp.c
@@ -39,22 +39,13 @@
  *  + _Scheduler_priority_SMP_Do_update
  */
 
-static bool _Scheduler_priority_affinity_SMP_Insert_priority_lifo_order(
+static bool _Scheduler_priority_affinity_SMP_Priority_less_equal(
   const void       *to_insert,
   const Chain_Node *next
 )
 {
   return next != NULL
-    && _Scheduler_SMP_Insert_priority_lifo_order( to_insert, next );
-}
-
-static bool _Scheduler_priority_affinity_SMP_Insert_priority_fifo_order(
-  const void       *to_insert,
-  const Chain_Node *next
-)
-{
-  return next != NULL
-    && _Scheduler_SMP_Insert_priority_fifo_order( to_insert, next );
+    && _Scheduler_SMP_Priority_less_equal( to_insert, next );
 }
 
 static Scheduler_priority_affinity_SMP_Node *
@@ -242,19 +233,21 @@ static Scheduler_Node * _Scheduler_priority_affinity_SMP_Get_lowest_scheduled(
 /*
  * This method is unique to this scheduler because it must pass
  * _Scheduler_priority_affinity_SMP_Get_lowest_scheduled into
- * _Scheduler_SMP_Enqueue_ordered.
+ * _Scheduler_SMP_Enqueue.
  */
 static bool _Scheduler_priority_affinity_SMP_Enqueue_fifo(
   Scheduler_Context *context,
-  Scheduler_Node    *node
+  Scheduler_Node    *node,
+  Priority_Control   insert_priority
 )
 {
-  return _Scheduler_SMP_Enqueue_ordered(
+  return _Scheduler_SMP_Enqueue(
     context,
     node,
-    _Scheduler_priority_affinity_SMP_Insert_priority_fifo_order,
-    _Scheduler_priority_SMP_Insert_ready_fifo,
-    _Scheduler_SMP_Insert_scheduled_fifo,
+    insert_priority,
+    _Scheduler_priority_affinity_SMP_Priority_less_equal,
+    _Scheduler_priority_SMP_Insert_ready,
+    _Scheduler_SMP_Insert_scheduled,
     _Scheduler_priority_SMP_Move_from_scheduled_to_ready,
     _Scheduler_priority_affinity_SMP_Get_lowest_scheduled,
     _Scheduler_SMP_Allocate_processor_exact
@@ -280,6 +273,7 @@ static void _Scheduler_priority_affinity_SMP_Check_for_migrations(
 
   while (1) {
     Priority_Control lowest_scheduled_priority;
+    Priority_Control insert_priority;
 
     if ( _Priority_bit_map_Is_empty( &self->Bit_map ) ) {
       /* Nothing to do */
@@ -312,7 +306,7 @@ static void _Scheduler_priority_affinity_SMP_Check_for_migrations(
       _Scheduler_SMP_Node_priority( lowest_scheduled );
 
     if (
-      _Scheduler_SMP_Insert_priority_lifo_order(
+      _Scheduler_SMP_Priority_less_equal(
         &lowest_scheduled_priority,
         &highest_ready->Node.Chain
       )
@@ -326,11 +320,14 @@ static void _Scheduler_priority_affinity_SMP_Check_for_migrations(
      */
 
     _Scheduler_priority_SMP_Extract_from_ready( context, highest_ready );
+    insert_priority = _Scheduler_SMP_Node_priority( highest_ready );
+    insert_priority = SCHEDULER_PRIORITY_APPEND( insert_priority );
     _Scheduler_SMP_Enqueue_to_scheduled(
       context,
       highest_ready,
+      insert_priority,
       lowest_scheduled,
-      _Scheduler_SMP_Insert_scheduled_fifo,
+      _Scheduler_SMP_Insert_scheduled,
       _Scheduler_priority_SMP_Move_from_scheduled_to_ready,
       _Scheduler_SMP_Allocate_processor_exact
     );
@@ -364,22 +361,21 @@ void _Scheduler_priority_affinity_SMP_Unblock(
 
 /*
  *  This is unique to this scheduler because it passes scheduler specific
- *  get_lowest_scheduled helper to _Scheduler_SMP_Enqueue_ordered.
+ *  get_lowest_scheduled helper to _Scheduler_SMP_Enqueue.
  */
-static bool _Scheduler_priority_affinity_SMP_Enqueue_ordered(
-  Scheduler_Context     *context,
-  Scheduler_Node        *node,
-  Chain_Node_order       order,
-  Scheduler_SMP_Insert   insert_ready,
-  Scheduler_SMP_Insert   insert_scheduled
+static bool _Scheduler_priority_affinity_SMP_Enqueue(
+  Scheduler_Context *context,
+  Scheduler_Node    *node,
+  Priority_Control   insert_priority
 )
 {
-  return _Scheduler_SMP_Enqueue_ordered(
+  return _Scheduler_SMP_Enqueue(
     context,
     node,
-    order,
-    insert_ready,
-    insert_scheduled,
+    insert_priority,
+    _Scheduler_priority_affinity_SMP_Priority_less_equal,
+    _Scheduler_priority_SMP_Insert_ready,
+    _Scheduler_SMP_Insert_scheduled,
     _Scheduler_priority_SMP_Move_from_scheduled_to_ready,
     _Scheduler_priority_affinity_SMP_Get_lowest_scheduled,
     _Scheduler_SMP_Allocate_processor_exact
@@ -387,88 +383,30 @@ static bool _Scheduler_priority_affinity_SMP_Enqueue_ordered(
 }
 
 /*
- *  This is unique to this scheduler because it is on the path
- *  to _Scheduler_priority_affinity_SMP_Enqueue_ordered() which
- *  invokes a scheduler unique get_lowest_scheduled helper.
- */
-static bool _Scheduler_priority_affinity_SMP_Enqueue_lifo(
-  Scheduler_Context *context,
-  Scheduler_Node    *node
-)
-{
-  return _Scheduler_priority_affinity_SMP_Enqueue_ordered(
-    context,
-    node,
-    _Scheduler_priority_affinity_SMP_Insert_priority_lifo_order,
-    _Scheduler_priority_SMP_Insert_ready_lifo,
-    _Scheduler_SMP_Insert_scheduled_lifo
-  );
-}
-
-/*
  * This method is unique to this scheduler because it must
- * invoke _Scheduler_SMP_Enqueue_scheduled_ordered() with
+ * invoke _Scheduler_SMP_Enqueue_scheduled() with
  * this scheduler's get_highest_ready() helper.
  */
-static bool _Scheduler_priority_affinity_SMP_Enqueue_scheduled_ordered(
-  Scheduler_Context    *context,
-  Scheduler_Node       *node,
-  Chain_Node_order      order,
-  Scheduler_SMP_Insert  insert_ready,
-  Scheduler_SMP_Insert  insert_scheduled
+static bool _Scheduler_priority_affinity_SMP_Enqueue_scheduled(
+  Scheduler_Context *context,
+  Scheduler_Node    *node,
+  Priority_Control   insert_priority
 )
 {
-  return _Scheduler_SMP_Enqueue_scheduled_ordered(
+  return _Scheduler_SMP_Enqueue_scheduled(
     context,
     node,
-    order,
+    insert_priority,
+    _Scheduler_SMP_Priority_less_equal,
     _Scheduler_priority_SMP_Extract_from_ready,
     _Scheduler_priority_affinity_SMP_Get_highest_ready,
-    insert_ready,
-    insert_scheduled,
+    _Scheduler_priority_SMP_Insert_ready,
+    _Scheduler_SMP_Insert_scheduled,
     _Scheduler_priority_SMP_Move_from_ready_to_scheduled,
     _Scheduler_SMP_Allocate_processor_exact
   );
 }
 
-/*
- *  This is unique to this scheduler because it is on the path
- *  to _Scheduler_priority_affinity_SMP_Enqueue_scheduled__ordered() which
- *  invokes a scheduler unique get_lowest_scheduled helper.
- */
-static bool _Scheduler_priority_affinity_SMP_Enqueue_scheduled_lifo(
-  Scheduler_Context *context,
-  Scheduler_Node    *node
-)
-{
-  return _Scheduler_priority_affinity_SMP_Enqueue_scheduled_ordered(
-    context,
-    node,
-    _Scheduler_SMP_Insert_priority_lifo_order,
-    _Scheduler_priority_SMP_Insert_ready_lifo,
-    _Scheduler_SMP_Insert_scheduled_lifo
-  );
-}
-
-/*
- *  This is unique to this scheduler because it is on the path
- *  to _Scheduler_priority_affinity_SMP_Enqueue_scheduled__ordered() which
- *  invokes a scheduler unique get_lowest_scheduled helper.
- */
-static bool _Scheduler_priority_affinity_SMP_Enqueue_scheduled_fifo(
-  Scheduler_Context *context,
-  Scheduler_Node    *node
-)
-{
-  return _Scheduler_priority_affinity_SMP_Enqueue_scheduled_ordered(
-    context,
-    node,
-    _Scheduler_SMP_Insert_priority_fifo_order,
-    _Scheduler_priority_SMP_Insert_ready_fifo,
-    _Scheduler_SMP_Insert_scheduled_fifo
-  );
-}
-
 static bool _Scheduler_priority_affinity_SMP_Do_ask_for_help(
   Scheduler_Context *context,
   Thread_Control    *the_thread,
@@ -479,9 +417,9 @@ static bool _Scheduler_priority_affinity_SMP_Do_ask_for_help(
     context,
     the_thread,
     node,
-    _Scheduler_SMP_Insert_priority_lifo_order,
-    _Scheduler_priority_SMP_Insert_ready_lifo,
-    _Scheduler_SMP_Insert_scheduled_lifo,
+    _Scheduler_SMP_Priority_less_equal,
+    _Scheduler_priority_SMP_Insert_ready,
+    _Scheduler_SMP_Insert_scheduled,
     _Scheduler_priority_SMP_Move_from_scheduled_to_ready,
     _Scheduler_SMP_Get_lowest_scheduled,
     _Scheduler_SMP_Allocate_processor_lazy
@@ -502,10 +440,8 @@ void _Scheduler_priority_affinity_SMP_Update_priority(
     node,
     _Scheduler_priority_SMP_Extract_from_ready,
     _Scheduler_priority_SMP_Do_update,
-    _Scheduler_priority_affinity_SMP_Enqueue_fifo,
-    _Scheduler_priority_affinity_SMP_Enqueue_lifo,
-    _Scheduler_priority_affinity_SMP_Enqueue_scheduled_fifo,
-    _Scheduler_priority_affinity_SMP_Enqueue_scheduled_lifo,
+    _Scheduler_priority_affinity_SMP_Enqueue,
+    _Scheduler_priority_affinity_SMP_Enqueue_scheduled,
     _Scheduler_priority_affinity_SMP_Do_ask_for_help
   );
 
@@ -574,7 +510,7 @@ void _Scheduler_priority_affinity_SMP_Add_processor(
     context,
     idle,
     _Scheduler_priority_SMP_Has_ready,
-    _Scheduler_priority_affinity_SMP_Enqueue_scheduled_fifo,
+    _Scheduler_priority_affinity_SMP_Enqueue_scheduled,
     _Scheduler_SMP_Do_nothing_register_idle
   );
 }
@@ -590,7 +526,7 @@ Thread_Control *_Scheduler_priority_affinity_SMP_Remove_processor(
     context,
     cpu,
     _Scheduler_priority_SMP_Extract_from_ready,
-    _Scheduler_priority_affinity_SMP_Enqueue_fifo
+    _Scheduler_priority_affinity_SMP_Enqueue
   );
 }