score: Turn thread lock into thread wait lock

The _Thread_Lock_acquire() function had a potentially infinite run-time
due to the lack of fairness at atomic operations level.

Update #2412.
Update #2556.
Update #2765.

6 files changed, 142 insertions, 85 deletions

diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c
index e3cf4a165a..c569530d74 100644
--- a/cpukit/score/src/threadchangepriority.c
+++ b/cpukit/score/src/threadchangepriority.c
@@ -27,7 +27,8 @@ static Thread_Control *_Thread_Apply_priority_locked(
   Priority_Control               new_priority,
   void                          *arg,
   Thread_Change_priority_filter  filter,
-  bool                           prepend_it
+  bool                           prepend_it,
+  Thread_queue_Context          *queue_context
 )
 {
   /*
@@ -45,17 +46,11 @@ static Thread_Control *_Thread_Apply_priority_locked(
    *  we are not REALLY changing priority.
    */
   if ( ( *filter )( the_thread, &new_priority, arg ) ) {
-    Scheduler_Node *own_node;
-
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
-    _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
-
-    the_thread->current_priority = new_priority;
-
-    ( *the_thread->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      queue_context,
       the_thread,
       new_priority,
-      the_thread->Wait.queue
+      prepend_it
     );
   } else {
     the_thread = NULL;
@@ -72,19 +67,20 @@ Thread_Control *_Thread_Apply_priority(
   bool                           prepend_it
 )
 {
-  ISR_lock_Context  lock_context;
-  ISR_lock_Control *lock;
+  Thread_queue_Context  queue_context;
+  Thread_Control       *the_thread_to_update;
 
-  lock = _Thread_Lock_acquire( the_thread, &lock_context );
-  the_thread = _Thread_Apply_priority_locked(
+  _Thread_Wait_acquire( the_thread, &queue_context );
+  the_thread_to_update = _Thread_Apply_priority_locked(
     the_thread,
     new_priority,
     arg,
     filter,
-    prepend_it
+    prepend_it,
+    &queue_context
   );
-  _Thread_Lock_release( lock, &lock_context );
-  return the_thread;
+  _Thread_Wait_release( the_thread, &queue_context );
+  return the_thread_to_update;
 }
 
 void _Thread_Update_priority( Thread_Control *the_thread )
diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c
index 940537f135..209be9ebb1 100644
--- a/cpukit/score/src/threadinitialize.c
+++ b/cpukit/score/src/threadinitialize.c
@@ -182,13 +182,11 @@ bool _Thread_Initialize(
   the_thread->Scheduler.control = scheduler;
   the_thread->Scheduler.own_node = scheduler_node;
   _Resource_Node_initialize( &the_thread->Resource_node );
-  _Atomic_Store_uintptr(
-    &the_thread->Lock.current.atomic,
-    (uintptr_t) &the_thread->Lock.Default,
-    ATOMIC_ORDER_RELAXED
+  _ISR_lock_Initialize(
+    &the_thread->Wait.Lock.Default,
+    "Thread Wait Default Lock"
   );
-  _SMP_ticket_lock_Initialize( &the_thread->Lock.Default );
-  _SMP_lock_Stats_initialize( &the_thread->Lock.Stats, "Thread Lock" );
+  _Chain_Initialize_empty( &the_thread->Wait.Lock.Pending_requests );
   _SMP_lock_Stats_initialize( &the_thread->Potpourri_stats, "Thread Potpourri" );
 #endif
 
diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c
index 818073c2b1..19c345b6c5 100644
--- a/cpukit/score/src/threadqenqueue.c
+++ b/cpukit/score/src/threadqenqueue.c
@@ -34,6 +34,51 @@
 #define THREAD_QUEUE_READY_AGAIN \
   (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN)
 
+static void _Thread_queue_Path_release( Thread_queue_Path *path )
+{
+#if defined(RTEMS_SMP)
+  Thread_queue_Link *link;
+
+  link = &path->Start;
+
+  if ( link->owner != NULL ) {
+    _Thread_Wait_release_critical( link->owner, &link->Queue_context );
+  }
+#else
+  (void) path;
+#endif
+}
+
+static void _Thread_queue_Path_acquire(
+  Thread_Control     *the_thread,
+  Thread_queue_Queue *queue,
+  Thread_queue_Path  *path
+)
+{
+#if defined(RTEMS_SMP)
+  Thread_Control     *owner;
+  Thread_queue_Link  *link;
+
+  owner = queue->owner;
+
+  if ( owner == NULL ) {
+    return;
+  }
+
+  link = &path->Start;
+  link->owner = owner;
+
+  _Thread_Wait_acquire_default_critical(
+    owner,
+    &link->Queue_context.Lock_context
+  );
+#else
+  (void) the_thread;
+  (void) queue;
+  (void) path;
+#endif
+}
+
 void _Thread_queue_Enqueue_critical(
   Thread_queue_Queue            *queue,
   const Thread_queue_Operations *operations,
@@ -52,14 +97,13 @@ void _Thread_queue_Enqueue_critical(
   }
 #endif
 
-  _Thread_Lock_set( the_thread, &queue->Lock );
-
-  the_thread->Wait.return_code = STATUS_SUCCESSFUL;
-  _Thread_Wait_set_queue( the_thread, queue );
-  _Thread_Wait_set_operations( the_thread, operations );
+  _Thread_Wait_claim( the_thread, queue, operations );
 
+  _Thread_queue_Path_acquire( the_thread, queue, &path );
   ( *operations->enqueue )( queue, the_thread, &path );
+  _Thread_queue_Path_release( &path );
 
+  the_thread->Wait.return_code = STATUS_SUCCESSFUL;
   _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK );
   cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context );
   _Thread_queue_Queue_release( queue, &queue_context->Lock_context );
@@ -173,9 +217,7 @@ bool _Thread_queue_Do_extract_locked(
     unblock = true;
   }
 
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
+  _Thread_Wait_restore_default( the_thread );
 
   return unblock;
 }
@@ -227,30 +269,35 @@ void _Thread_queue_Extract_critical(
 
 void _Thread_queue_Extract( Thread_Control *the_thread )
 {
-  Thread_queue_Context  queue_context;
-  void                 *lock;
-  Thread_queue_Queue   *queue;
+  Thread_queue_Context queue_context;
 
   _Thread_queue_Context_initialize( &queue_context );
-  lock = _Thread_Lock_acquire( the_thread, &queue_context.Lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
-  queue = the_thread->Wait.queue;
-
-  if ( queue != NULL ) {
-    _SMP_Assert( lock == &queue->Lock );
+  if (
+    _Thread_queue_Context_get_queue( &queue_context, the_thread ) != NULL
+  ) {
+    bool unblock;
 
+    _Thread_Wait_remove_request( the_thread, &queue_context );
     _Thread_queue_Context_set_MP_callout(
       &queue_context,
       _Thread_queue_MP_callout_do_nothing
     );
-    _Thread_queue_Extract_critical(
-      queue,
-      the_thread->Wait.operations,
+    unblock = _Thread_queue_Extract_locked(
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      _Thread_queue_Context_get_operations( &queue_context, the_thread ),
       the_thread,
-      &queue_context
+      &queue_context.Lock_context
+    );
+    _Thread_queue_Unblock_critical(
+      unblock,
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      the_thread,
+      &queue_context.Lock_context
     );
   } else {
-    _Thread_Lock_release( lock, &queue_context.Lock_context );
+    _Thread_Wait_release( the_thread, &queue_context );
   }
 }
 
@@ -273,8 +320,6 @@ Thread_Control *_Thread_queue_Do_dequeue(
   the_thread = _Thread_queue_First_locked( the_thread_queue, operations );
 
   if ( the_thread != NULL ) {
-    _SMP_Assert( the_thread->Lock.current.normal == &the_thread_queue->Queue.Lock );
-
     _Thread_queue_Extract_critical(
       &the_thread_queue->Queue,
       operations,
diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c
index c288ceeca9..8059446578 100644
--- a/cpukit/score/src/threadqops.c
+++ b/cpukit/score/src/threadqops.c
@@ -22,13 +22,16 @@
 #include <rtems/score/rbtreeimpl.h>
 #include <rtems/score/schedulerimpl.h>
 
-static void _Thread_queue_Do_nothing_priority_change(
+static void _Thread_queue_Default_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
-  /* Do nothing */
+  (void) queue;
+
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
 }
 
 static void _Thread_queue_Do_nothing_extract(
@@ -39,6 +42,31 @@ static void _Thread_queue_Do_nothing_extract(
   /* Do nothing */
 }
 
+#if defined(RTEMS_SMP)
+static void _Thread_queue_Stale_queue_priority_change(
+  Thread_Control     *the_thread,
+  Priority_Control    new_priority,
+  bool                prepend_it,
+  Thread_queue_Queue *queue
+)
+{
+  ISR_lock_Context lock_context;
+
+  (void) queue;
+
+  /*
+   * This operation is used to change the priority in case we have a thread
+   * queue context with a stale thread queue.  We own the thread queue lock of
+   * the former thread queue.  In addition, we need the thread wait default
+   * lock, see _Thread_Wait_restore_default().
+   */
+
+  _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+  _Thread_Wait_release_default_critical( the_thread, &lock_context );
+}
+#endif
+
 static Thread_queue_Heads *_Thread_queue_Queue_enqueue(
   Thread_queue_Queue *queue,
   Thread_Control     *the_thread,
@@ -190,6 +218,7 @@ static bool _Thread_queue_Priority_less(
 static void _Thread_queue_Priority_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
@@ -198,6 +227,8 @@ static void _Thread_queue_Priority_priority_change(
 
   _Assert( heads != NULL );
 
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+
   priority_queue = _Thread_queue_Priority_queue( heads, the_thread );
 
   _RBTree_Extract(
@@ -353,22 +384,16 @@ static void _Thread_queue_Priority_inherit_enqueue(
 #endif
 
   if ( priority < owner->current_priority ) {
-    Scheduler_Node *own_node;
-
     path->update_priority = owner;
 
     owner->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
-    own_node = _Scheduler_Thread_get_own_node( owner );
-    _Scheduler_Node_set_priority( own_node, priority, false );
-
-    owner->current_priority = priority;
-
-    ( *owner->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      &path->Start.Queue_context,
       owner,
       priority,
-      owner->Wait.queue
+      false
     );
   } else {
     path->update_priority = NULL;
@@ -385,24 +410,21 @@ void _Thread_queue_Boost_priority(
 
   if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) {
     const Scheduler_Control *scheduler;
-    Scheduler_Node          *own_node;
     Priority_Control         boost_priority;
 
     the_thread->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
     scheduler = _Scheduler_Get_own( the_thread );
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
     boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR );
-    _Scheduler_Node_set_priority( own_node, boost_priority, false );
 
-    the_thread->current_priority = boost_priority;
+    _Scheduler_Thread_set_priority( the_thread, boost_priority, false );
   }
 }
 #endif
 
 const Thread_queue_Operations _Thread_queue_Operations_default = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .extract = _Thread_queue_Do_nothing_extract
   /*
    * The default operations are only used in _Thread_Change_priority() and
@@ -412,7 +434,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = {
 };
 
 const Thread_queue_Operations _Thread_queue_Operations_FIFO = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .enqueue = _Thread_queue_FIFO_enqueue,
   .extract = _Thread_queue_FIFO_extract,
   .first = _Thread_queue_FIFO_first
@@ -431,3 +453,10 @@ const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = {
   .extract = _Thread_queue_Priority_extract,
   .first = _Thread_queue_Priority_first
 };
+
+#if defined(RTEMS_SMP)
+const Thread_queue_Operations _Thread_queue_Operations_stale_queue = {
+  .priority_change = _Thread_queue_Stale_queue_priority_change,
+  .extract = _Thread_queue_Do_nothing_extract
+};
+#endif
diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c
index 757f552699..81ba73d704 100644
--- a/cpukit/score/src/threadrestart.c
+++ b/cpukit/score/src/threadrestart.c
@@ -215,8 +215,7 @@ static void _Thread_Free( Thread_Control *the_thread )
   _Workspace_Free( the_thread->Start.tls_area );
 
 #if defined(RTEMS_SMP)
-  _SMP_ticket_lock_Destroy( &the_thread->Lock.Default );
-  _SMP_lock_Stats_destroy( &the_thread->Lock.Stats );
+  _ISR_lock_Destroy( &the_thread->Wait.Lock.Default );
   _SMP_lock_Stats_destroy( &the_thread->Potpourri_stats );
 #endif
 
diff --git a/cpukit/score/src/threadtimeout.c b/cpukit/score/src/threadtimeout.c
index a2ba61f9e7..b6b6cc4ac2 100644
--- a/cpukit/score/src/threadtimeout.c
+++ b/cpukit/score/src/threadtimeout.c
@@ -22,28 +22,15 @@
 #include <rtems/score/threadimpl.h>
 #include <rtems/score/status.h>
 
-static void _Thread_Do_timeout( Thread_Control *the_thread )
-{
-  the_thread->Wait.return_code = STATUS_TIMEOUT;
-  ( *the_thread->Wait.operations->extract )(
-    the_thread->Wait.queue,
-    the_thread
-  );
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
-}
-
 void _Thread_Timeout( Watchdog_Control *watchdog )
 {
-  Thread_Control    *the_thread;
-  void              *thread_lock;
-  ISR_lock_Context   lock_context;
-  Thread_Wait_flags  wait_flags;
-  bool               unblock;
+  Thread_Control       *the_thread;
+  Thread_queue_Context  queue_context;
+  Thread_Wait_flags     wait_flags;
+  bool                  unblock;
 
   the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog );
-  thread_lock = _Thread_Lock_acquire( the_thread, &lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
   wait_flags = _Thread_Wait_flags_get( the_thread );
 
@@ -52,7 +39,9 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     Thread_Wait_flags ready_again;
     bool              success;
 
-    _Thread_Do_timeout( the_thread );
+    _Thread_Wait_cancel( the_thread, &queue_context );
+
+    the_thread->Wait.return_code = STATUS_TIMEOUT;
 
     wait_class = wait_flags & THREAD_WAIT_CLASS_MASK;
     ready_again = wait_class | THREAD_WAIT_STATE_READY_AGAIN;
@@ -76,9 +65,10 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     unblock = false;
   }
 
-  _Thread_Lock_release( thread_lock, &lock_context );
+  _Thread_Wait_release( the_thread, &queue_context );
 
   if ( unblock ) {
+    _Thread_Wait_tranquilize( the_thread );
     _Thread_Unblock( the_thread );
 
 #if defined(RTEMS_MULTIPROCESSING)