score: Turn thread lock into thread wait lock

The _Thread_Lock_acquire() function had a potentially infinite run-time
due to the lack of fairness at atomic operations level.

Update #2412.
Update #2556.
Update #2765.

20 files changed, 769 insertions, 424 deletions

diff --git a/cpukit/libmisc/monitor/mon-task.c b/cpukit/libmisc/monitor/mon-task.c
index 96891e26de..eedba3e1d4 100644
--- a/cpukit/libmisc/monitor/mon-task.c
+++ b/cpukit/libmisc/monitor/mon-task.c
@@ -20,17 +20,16 @@ rtems_monitor_task_wait_info(
     Thread_Control       *rtems_thread
 )
 {
-    ISR_lock_Context  lock_context;
-    void             *lock;
+    Thread_queue_Context queue_context;
 
-    lock = _Thread_Lock_acquire( rtems_thread, &lock_context );
+    _Thread_Wait_acquire( rtems_thread, &queue_context );
 
     canonical_task->state = rtems_thread->current_state;
     canonical_task->wait_id = _Thread_Wait_get_id( rtems_thread );
     canonical_task->wait_queue = rtems_thread->Wait.queue;
     canonical_task->wait_operations = rtems_thread->Wait.operations;
 
-    _Thread_Lock_release( lock, &lock_context );
+    _Thread_Wait_release( rtems_thread, &queue_context );
 }
 
 void
diff --git a/cpukit/posix/src/pthreadgetschedparam.c b/cpukit/posix/src/pthreadgetschedparam.c
index a0a4c6e15b..1b7c731587 100644
--- a/cpukit/posix/src/pthreadgetschedparam.c
+++ b/cpukit/posix/src/pthreadgetschedparam.c
@@ -36,7 +36,7 @@ int pthread_getschedparam(
 )
 {
   Thread_Control          *the_thread;
-  ISR_lock_Context         lock_context;
+  Thread_queue_Context     queue_context;
   POSIX_API_Control       *api;
   const Scheduler_Control *scheduler;
   Priority_Control         priority;
@@ -45,7 +45,7 @@ int pthread_getschedparam(
     return EINVAL;
   }
 
-  the_thread = _Thread_Get( thread, &lock_context );
+  the_thread = _Thread_Get( thread, &queue_context.Lock_context );
 
   if ( the_thread == NULL ) {
     return ESRCH;
@@ -53,7 +53,7 @@ int pthread_getschedparam(
 
   api = the_thread->API_Extensions[ THREAD_API_POSIX ];
 
-  _Thread_Lock_acquire_default_critical( the_thread, &lock_context );
+  _Thread_Wait_acquire_critical( the_thread, &queue_context );
 
   *policy = api->Attributes.schedpolicy;
   *param  = api->Attributes.schedparam;
@@ -61,7 +61,7 @@ int pthread_getschedparam(
   scheduler = _Scheduler_Get_own( the_thread );
   priority = the_thread->real_priority;
 
-  _Thread_Lock_release_default( the_thread, &lock_context );
+  _Thread_Wait_release( the_thread, &queue_context );
 
   param->sched_priority = _POSIX_Priority_From_core( scheduler, priority );
   return 0;
diff --git a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
index 61ebb5a0df..9963cab612 100644
--- a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
+++ b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
@@ -73,7 +73,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Acquire_critical(
   ISR_lock_Context *lock_context
 )
 {
-  _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, lock_context );
 }
 
 RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
@@ -81,7 +81,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
   ISR_lock_Context *lock_context
 )
 {
-  _Thread_Lock_release_default( the_thread, lock_context );
+  _Thread_Wait_release_default( the_thread, lock_context );
 }
 
 RTEMS_INLINE_ROUTINE Rate_monotonic_Control *_Rate_monotonic_Get(
diff --git a/cpukit/rtems/src/eventreceive.c b/cpukit/rtems/src/eventreceive.c
index e03ff279eb..193960b7b2 100644
--- a/cpukit/rtems/src/eventreceive.c
+++ b/cpukit/rtems/src/eventreceive.c
@@ -38,7 +38,7 @@ rtems_status_code rtems_event_receive(
     RTEMS_API_Control *api;
     Event_Control     *event;
 
-    executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+    executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
     api = executing->API_Extensions[ THREAD_API_RTEMS ];
     event = &api->Event;
 
@@ -56,7 +56,7 @@ rtems_status_code rtems_event_receive(
       );
     } else {
       *event_out = event->pending_events;
-      _Thread_Lock_release_default( executing, &lock_context );
+      _Thread_Wait_release_default( executing, &lock_context );
       sc = RTEMS_SUCCESSFUL;
     }
   } else {
diff --git a/cpukit/rtems/src/eventseize.c b/cpukit/rtems/src/eventseize.c
index 7f5903df89..696481c2f5 100644
--- a/cpukit/rtems/src/eventseize.c
+++ b/cpukit/rtems/src/eventseize.c
@@ -50,13 +50,13 @@ rtems_status_code _Event_Seize(
        (seized_events == event_in || _Options_Is_any( option_set )) ) {
     event->pending_events =
       _Event_sets_Clear( pending_events, seized_events );
-    _Thread_Lock_release_default( executing, lock_context );
+    _Thread_Wait_release_default( executing, lock_context );
     *event_out = seized_events;
     return RTEMS_SUCCESSFUL;
   }
 
   if ( _Options_Is_no_wait( option_set ) ) {
-    _Thread_Lock_release_default( executing, lock_context );
+    _Thread_Wait_release_default( executing, lock_context );
     *event_out = seized_events;
     return RTEMS_UNSATISFIED;
   }
@@ -78,7 +78,7 @@ rtems_status_code _Event_Seize(
   _Thread_Wait_flags_set( executing, intend_to_block );
 
   cpu_self = _Thread_Dispatch_disable_critical( lock_context );
-  _Thread_Lock_release_default( executing, lock_context );
+  _Thread_Wait_release_default( executing, lock_context );
 
   if ( ticks ) {
     _Thread_Timer_insert_relative(
diff --git a/cpukit/rtems/src/eventsurrender.c b/cpukit/rtems/src/eventsurrender.c
index a9bef5916c..c805b0e89d 100644
--- a/cpukit/rtems/src/eventsurrender.c
+++ b/cpukit/rtems/src/eventsurrender.c
@@ -77,7 +77,7 @@ rtems_status_code _Event_Surrender(
   rtems_event_set seized_events;
   bool            unblock;
 
-  _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, lock_context );
 
   _Event_sets_Post( event_in, &event->pending_events );
   pending_events = event->pending_events;
@@ -116,14 +116,14 @@ rtems_status_code _Event_Surrender(
     Per_CPU_Control *cpu_self;
 
     cpu_self = _Thread_Dispatch_disable_critical( lock_context );
-    _Thread_Lock_release_default( the_thread, lock_context );
+    _Thread_Wait_release_default( the_thread, lock_context );
 
     _Thread_Timer_remove( the_thread );
     _Thread_Unblock( the_thread );
 
     _Thread_Dispatch_enable( cpu_self );
   } else {
-    _Thread_Lock_release_default( the_thread, lock_context );
+    _Thread_Wait_release_default( the_thread, lock_context );
   }
 
   return RTEMS_SUCCESSFUL;
diff --git a/cpukit/rtems/src/systemeventreceive.c b/cpukit/rtems/src/systemeventreceive.c
index a2215fa7ec..93a5694eec 100644
--- a/cpukit/rtems/src/systemeventreceive.c
+++ b/cpukit/rtems/src/systemeventreceive.c
@@ -44,7 +44,7 @@ rtems_status_code rtems_event_system_receive(
     RTEMS_API_Control *api;
     Event_Control     *event;
 
-    executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+    executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
     api = executing->API_Extensions[ THREAD_API_RTEMS ];
     event = &api->System_event;
 
@@ -62,7 +62,7 @@ rtems_status_code rtems_event_system_receive(
       );
     } else {
       *event_out = event->pending_events;
-      _Thread_Lock_release_default( executing, &lock_context );
+      _Thread_Wait_release_default( executing, &lock_context );
       sc = RTEMS_SUCCESSFUL;
     }
   } else {
diff --git a/cpukit/rtems/src/tasksetscheduler.c b/cpukit/rtems/src/tasksetscheduler.c
index 175f235f02..0ff74d9c97 100644
--- a/cpukit/rtems/src/tasksetscheduler.c
+++ b/cpukit/rtems/src/tasksetscheduler.c
@@ -28,10 +28,9 @@ rtems_status_code rtems_task_set_scheduler(
 {
   const Scheduler_Control *scheduler;
   Thread_Control          *the_thread;
-  ISR_lock_Context         lock_context;
-  ISR_lock_Context         state_lock_context;
+  Thread_queue_Context     queue_context;
+  ISR_lock_Context         state_context;
   Per_CPU_Control         *cpu_self;
-  void                    *lock;
   bool                     valid;
   Priority_Control         core_priority;
   Status_Control           status;
@@ -45,7 +44,7 @@ rtems_status_code rtems_task_set_scheduler(
     return RTEMS_INVALID_PRIORITY;
   }
 
-  the_thread = _Thread_Get( task_id, &lock_context );
+  the_thread = _Thread_Get( task_id, &queue_context.Lock_context );
 
   if ( the_thread == NULL ) {
 #if defined(RTEMS_MULTIPROCESSING)
@@ -57,16 +56,15 @@ rtems_status_code rtems_task_set_scheduler(
     return RTEMS_INVALID_ID;
   }
 
-  cpu_self = _Thread_Dispatch_disable_critical( &lock_context );
-  _ISR_lock_ISR_enable( &lock_context );
+  cpu_self = _Thread_Dispatch_disable_critical( &queue_context.Lock_context );
 
-  lock = _Thread_Lock_acquire( the_thread, &lock_context );
-  _Thread_State_acquire_critical( the_thread, &state_lock_context );
+  _Thread_Wait_acquire_critical( the_thread, &queue_context );
+  _Thread_State_acquire_critical( the_thread, &state_context );
 
   status = _Scheduler_Set( scheduler, the_thread, core_priority );
 
-  _Thread_State_release_critical( the_thread, &state_lock_context );
-  _Thread_Lock_release( lock, &lock_context );
+  _Thread_State_release_critical( the_thread, &state_context );
+  _Thread_Wait_release( the_thread, &queue_context );
   _Thread_Dispatch_enable( cpu_self );
   return _Status_Get( status );
 }
diff --git a/cpukit/score/include/rtems/score/schedulerimpl.h b/cpukit/score/include/rtems/score/schedulerimpl.h
index db4be99f79..f8c29e215c 100644
--- a/cpukit/score/include/rtems/score/schedulerimpl.h
+++ b/cpukit/score/include/rtems/score/schedulerimpl.h
@@ -825,6 +825,20 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Node_set_priority(
 #endif
 }
 
+RTEMS_INLINE_ROUTINE void _Scheduler_Thread_set_priority(
+  Thread_Control   *the_thread,
+  Priority_Control  new_priority,
+  bool              prepend_it
+)
+{
+  Scheduler_Node *own_node;
+
+  own_node = _Scheduler_Thread_get_own_node( the_thread );
+  _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
+
+  the_thread->current_priority = new_priority;
+}
+
 #if defined(RTEMS_SMP)
 /**
  * @brief Gets an idle thread from the scheduler instance.
diff --git a/cpukit/score/include/rtems/score/thread.h b/cpukit/score/include/rtems/score/thread.h
index 46c222ff52..a52c34f7fd 100644
--- a/cpukit/score/include/rtems/score/thread.h
+++ b/cpukit/score/include/rtems/score/thread.h
@@ -11,7 +11,7 @@
  *  COPYRIGHT (c) 1989-2014.
  *  On-Line Applications Research Corporation (OAR).
  *
- *  Copyright (c) 2014 embedded brains GmbH.
+ *  Copyright (c) 2014, 2016 embedded brains GmbH.
  *
  *  The license and distribution terms for this file may be
  *  found in the file LICENSE in this distribution or at
@@ -288,16 +288,6 @@ typedef struct {
   uint32_t              return_code;
 
   /**
-   * @brief The current thread queue.
-   *
-   * In case this field is @c NULL, then the thread is not blocked on a thread
-   * queue.  This field is protected by the thread lock.
-   *
-   * @see _Thread_Lock_set() and _Thread_Wait_set_queue().
-   */
-  Thread_queue_Queue *queue;
-
-  /**
    * @brief This field contains several flags used to control the wait class
    * and state of a thread in case fine-grained locking is used.
    */
@@ -307,12 +297,54 @@ typedef struct {
   Thread_Wait_flags     flags;
 #endif
 
+#if defined(RTEMS_SMP)
+  /**
+   * @brief Thread wait lock control block.
+   *
+   * Parts of the thread wait information are protected by the thread wait
+   * default lock and additionally a thread queue lock in case the thread
+   * is enqueued on a thread queue.
+   *
+   * The thread wait lock mechanism protects the following thread variables
+   *  - POSIX_API_Control::Attributes,
+   *  - Thread_Control::current_priority,
+   *  - Thread_Control::Wait::Lock::Pending_requests,
+   *  - Thread_Control::Wait::queue, and
+   *  - Thread_Control::Wait::operations.
+   *
+   * @see _Thread_Wait_acquire(), _Thread_Wait_release(), _Thread_Wait_claim(),
+   *   _Thread_Wait_restore_default() and _Thread_Wait_tranquilize().
+   */
+  struct {
+    /**
+     * @brief Thread wait default lock.
+     */
+    ISR_lock_Control Default;
+
+    /**
+     * @brief The pending thread wait lock acquire or tranquilize requests in
+     * case the thread is enqueued on a thread queue.
+     */
+    Chain_Control Pending_requests;
+  } Lock;
+#endif
+
+  /**
+   * @brief The current thread queue.
+   *
+   * If this field is NULL the thread is not enqueued on a thread queue.  This
+   * field is protected by the thread wait default lock.
+   *
+   * @see _Thread_Wait_claim().
+   */
+  Thread_queue_Queue *queue;
+
   /**
    * @brief The current thread queue operations.
    *
-   * This field is protected by the thread lock.
+   * This field is protected by the thread lock wait default lock.
    *
-   * @see _Thread_Lock_set() and _Thread_Wait_set_operations().
+   * @see _Thread_Wait_claim().
    */
   const Thread_queue_Operations *operations;
 
@@ -639,66 +671,6 @@ typedef struct  {
   void *        control;
 }Thread_Capture_control;
 
-#if defined(RTEMS_SMP)
-/**
- * @brief Thread lock control.
- *
- * The thread lock is either the default lock or the lock of the resource on
- * which the thread is currently blocked.  The generation number takes care
- * that the up to date lock is used.  Only resources using fine grained locking
- * provide their own lock.
- *
- * The thread lock protects the following thread variables
- *  - POSIX_API_Control::Attributes,
- *  - Thread_Control::current_priority,
- *  - Thread_Control::Wait::queue, and
- *  - Thread_Control::Wait::operations.
- *
- * @see _Thread_Lock_acquire(), _Thread_Lock_release(), _Thread_Lock_set() and
- * _Thread_Lock_restore_default().
- */
-typedef struct {
-  /**
-   * @brief The current thread lock.
-   *
-   * This is a plain ticket lock without SMP lock statistics support.  This
-   * enables external libraries to use thread locks since they are independent
-   * of the actual RTEMS build configuration, e.g. profiling enabled or
-   * disabled.
-   */
-  union {
-    /**
-     * @brief The current thread lock as an atomic unsigned integer pointer value.
-     */
-    Atomic_Uintptr atomic;
-
-    /**
-     * @brief The current thread lock as a normal pointer.
-     *
-     * Only provided for debugging purposes.
-     */
-    SMP_ticket_lock_Control *normal;
-  } current;
-
-  /**
-   * @brief The default thread lock in case the thread is not blocked on a
-   * resource.
-   */
-  SMP_ticket_lock_Control Default;
-
-#if defined(RTEMS_PROFILING)
-  /**
-   * @brief The thread lock statistics.
-   *
-   * These statistics are used by the executing thread in case it acquires a
-   * thread lock.  Thus the statistics are an aggregation of acquire and
-   * release operations of different locks.
-   */
-  SMP_lock_Stats Stats;
-#endif
-} Thread_Lock_control;
-#endif
-
 /**
  *  This structure defines the Thread Control Block (TCB).
  *
@@ -770,13 +742,6 @@ struct _Thread_Control {
 #endif
      /*================= end of common block =================*/
 
-#if defined(RTEMS_SMP)
-  /**
-   * @brief Thread lock control.
-   */
-  Thread_Lock_control Lock;
-#endif
-
 #if defined(RTEMS_SMP) && defined(RTEMS_PROFILING)
   /**
    * @brief Potpourri lock statistics.
diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h
index 92968a2e8e..ea1c61fa1e 100644
--- a/cpukit/score/include/rtems/score/threadimpl.h
+++ b/cpukit/score/include/rtems/score/threadimpl.h
@@ -11,7 +11,7 @@
  *  COPYRIGHT (c) 1989-2008.
  *  On-Line Applications Research Corporation (OAR).
  *
- *  Copyright (c) 2014-2015 embedded brains GmbH.
+ *  Copyright (c) 2014, 2016 embedded brains GmbH.
  *
  *  The license and distribution terms for this file may be
  *  found in the file LICENSE in this distribution or at
@@ -933,44 +933,35 @@ RTEMS_INLINE_ROUTINE bool _Thread_Owns_resources(
 }
 
 /**
- * @brief Acquires the default thread lock inside a critical section
+ * @brief Acquires the thread wait default lock inside a critical section
  * (interrupts disabled).
  *
  * @param[in] the_thread The thread.
  * @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ *   release.
  *
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default_critical().
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default_critical(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default_critical(
   Thread_Control   *the_thread,
   ISR_lock_Context *lock_context
 )
 {
-  _Assert( _ISR_Get_level() != 0 );
-#if defined(RTEMS_SMP)
-  _SMP_ticket_lock_Acquire(
-    &the_thread->Lock.Default,
-    &_Thread_Executing->Lock.Stats,
-    &lock_context->Lock_context.Stats_context
-  );
-#else
-  (void) the_thread;
-  (void) lock_context;
-#endif
+  _ISR_lock_Acquire( &the_thread->Wait.Lock.Default, lock_context );
 }
 
 /**
- * @brief Acquires the default thread lock and returns the executing thread.
+ * @brief Acquires the thread wait default lock and returns the executing
+ * thread.
  *
  * @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ *   release.
  *
  * @return The executing thread.
  *
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default().
  */
-RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing(
+RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Wait_acquire_default_for_executing(
   ISR_lock_Context *lock_context
 )
 {
@@ -978,247 +969,401 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing(
 
   _ISR_lock_ISR_disable( lock_context );
   executing = _Thread_Executing;
-  _Thread_Lock_acquire_default_critical( executing, lock_context );
+  _Thread_Wait_acquire_default_critical( executing, lock_context );
 
   return executing;
 }
 
 /**
- * @brief Acquires the default thread lock.
+ * @brief Acquires the thread wait default lock and disables interrupts.
  *
  * @param[in] the_thread The thread.
  * @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ *   release.
  *
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default().
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default(
   Thread_Control   *the_thread,
   ISR_lock_Context *lock_context
 )
 {
   _ISR_lock_ISR_disable( lock_context );
-  _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, lock_context );
 }
 
 /**
- * @brief Releases the thread lock inside a critical section (interrupts
- * disabled).
+ * @brief Releases the thread wait default lock inside a critical section
+ * (interrupts disabled).
  *
  * The previous interrupt status is not restored.
  *
- * @param[in] lock The lock.
+ * @param[in] the_thread The thread.
  * @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ *   acquire.
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_critical(
-  void             *lock,
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default_critical(
+  Thread_Control   *the_thread,
   ISR_lock_Context *lock_context
 )
 {
-#if defined(RTEMS_SMP)
-  _SMP_ticket_lock_Release(
-    (SMP_ticket_lock_Control *) lock,
-    &lock_context->Lock_context.Stats_context
-  );
-#else
-  (void) lock;
-  (void) lock_context;
-#endif
+  _ISR_lock_Release( &the_thread->Wait.Lock.Default, lock_context );
 }
 
 /**
- * @brief Releases the thread lock.
+ * @brief Releases the thread wait default lock and restores the previous
+ * interrupt status.
  *
- * @param[in] lock The lock returned by _Thread_Lock_acquire().
- * @param[in] lock_context The lock context used for _Thread_Lock_acquire().
+ * @param[in] the_thread The thread.
+ * @param[in] lock_context The lock context used for the corresponding lock
+ *   acquire.
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release(
-  void             *lock,
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default(
+  Thread_Control   *the_thread,
   ISR_lock_Context *lock_context
 )
 {
-  _Thread_Lock_release_critical( lock, lock_context );
+  _Thread_Wait_release_default_critical( the_thread, lock_context );
   _ISR_lock_ISR_enable( lock_context );
 }
 
+#if defined(RTEMS_SMP)
+#define THREAD_QUEUE_CONTEXT_OF_REQUEST( node ) \
+  RTEMS_CONTAINER_OF( node, Thread_queue_Context, Wait.Gate.Node )
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request_locked(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
+)
+{
+  _Chain_Extract_unprotected( &queue_context->Wait.Gate.Node );
+
+  if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) {
+    Thread_queue_Context *first;
+
+    first = THREAD_QUEUE_CONTEXT_OF_REQUEST(
+      _Chain_First( &the_thread->Wait.Lock.Pending_requests )
+    );
+
+    _Thread_queue_Gate_open( &first->Wait.Gate );
+  }
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_queue_critical(
+  SMP_ticket_lock_Control *queue_lock,
+  Thread_queue_Context    *queue_context
+)
+{
+  _SMP_ticket_lock_Acquire(
+    queue_lock,
+    &_Thread_Executing->Potpourri_stats,
+    &queue_context->Lock_context.Lock_context.Stats_context
+  );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_queue_critical(
+  SMP_ticket_lock_Control *queue_lock,
+  Thread_queue_Context    *queue_context
+)
+{
+  _SMP_ticket_lock_Release(
+    queue_lock,
+    &queue_context->Lock_context.Lock_context.Stats_context
+  );
+}
+#endif
+
 /**
- * @brief Releases the default thread lock inside a critical section
- * (interrupts disabled).
- *
- * The previous interrupt status is not restored.
+ * @brief Acquires the thread wait lock inside a critical section (interrupts
+ * disabled).
  *
  * @param[in] the_thread The thread.
- * @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ * @param[in] queue_context The thread queue context for the corresponding
+ *   _Thread_Wait_release_critical().
+ *
+ * @see _Thread_queue_Context_initialize().
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default_critical(
-  Thread_Control   *the_thread,
-  ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_critical(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
 )
 {
-  _Thread_Lock_release_critical(
 #if defined(RTEMS_SMP)
-    &the_thread->Lock.Default,
+  Thread_queue_Queue *queue;
+
+  _Thread_Wait_acquire_default_critical(
+    the_thread,
+    &queue_context->Lock_context
+  );
+
+  queue = the_thread->Wait.queue;
+  queue_context->Wait.queue = queue;
+  queue_context->Wait.operations = the_thread->Wait.operations;
+
+  if ( queue != NULL ) {
+    queue_context->Wait.queue_lock = &queue->Lock;
+    _Chain_Initialize_node( &queue_context->Wait.Gate.Node );
+    _Chain_Append_unprotected(
+      &the_thread->Wait.Lock.Pending_requests,
+      &queue_context->Wait.Gate.Node
+    );
+    _Thread_Wait_release_default_critical(
+      the_thread,
+      &queue_context->Lock_context
+    );
+    _Thread_Wait_acquire_queue_critical( &queue->Lock, queue_context );
+  } else {
+    queue_context->Wait.queue_lock = NULL;
+  }
 #else
-    NULL,
+  (void) the_thread;
+  (void) queue_context;
 #endif
-    lock_context
-  );
 }
 
 /**
- * @brief Releases the default thread lock.
+ * @brief Acquires the thread wait default lock and disables interrupts.
  *
  * @param[in] the_thread The thread.
- * @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ * @param[in] queue_context The thread queue context for the corresponding
+ *   _Thread_Wait_release().
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default(
-  Thread_Control   *the_thread,
-  ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
 )
 {
-  _Thread_Lock_release_default_critical( the_thread, lock_context );
-  _ISR_lock_ISR_enable( lock_context );
+  _Thread_queue_Context_initialize( queue_context );
+  _ISR_lock_ISR_disable( &queue_context->Lock_context );
+  _Thread_Wait_acquire_critical( the_thread, queue_context );
 }
 
 /**
- * @brief Acquires the thread lock.
+ * @brief Releases the thread wait lock inside a critical section (interrupts
+ * disabled).
  *
- * @param[in] the_thread The thread.
- * @param[in] lock_context The lock context for _Thread_Lock_release().
+ * The previous interrupt status is not restored.
  *
- * @return The lock required by _Thread_Lock_release().
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ *   _Thread_Wait_acquire_critical().
  */
-RTEMS_INLINE_ROUTINE void *_Thread_Lock_acquire(
-  Thread_Control   *the_thread,
-  ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_critical(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
 )
 {
 #if defined(RTEMS_SMP)
-  SMP_ticket_lock_Control *lock_0;
-
-  while ( true ) {
-    SMP_ticket_lock_Control *lock_1;
-
-    _ISR_lock_ISR_disable( lock_context );
-
-    /*
-     * We assume that a normal load of pointer is identical to a relaxed atomic
-     * load.  Here, we may read an out-of-date lock.  However, only the owner
-     * of this out-of-date lock is allowed to set a new one.  Thus, we read at
-     * least this new lock ...
-     */
-    lock_0 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr(
-      &the_thread->Lock.current.atomic,
-      ATOMIC_ORDER_RELAXED
-    );
+  SMP_ticket_lock_Control *queue_lock;
 
-    _SMP_ticket_lock_Acquire(
-      lock_0,
-      &_Thread_Executing->Lock.Stats,
-      &lock_context->Lock_context.Stats_context
-    );
+  queue_lock = queue_context->Wait.queue_lock;
 
-    /*
-     * We must use a load acquire here paired with the store release in
-     * _Thread_Lock_set_unprotected() to observe corresponding thread wait
-     * queue and thread wait operations.  It is important to do this after the
-     * lock acquire, since we may have the following scenario.
-     *
-     * - We read the default lock and try to acquire it.
-     * - The thread lock changes to a thread queue lock.
-     * - The thread lock is restored to the default lock.
-     * - We acquire the default lock and read it here again.
-     * - Now, we must read the restored default thread wait queue and thread
-     *   wait operations and this is not synchronized via the default thread
-     *   lock.
-     */
-    lock_1 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr(
-      &the_thread->Lock.current.atomic,
-      ATOMIC_ORDER_ACQUIRE
+  if ( queue_lock != NULL ) {
+    _Thread_Wait_release_queue_critical( queue_lock, queue_context );
+    _Thread_Wait_acquire_default_critical(
+      the_thread,
+      &queue_context->Lock_context
     );
 
-    /*
-     * ... here, and so on.
-     */
-    if ( lock_0 == lock_1 ) {
-      return lock_0;
-    }
-
-    _Thread_Lock_release( lock_0, lock_context );
+    _Thread_Wait_remove_request_locked( the_thread, queue_context );
   }
-#else
-  _ISR_Local_disable( lock_context->isr_level );
 
-  return NULL;
+  _Thread_Wait_release_default_critical(
+    the_thread,
+    &queue_context->Lock_context
+  );
+#else
+  (void) the_thread;
+  (void) queue_context;
 #endif
 }
 
-#if defined(RTEMS_SMP)
-/*
- * Internal function, use _Thread_Lock_set() or _Thread_Lock_restore_default()
- * instead.
+/**
+ * @brief Releases the thread wait lock and restores the previous interrupt
+ * status.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ *   _Thread_Wait_acquire().
  */
-RTEMS_INLINE_ROUTINE void _Thread_Lock_set_unprotected(
-  Thread_Control          *the_thread,
-  SMP_ticket_lock_Control *new_lock
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
 )
 {
-  _Atomic_Store_uintptr(
-    &the_thread->Lock.current.atomic,
-    (uintptr_t) new_lock,
-    ATOMIC_ORDER_RELEASE
-  );
+  _Thread_Wait_release_critical( the_thread, queue_context );
+  _ISR_lock_ISR_enable( &queue_context->Lock_context );
 }
-#endif
 
 /**
- * @brief Sets a new thread lock.
+ * @brief Claims the thread wait queue and operations.
  *
- * The caller must not be the owner of the default thread lock.  The caller
- * must be the owner of the new lock.
+ * The caller must not be the owner of the default thread wait lock.  The
+ * caller must be the owner of the corresponding thread queue lock.
  *
  * @param[in] the_thread The thread.
- * @param[in] new_lock The new thread lock.
+ * @param[in] queue The new thread queue.
+ * @param[in] operations The new thread operations.
+ *
+ * @see _Thread_Wait_restore_default().
  */
-#if defined(RTEMS_SMP)
-RTEMS_INLINE_ROUTINE void _Thread_Lock_set(
-  Thread_Control          *the_thread,
-  SMP_ticket_lock_Control *new_lock
+RTEMS_INLINE_ROUTINE void _Thread_Wait_claim(
+  Thread_Control                *the_thread,
+  Thread_queue_Queue            *queue,
+  const Thread_queue_Operations *operations
 )
 {
   ISR_lock_Context lock_context;
 
-  _Thread_Lock_acquire_default_critical( the_thread, &lock_context );
-  _Assert( the_thread->Lock.current.normal == &the_thread->Lock.Default );
-  _Thread_Lock_set_unprotected( the_thread, new_lock );
-  _Thread_Lock_release_default_critical( the_thread, &lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+
+  _Assert( the_thread->Wait.queue == NULL );
+  the_thread->Wait.queue = queue;
+  the_thread->Wait.operations = operations;
+
+  _Thread_Wait_release_default_critical( the_thread, &lock_context );
 }
+
+/**
+ * @brief Removes a thread wait lock request.
+ *
+ * On SMP configurations, removes a thread wait lock request.
+ *
+ * On other configurations, this function does nothing.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ *   _Thread_Wait_acquire().
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
+)
+{
+#if defined(RTEMS_SMP)
+  ISR_lock_Context lock_context;
+
+  _Thread_Wait_acquire_default( the_thread, &lock_context );
+  _Thread_Wait_remove_request_locked( the_thread, queue_context );
+  _Thread_Wait_release_default( the_thread, &lock_context );
 #else
-#define _Thread_Lock_set( the_thread, new_lock ) \
-  do { } while ( 0 )
+  (void) the_thread;
+  (void) queue_context;
 #endif
+}
 
 /**
- * @brief Restores the default thread lock.
+ * @brief Restores the default thread wait queue and operations.
+ *
+ * The caller must be the owner of the current thread wait queue lock.
  *
- * The caller must be the owner of the current thread lock.
+ * On SMP configurations, the pending requests are updated to use the stale
+ * thread queue operations.
  *
  * @param[in] the_thread The thread.
+ *
+ * @see _Thread_Wait_claim().
  */
-#if defined(RTEMS_SMP)
-RTEMS_INLINE_ROUTINE void _Thread_Lock_restore_default(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default(
   Thread_Control *the_thread
 )
 {
-  _Thread_Lock_set_unprotected( the_thread, &the_thread->Lock.Default );
+#if defined(RTEMS_SMP)
+  ISR_lock_Context  lock_context;
+  Chain_Node       *node;
+  const Chain_Node *tail;
+
+  _Thread_Wait_acquire_default_critical(
+    the_thread,
+    &lock_context
+  );
+
+  node = _Chain_First( &the_thread->Wait.Lock.Pending_requests );
+  tail = _Chain_Immutable_tail( &the_thread->Wait.Lock.Pending_requests );
+
+  while ( node != tail ) {
+    Thread_queue_Context *queue_context;
+
+    queue_context = THREAD_QUEUE_CONTEXT_OF_REQUEST( node );
+    queue_context->Wait.queue = NULL;
+    queue_context->Wait.operations = &_Thread_queue_Operations_stale_queue;
+
+    node = _Chain_Next( node );
+  }
+#endif
+
+  the_thread->Wait.queue = NULL;
+  the_thread->Wait.operations = &_Thread_queue_Operations_default;
+
+#if defined(RTEMS_SMP)
+  _Thread_Wait_release_default_critical(
+    the_thread,
+    &lock_context
+  );
+#endif
 }
+
+/**
+ * @brief Tranquilizes the thread after a wait a thread queue.
+ *
+ * After the violent blocking procedure this function makes the thread calm and
+ * peaceful again so that it can carry out its normal work.
+ *
+ * On SMP configurations, ensures that all pending thread wait lock requests
+ * completed before the thread is able to begin a new thread wait procedure.
+ *
+ * On other configurations, this function does nothing.
+ *
+ * @param[in] the_thread The thread.
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_tranquilize(
+  Thread_Control *the_thread
+)
+{
+#if defined(RTEMS_SMP)
+  Thread_queue_Context queue_context;
+
+  _Thread_queue_Context_initialize( &queue_context );
+  _Thread_Wait_acquire_default( the_thread, &queue_context.Lock_context );
+
+  if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) {
+    _Thread_queue_Gate_add(
+      &the_thread->Wait.Lock.Pending_requests,
+      &queue_context.Wait.Gate
+    );
+    _Thread_Wait_release_default( the_thread, &queue_context.Lock_context );
+    _Thread_queue_Gate_wait( &queue_context.Wait.Gate );
+    _Thread_Wait_remove_request( the_thread, &queue_context );
+  } else {
+    _Thread_Wait_release_default( the_thread, &queue_context.Lock_context );
+  }
 #else
-#define _Thread_Lock_restore_default( the_thread ) \
-  do { } while ( 0 )
+  (void) the_thread;
 #endif
+}
+
+/**
+ * @brief Cancels a thread wait on a thread queue.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ *   _Thread_Wait_acquire().
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_cancel(
+  Thread_Control       *the_thread,
+  Thread_queue_Context *queue_context
+)
+{
+  _Thread_queue_Context_extract( queue_context, the_thread );
+
+#if defined(RTEMS_SMP)
+  if ( queue_context->Wait.queue != NULL ) {
+#endif
+    _Thread_Wait_restore_default( the_thread );
+#if defined(RTEMS_SMP)
+  }
+#endif
+}
 
 /**
  * @brief The initial thread wait flags value set by _Thread_Initialize().
@@ -1388,58 +1533,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Wait_flags_try_change_acquire(
 }
 
 /**
- * @brief Sets the thread queue.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- * @param[in] new_queue The new queue.
- *
- * @see _Thread_Lock_set().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_set_queue(
-  Thread_Control     *the_thread,
-  Thread_queue_Queue *new_queue
-)
-{
-  the_thread->Wait.queue = new_queue;
-}
-
-/**
- * @brief Sets the thread queue operations.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- * @param[in] new_operations The new queue operations.
- *
- * @see _Thread_Lock_set() and _Thread_Wait_restore_default_operations().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_set_operations(
-  Thread_Control                *the_thread,
-  const Thread_queue_Operations *new_operations
-)
-{
-  the_thread->Wait.operations = new_operations;
-}
-
-/**
- * @brief Restores the default thread queue operations.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- *
- * @see _Thread_Wait_set_operations().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default_operations(
-  Thread_Control *the_thread
-)
-{
-  the_thread->Wait.operations = &_Thread_queue_Operations_default;
-}
-
-/**
  * @brief Returns the object identifier of the object containing the current
  * thread wait queue.
  *
@@ -1541,6 +1634,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Remove_timer_and_unblock(
   Thread_queue_Queue *queue
 )
 {
+  _Thread_Wait_tranquilize( the_thread );
   _Thread_Timer_remove( the_thread );
 
 #if defined(RTEMS_MULTIPROCESSING)
diff --git a/cpukit/score/include/rtems/score/threadq.h b/cpukit/score/include/rtems/score/threadq.h
index 75aab20993..9a178049ea 100644
--- a/cpukit/score/include/rtems/score/threadq.h
+++ b/cpukit/score/include/rtems/score/threadq.h
@@ -43,6 +43,10 @@ extern "C" {
 
 typedef struct _Thread_Control Thread_Control;
 
+typedef struct Thread_queue_Queue Thread_queue_Queue;
+
+typedef struct Thread_queue_Operations Thread_queue_Operations;
+
 typedef struct Thread_queue_Path Thread_queue_Path;
 
 #if defined(RTEMS_MULTIPROCESSING)
@@ -53,6 +57,8 @@ typedef struct Thread_queue_Path Thread_queue_Path;
  *   control is actually a thread proxy if and only if
  *   _Objects_Is_local_id( the_proxy->Object.id ) is false.
  * @param mp_id Object identifier of the object containing the thread queue.
+ *
+ * @see _Thread_queue_Context_set_MP_callout().
  */
 typedef void ( *Thread_queue_MP_callout )(
   Thread_Control *the_proxy,
@@ -60,6 +66,24 @@ typedef void ( *Thread_queue_MP_callout )(
 );
 #endif
 
+#if defined(RTEMS_SMP)
+/**
+ * @brief The thread queue gate is an SMP synchronization means.
+ *
+ * The gates are added to a list of requests.  A busy wait is performed to make
+ * sure that preceding requests are carried out.  Each predecessor notifies its
+ * successor about on request completion.
+ *
+ * @see _Thread_queue_Gate_add(), _Thread_queue_Gate_wait(), and
+ *   _Thread_queue_Gate_open().
+ */
+typedef struct {
+  Chain_Node Node;
+
+  Atomic_Uint go_ahead;
+} Thread_queue_Gate;
+#endif
+
 /**
  * @brief Thread queue context for the thread queue methods.
  *
@@ -105,8 +129,64 @@ typedef struct {
 #if defined(RTEMS_MULTIPROCESSING)
   Thread_queue_MP_callout mp_callout;
 #endif
+
+#if defined(RTEMS_SMP)
+  /**
+   * @brief Data to support thread queue enqueue operations.
+   */
+  struct {
+    /**
+     * @brief Gate to synchronize thread wait lock requests.
+     *
+     * @see _Thread_Wait_acquire_critical() and _Thread_Wait_tranquilize().
+     */
+    Thread_queue_Gate Gate;
+
+    /**
+     * @brief The thread queue lock in case the thread is blocked on a thread
+     * queue at thread wait lock acquire time.
+     */
+    SMP_ticket_lock_Control *queue_lock;
+
+    /**
+     * @brief The thread queue after thread wait lock acquire.
+     *
+     * In case the thread queue is NULL and the thread queue lock is non-NULL
+     * in this context, then we have a stale thread queue.  This happens in
+     * case the thread wait default is restored while we wait on the thread
+     * queue lock, e.g. during a mutex ownership transfer.
+     *
+     * @see _Thread_Wait_restore_default().
+     */
+    Thread_queue_Queue *queue;
+
+    /**
+     * @brief The thread queue operations after thread wait lock acquire.
+     */
+    const Thread_queue_Operations *operations;
+  } Wait;
+#endif
 } Thread_queue_Context;
 
+#if defined(RTEMS_SMP)
+/**
+ * @brief A thread queue link from one thread to another specified by the
+ * thread queue owner and thread wait queue relationships.
+ */
+typedef struct {
+  /**
+   * @brief The owner of this thread queue link.
+   */
+  Thread_Control *owner;
+
+  /**
+   * @brief The queue context used to acquire the thread wait lock of the
+   * owner.
+   */
+  Thread_queue_Context Queue_context;
+} Thread_queue_Link;
+#endif
+
 /**
  * @brief Thread priority queue.
  */
@@ -191,7 +271,7 @@ typedef struct _Thread_queue_Heads {
     sizeof( Thread_queue_Heads )
 #endif
 
-typedef struct {
+struct Thread_queue_Queue {
   /**
    * @brief Lock to protect this thread queue.
    *
@@ -221,13 +301,15 @@ typedef struct {
    * @brief The thread queue owner.
    */
   Thread_Control *owner;
-} Thread_queue_Queue;
+};
 
 /**
  * @brief Thread queue priority change operation.
  *
  * @param[in] the_thread The thread.
  * @param[in] new_priority The new priority value.
+ * @param[in] prepend_it In case this is true, then the thread is prepended to
+ *   its priority group in its scheduler instance, otherwise it is appended.
  * @param[in] queue The actual thread queue.
  *
  * @see Thread_queue_Operations.
@@ -235,6 +317,7 @@ typedef struct {
 typedef void ( *Thread_queue_Priority_change_operation )(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 );
 
@@ -247,8 +330,6 @@ typedef void ( *Thread_queue_Priority_change_operation )(
  *
  * @param[in] queue The actual thread queue.
  * @param[in] the_thread The thread to enqueue on the queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef void ( *Thread_queue_Enqueue_operation )(
   Thread_queue_Queue *queue,
@@ -261,8 +342,6 @@ typedef void ( *Thread_queue_Enqueue_operation )(
  *
  * @param[in] queue The actual thread queue.
  * @param[in] the_thread The thread to extract from the thread queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef void ( *Thread_queue_Extract_operation )(
   Thread_queue_Queue *queue,
@@ -277,8 +356,6 @@ typedef void ( *Thread_queue_Extract_operation )(
  * @retval NULL No thread is present on the thread queue.
  * @retval first The first thread of the thread queue according to the insert
  * order.  This thread remains on the thread queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef Thread_Control *( *Thread_queue_First_operation )(
   Thread_queue_Heads *heads
@@ -289,7 +366,7 @@ typedef Thread_Control *( *Thread_queue_First_operation )(
  *
  * @see _Thread_wait_Set_operations().
  */
-typedef struct {
+struct Thread_queue_Operations {
   /**
    * @brief Thread queue priority change operation.
    *
@@ -320,7 +397,7 @@ typedef struct {
    * @brief Thread queue first operation.
    */
   Thread_queue_First_operation first;
-} Thread_queue_Operations;
+};
 
 /**
  *  This is the structure used to manage sets of tasks which are blocked
diff --git a/cpukit/score/include/rtems/score/threadqimpl.h b/cpukit/score/include/rtems/score/threadqimpl.h
index 1f17f1a4f1..3d4f6136f6 100644
--- a/cpukit/score/include/rtems/score/threadqimpl.h
+++ b/cpukit/score/include/rtems/score/threadqimpl.h
@@ -26,6 +26,10 @@
 #include <rtems/score/smp.h>
 #include <rtems/score/thread.h>
 
+#if defined(RTEMS_DEBUG)
+#include <string.h>
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -46,6 +50,13 @@ extern "C" {
  * relationships.
  */
 struct Thread_queue_Path {
+#if defined(RTEMS_SMP)
+  /**
+   * @brief The start of a thread queue path.
+   */
+  Thread_queue_Link Start;
+#endif
+
   /**
    * @brief A potential thread to update the priority via
    * _Thread_Update_priority().
@@ -84,10 +95,8 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_initialize(
 )
 {
 #if defined(RTEMS_DEBUG)
+  memset( queue_context, 0, sizeof( *queue_context ) );
   queue_context->expected_thread_dispatch_disable_level = 0xdeadbeef;
-#if defined(RTEMS_MULTIPROCESSING)
-  queue_context->mp_callout = NULL;
-#endif
 #else
   (void) queue_context;
 #endif
@@ -187,6 +196,135 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_set_MP_callout(
   } while ( 0 )
 #endif
 
+/**
+ * @brief Gets the thread wait queue of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+  ( queue_context )->Wait.queue
+#else
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+  ( the_thread )->Wait.queue
+#endif
+
+/**
+ * @brief Gets the thread wait operations of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+  ( queue_context )->Wait.operations
+#else
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+  ( the_thread )->Wait.operations
+#endif
+
+/**
+ * @brief Thread priority change by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ * @param new_priority The new thread priority.
+ * @param prepend_it Prepend it to its priority group or not.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_priority_change( \
+    queue_context, \
+    the_thread, \
+    new_priority, \
+    prepend_it \
+  ) \
+    ( *( queue_context )->Wait.operations->priority_change )( \
+      the_thread, \
+      new_priority, \
+      prepend_it, \
+      ( queue_context )->Wait.queue \
+    )
+#else
+#define _Thread_queue_Context_priority_change( \
+    queue_context, \
+    the_thread, \
+    new_priority, \
+    prepend_it \
+  ) \
+    ( *( the_thread )->Wait.operations->priority_change )( \
+      the_thread, \
+      new_priority, \
+      prepend_it, \
+      ( the_thread )->Wait.queue \
+    )
+#endif
+
+/**
+ * @brief Thread queue extract by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_extract( \
+    queue_context, \
+    the_thread \
+  ) \
+    ( *( queue_context )->Wait.operations->extract )( \
+      ( queue_context )->Wait.queue, \
+      the_thread \
+    )
+#else
+#define _Thread_queue_Context_extract( \
+    queue_context, \
+    the_thread \
+  ) \
+    ( *( the_thread )->Wait.operations->extract )( \
+      ( the_thread )->Wait.queue, \
+      the_thread \
+    )
+#endif
+
+#if defined(RTEMS_SMP)
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_add(
+  Chain_Control     *chain,
+  Thread_queue_Gate *gate
+)
+{
+  _Atomic_Store_uint( &gate->go_ahead, 0, ATOMIC_ORDER_RELAXED );
+  _Chain_Append_unprotected( chain, &gate->Node );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_open(
+  Thread_queue_Gate *gate
+)
+{
+  _Atomic_Store_uint( &gate->go_ahead, 1, ATOMIC_ORDER_RELAXED );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_wait(
+  Thread_queue_Gate *gate
+)
+{
+  while ( _Atomic_Load_uint( &gate->go_ahead, ATOMIC_ORDER_RELAXED ) == 0 ) {
+    /* Wait */
+  }
+}
+#endif
+
 RTEMS_INLINE_ROUTINE void _Thread_queue_Heads_initialize(
   Thread_queue_Heads *heads
 )
@@ -911,6 +1049,10 @@ extern const Thread_queue_Operations _Thread_queue_Operations_priority;
 
 extern const Thread_queue_Operations _Thread_queue_Operations_priority_inherit;
 
+#if defined(RTEMS_SMP)
+extern const Thread_queue_Operations _Thread_queue_Operations_stale_queue;
+#endif
+
 /**@}*/
 
 #ifdef __cplusplus
diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c
index e3cf4a165a..c569530d74 100644
--- a/cpukit/score/src/threadchangepriority.c
+++ b/cpukit/score/src/threadchangepriority.c
@@ -27,7 +27,8 @@ static Thread_Control *_Thread_Apply_priority_locked(
   Priority_Control               new_priority,
   void                          *arg,
   Thread_Change_priority_filter  filter,
-  bool                           prepend_it
+  bool                           prepend_it,
+  Thread_queue_Context          *queue_context
 )
 {
   /*
@@ -45,17 +46,11 @@ static Thread_Control *_Thread_Apply_priority_locked(
    *  we are not REALLY changing priority.
    */
   if ( ( *filter )( the_thread, &new_priority, arg ) ) {
-    Scheduler_Node *own_node;
-
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
-    _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
-
-    the_thread->current_priority = new_priority;
-
-    ( *the_thread->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      queue_context,
       the_thread,
       new_priority,
-      the_thread->Wait.queue
+      prepend_it
     );
   } else {
     the_thread = NULL;
@@ -72,19 +67,20 @@ Thread_Control *_Thread_Apply_priority(
   bool                           prepend_it
 )
 {
-  ISR_lock_Context  lock_context;
-  ISR_lock_Control *lock;
+  Thread_queue_Context  queue_context;
+  Thread_Control       *the_thread_to_update;
 
-  lock = _Thread_Lock_acquire( the_thread, &lock_context );
-  the_thread = _Thread_Apply_priority_locked(
+  _Thread_Wait_acquire( the_thread, &queue_context );
+  the_thread_to_update = _Thread_Apply_priority_locked(
     the_thread,
     new_priority,
     arg,
     filter,
-    prepend_it
+    prepend_it,
+    &queue_context
   );
-  _Thread_Lock_release( lock, &lock_context );
-  return the_thread;
+  _Thread_Wait_release( the_thread, &queue_context );
+  return the_thread_to_update;
 }
 
 void _Thread_Update_priority( Thread_Control *the_thread )
diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c
index 940537f135..209be9ebb1 100644
--- a/cpukit/score/src/threadinitialize.c
+++ b/cpukit/score/src/threadinitialize.c
@@ -182,13 +182,11 @@ bool _Thread_Initialize(
   the_thread->Scheduler.control = scheduler;
   the_thread->Scheduler.own_node = scheduler_node;
   _Resource_Node_initialize( &the_thread->Resource_node );
-  _Atomic_Store_uintptr(
-    &the_thread->Lock.current.atomic,
-    (uintptr_t) &the_thread->Lock.Default,
-    ATOMIC_ORDER_RELAXED
+  _ISR_lock_Initialize(
+    &the_thread->Wait.Lock.Default,
+    "Thread Wait Default Lock"
   );
-  _SMP_ticket_lock_Initialize( &the_thread->Lock.Default );
-  _SMP_lock_Stats_initialize( &the_thread->Lock.Stats, "Thread Lock" );
+  _Chain_Initialize_empty( &the_thread->Wait.Lock.Pending_requests );
   _SMP_lock_Stats_initialize( &the_thread->Potpourri_stats, "Thread Potpourri" );
 #endif
 
diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c
index 818073c2b1..19c345b6c5 100644
--- a/cpukit/score/src/threadqenqueue.c
+++ b/cpukit/score/src/threadqenqueue.c
@@ -34,6 +34,51 @@
 #define THREAD_QUEUE_READY_AGAIN \
   (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN)
 
+static void _Thread_queue_Path_release( Thread_queue_Path *path )
+{
+#if defined(RTEMS_SMP)
+  Thread_queue_Link *link;
+
+  link = &path->Start;
+
+  if ( link->owner != NULL ) {
+    _Thread_Wait_release_critical( link->owner, &link->Queue_context );
+  }
+#else
+  (void) path;
+#endif
+}
+
+static void _Thread_queue_Path_acquire(
+  Thread_Control     *the_thread,
+  Thread_queue_Queue *queue,
+  Thread_queue_Path  *path
+)
+{
+#if defined(RTEMS_SMP)
+  Thread_Control     *owner;
+  Thread_queue_Link  *link;
+
+  owner = queue->owner;
+
+  if ( owner == NULL ) {
+    return;
+  }
+
+  link = &path->Start;
+  link->owner = owner;
+
+  _Thread_Wait_acquire_default_critical(
+    owner,
+    &link->Queue_context.Lock_context
+  );
+#else
+  (void) the_thread;
+  (void) queue;
+  (void) path;
+#endif
+}
+
 void _Thread_queue_Enqueue_critical(
   Thread_queue_Queue            *queue,
   const Thread_queue_Operations *operations,
@@ -52,14 +97,13 @@ void _Thread_queue_Enqueue_critical(
   }
 #endif
 
-  _Thread_Lock_set( the_thread, &queue->Lock );
-
-  the_thread->Wait.return_code = STATUS_SUCCESSFUL;
-  _Thread_Wait_set_queue( the_thread, queue );
-  _Thread_Wait_set_operations( the_thread, operations );
+  _Thread_Wait_claim( the_thread, queue, operations );
 
+  _Thread_queue_Path_acquire( the_thread, queue, &path );
   ( *operations->enqueue )( queue, the_thread, &path );
+  _Thread_queue_Path_release( &path );
 
+  the_thread->Wait.return_code = STATUS_SUCCESSFUL;
   _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK );
   cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context );
   _Thread_queue_Queue_release( queue, &queue_context->Lock_context );
@@ -173,9 +217,7 @@ bool _Thread_queue_Do_extract_locked(
     unblock = true;
   }
 
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
+  _Thread_Wait_restore_default( the_thread );
 
   return unblock;
 }
@@ -227,30 +269,35 @@ void _Thread_queue_Extract_critical(
 
 void _Thread_queue_Extract( Thread_Control *the_thread )
 {
-  Thread_queue_Context  queue_context;
-  void                 *lock;
-  Thread_queue_Queue   *queue;
+  Thread_queue_Context queue_context;
 
   _Thread_queue_Context_initialize( &queue_context );
-  lock = _Thread_Lock_acquire( the_thread, &queue_context.Lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
-  queue = the_thread->Wait.queue;
-
-  if ( queue != NULL ) {
-    _SMP_Assert( lock == &queue->Lock );
+  if (
+    _Thread_queue_Context_get_queue( &queue_context, the_thread ) != NULL
+  ) {
+    bool unblock;
 
+    _Thread_Wait_remove_request( the_thread, &queue_context );
     _Thread_queue_Context_set_MP_callout(
       &queue_context,
       _Thread_queue_MP_callout_do_nothing
     );
-    _Thread_queue_Extract_critical(
-      queue,
-      the_thread->Wait.operations,
+    unblock = _Thread_queue_Extract_locked(
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      _Thread_queue_Context_get_operations( &queue_context, the_thread ),
       the_thread,
-      &queue_context
+      &queue_context.Lock_context
+    );
+    _Thread_queue_Unblock_critical(
+      unblock,
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      the_thread,
+      &queue_context.Lock_context
     );
   } else {
-    _Thread_Lock_release( lock, &queue_context.Lock_context );
+    _Thread_Wait_release( the_thread, &queue_context );
   }
 }
 
@@ -273,8 +320,6 @@ Thread_Control *_Thread_queue_Do_dequeue(
   the_thread = _Thread_queue_First_locked( the_thread_queue, operations );
 
   if ( the_thread != NULL ) {
-    _SMP_Assert( the_thread->Lock.current.normal == &the_thread_queue->Queue.Lock );
-
     _Thread_queue_Extract_critical(
       &the_thread_queue->Queue,
       operations,
diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c
index c288ceeca9..8059446578 100644
--- a/cpukit/score/src/threadqops.c
+++ b/cpukit/score/src/threadqops.c
@@ -22,13 +22,16 @@
 #include <rtems/score/rbtreeimpl.h>
 #include <rtems/score/schedulerimpl.h>
 
-static void _Thread_queue_Do_nothing_priority_change(
+static void _Thread_queue_Default_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
-  /* Do nothing */
+  (void) queue;
+
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
 }
 
 static void _Thread_queue_Do_nothing_extract(
@@ -39,6 +42,31 @@ static void _Thread_queue_Do_nothing_extract(
   /* Do nothing */
 }
 
+#if defined(RTEMS_SMP)
+static void _Thread_queue_Stale_queue_priority_change(
+  Thread_Control     *the_thread,
+  Priority_Control    new_priority,
+  bool                prepend_it,
+  Thread_queue_Queue *queue
+)
+{
+  ISR_lock_Context lock_context;
+
+  (void) queue;
+
+  /*
+   * This operation is used to change the priority in case we have a thread
+   * queue context with a stale thread queue.  We own the thread queue lock of
+   * the former thread queue.  In addition, we need the thread wait default
+   * lock, see _Thread_Wait_restore_default().
+   */
+
+  _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+  _Thread_Wait_release_default_critical( the_thread, &lock_context );
+}
+#endif
+
 static Thread_queue_Heads *_Thread_queue_Queue_enqueue(
   Thread_queue_Queue *queue,
   Thread_Control     *the_thread,
@@ -190,6 +218,7 @@ static bool _Thread_queue_Priority_less(
 static void _Thread_queue_Priority_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
@@ -198,6 +227,8 @@ static void _Thread_queue_Priority_priority_change(
 
   _Assert( heads != NULL );
 
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+
   priority_queue = _Thread_queue_Priority_queue( heads, the_thread );
 
   _RBTree_Extract(
@@ -353,22 +384,16 @@ static void _Thread_queue_Priority_inherit_enqueue(
 #endif
 
   if ( priority < owner->current_priority ) {
-    Scheduler_Node *own_node;
-
     path->update_priority = owner;
 
     owner->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
-    own_node = _Scheduler_Thread_get_own_node( owner );
-    _Scheduler_Node_set_priority( own_node, priority, false );
-
-    owner->current_priority = priority;
-
-    ( *owner->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      &path->Start.Queue_context,
       owner,
       priority,
-      owner->Wait.queue
+      false
     );
   } else {
     path->update_priority = NULL;
@@ -385,24 +410,21 @@ void _Thread_queue_Boost_priority(
 
   if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) {
     const Scheduler_Control *scheduler;
-    Scheduler_Node          *own_node;
     Priority_Control         boost_priority;
 
     the_thread->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
     scheduler = _Scheduler_Get_own( the_thread );
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
     boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR );
-    _Scheduler_Node_set_priority( own_node, boost_priority, false );
 
-    the_thread->current_priority = boost_priority;
+    _Scheduler_Thread_set_priority( the_thread, boost_priority, false );
   }
 }
 #endif
 
 const Thread_queue_Operations _Thread_queue_Operations_default = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .extract = _Thread_queue_Do_nothing_extract
   /*
    * The default operations are only used in _Thread_Change_priority() and
@@ -412,7 +434,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = {
 };
 
 const Thread_queue_Operations _Thread_queue_Operations_FIFO = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .enqueue = _Thread_queue_FIFO_enqueue,
   .extract = _Thread_queue_FIFO_extract,
   .first = _Thread_queue_FIFO_first
@@ -431,3 +453,10 @@ const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = {
   .extract = _Thread_queue_Priority_extract,
   .first = _Thread_queue_Priority_first
 };
+
+#if defined(RTEMS_SMP)
+const Thread_queue_Operations _Thread_queue_Operations_stale_queue = {
+  .priority_change = _Thread_queue_Stale_queue_priority_change,
+  .extract = _Thread_queue_Do_nothing_extract
+};
+#endif
diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c
index 757f552699..81ba73d704 100644
--- a/cpukit/score/src/threadrestart.c
+++ b/cpukit/score/src/threadrestart.c
@@ -215,8 +215,7 @@ static void _Thread_Free( Thread_Control *the_thread )
   _Workspace_Free( the_thread->Start.tls_area );
 
 #if defined(RTEMS_SMP)
-  _SMP_ticket_lock_Destroy( &the_thread->Lock.Default );
-  _SMP_lock_Stats_destroy( &the_thread->Lock.Stats );
+  _ISR_lock_Destroy( &the_thread->Wait.Lock.Default );
   _SMP_lock_Stats_destroy( &the_thread->Potpourri_stats );
 #endif
 
diff --git a/cpukit/score/src/threadtimeout.c b/cpukit/score/src/threadtimeout.c
index a2ba61f9e7..b6b6cc4ac2 100644
--- a/cpukit/score/src/threadtimeout.c
+++ b/cpukit/score/src/threadtimeout.c
@@ -22,28 +22,15 @@
 #include <rtems/score/threadimpl.h>
 #include <rtems/score/status.h>
 
-static void _Thread_Do_timeout( Thread_Control *the_thread )
-{
-  the_thread->Wait.return_code = STATUS_TIMEOUT;
-  ( *the_thread->Wait.operations->extract )(
-    the_thread->Wait.queue,
-    the_thread
-  );
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
-}
-
 void _Thread_Timeout( Watchdog_Control *watchdog )
 {
-  Thread_Control    *the_thread;
-  void              *thread_lock;
-  ISR_lock_Context   lock_context;
-  Thread_Wait_flags  wait_flags;
-  bool               unblock;
+  Thread_Control       *the_thread;
+  Thread_queue_Context  queue_context;
+  Thread_Wait_flags     wait_flags;
+  bool                  unblock;
 
   the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog );
-  thread_lock = _Thread_Lock_acquire( the_thread, &lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
   wait_flags = _Thread_Wait_flags_get( the_thread );
 
@@ -52,7 +39,9 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     Thread_Wait_flags ready_again;
     bool              success;
 
-    _Thread_Do_timeout( the_thread );
+    _Thread_Wait_cancel( the_thread, &queue_context );
+
+    the_thread->Wait.return_code = STATUS_TIMEOUT;
 
     wait_class = wait_flags & THREAD_WAIT_CLASS_MASK;
     ready_again = wait_class | THREAD_WAIT_STATE_READY_AGAIN;
@@ -76,9 +65,10 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     unblock = false;
   }
 
-  _Thread_Lock_release( thread_lock, &lock_context );
+  _Thread_Wait_release( the_thread, &queue_context );
 
   if ( unblock ) {
+    _Thread_Wait_tranquilize( the_thread );
     _Thread_Unblock( the_thread );
 
 #if defined(RTEMS_MULTIPROCESSING)
diff --git a/testsuites/sptests/spthreadq01/init.c b/testsuites/sptests/spthreadq01/init.c
index 85df6bd15c..afaa6527e0 100644
--- a/testsuites/sptests/spthreadq01/init.c
+++ b/testsuites/sptests/spthreadq01/init.c
@@ -68,13 +68,12 @@ static void wake_up_master(test_context *ctx)
 
 static rtems_id get_wait_id(test_context *ctx)
 {
-  ISR_lock_Context lock_context;
-  void *lock;
+  Thread_queue_Context queue_context;
   rtems_id id;
 
-  lock = _Thread_Lock_acquire(ctx->master, &lock_context);
+  _Thread_Wait_acquire(ctx->master, &queue_context);
   id = _Thread_Wait_get_id(ctx->master);
-  _Thread_Lock_release(lock, &lock_context);
+  _Thread_Wait_release(ctx->master, &queue_context);
 
   return id;
 }