score: New thread queue implementation

Use thread wait flags for synchronization.  The enqueue operation is now
part of the initial critical section.  This is the key change and
enables fine grained locking on SMP for objects using a thread queue
like semaphores and message queues.

Update #2273.

1 files changed, 104 insertions, 128 deletions

diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c
index be08ffcbf1..d02f2ee073 100644
--- a/cpukit/score/src/threadqenqueue.c
+++ b/cpukit/score/src/threadqenqueue.c
@@ -20,58 +20,28 @@
 
 #include <rtems/score/threadqimpl.h>
 #include <rtems/score/assert.h>
+#include <rtems/score/threaddispatch.h>
 #include <rtems/score/threadimpl.h>
 #include <rtems/score/watchdogimpl.h>
 
-/**
- *  @brief Finalize a blocking operation.
- *
- *  This method is used to finalize a blocking operation that was
- *  satisfied. It may be used with thread queues or any other synchronization
- *  object that uses the blocking states and watchdog times for timeout.
- *
- *  This method will restore the previous ISR disable level during the cancel
- *  operation.  Thus it is an implicit _ISR_Enable().
- *
- *  @param[in] the_thread is the thread whose blocking is canceled
- *  @param[in] lock_context is the previous ISR disable level
- */
-static void _Thread_blocking_operation_Finalize(
-  Thread_queue_Control *the_thread_queue,
-  Thread_Control       *the_thread,
-  ISR_lock_Context     *lock_context
-)
-{
-  /*
-   * The thread is not waiting on anything after this completes.
-   */
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
+#define THREAD_QUEUE_INTEND_TO_BLOCK \
+  (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_INTEND_TO_BLOCK)
 
-  _Thread_Lock_restore_default( the_thread );
+#define THREAD_QUEUE_BLOCKED \
+  (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_BLOCKED)
 
-  /*
-   *  If the sync state is timed out, this is very likely not needed.
-   *  But better safe than sorry when it comes to critical sections.
-   */
-  if ( _Watchdog_Is_active( &the_thread->Timer ) ) {
-    _Watchdog_Deactivate( &the_thread->Timer );
-    _Thread_queue_Release( the_thread_queue, lock_context );
-    _Watchdog_Remove_ticks( &the_thread->Timer );
-  } else
-    _Thread_queue_Release( the_thread_queue, lock_context );
-
-  /*
-   *  Global objects with thread queue's should not be operated on from an
-   *  ISR.  But the sync code still must allow short timeouts to be processed
-   *  correctly.
-   */
+#define THREAD_QUEUE_READY_AGAIN \
+  (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN)
 
+static void _Thread_queue_Unblock( Thread_Control *the_thread )
+{
+  _Watchdog_Remove_ticks( &the_thread->Timer );
   _Thread_Unblock( the_thread );
 
 #if defined(RTEMS_MULTIPROCESSING)
-  if ( !_Objects_Is_local_id( the_thread->Object.id ) )
+  if ( !_Objects_Is_local_id( the_thread->Object.id ) ) {
     _Thread_MP_Free_proxy( the_thread );
+  }
 #endif
 }
 
@@ -83,13 +53,20 @@ void _Thread_queue_Enqueue_critical(
   ISR_lock_Context     *lock_context
 )
 {
-  Thread_blocking_operation_States sync_state;
+  const Thread_queue_Operations *operations;
+  Per_CPU_Control               *cpu_self;
+  bool                           success;
 
   _Thread_Lock_set( the_thread, &the_thread_queue->Lock );
 
-  the_thread_queue->sync_state = THREAD_BLOCKING_OPERATION_NOTHING_HAPPENED;
+  operations = the_thread_queue->operations;
   _Thread_Wait_set_queue( the_thread, the_thread_queue );
+  _Thread_Wait_set_operations( the_thread, operations );
+
+  ( *operations->enqueue )( the_thread_queue, the_thread );
 
+  _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK );
+  cpu_self = _Thread_Dispatch_disable_critical();
   _Thread_queue_Release( the_thread_queue, lock_context );
 
 #if defined(RTEMS_MULTIPROCESSING)
@@ -105,121 +82,120 @@ void _Thread_queue_Enqueue_critical(
   /*
    *  If the thread wants to timeout, then schedule its timer.
    */
-  if ( timeout ) {
-    _Watchdog_Initialize(
-       &the_thread->Timer,
-       _Thread_queue_Timeout,
-       the_thread->Object.id,
-       NULL
+  if ( timeout != WATCHDOG_NO_TIMEOUT ) {
+    _Thread_Wait_set_timeout_code(
+      the_thread,
+      the_thread_queue->timeout_status
     );
-
+    _Watchdog_Initialize( &the_thread->Timer, _Thread_Timeout, 0, the_thread );
     _Watchdog_Insert_ticks( &the_thread->Timer, timeout );
   }
 
-  /*
-   * Now initiate the enqueuing and checking if the blocking operation
-   * should be completed or the thread has had its blocking condition
-   * satisfied before we got here.
-   */
-  _Thread_queue_Acquire( the_thread_queue, lock_context );
+  success = _Thread_Wait_flags_try_change(
+    the_thread,
+    THREAD_QUEUE_INTEND_TO_BLOCK,
+    THREAD_QUEUE_BLOCKED
+  );
+  if ( !success ) {
+    _Thread_queue_Unblock( the_thread );
+  }
 
-  sync_state = the_thread_queue->sync_state;
-  the_thread_queue->sync_state = THREAD_BLOCKING_OPERATION_SYNCHRONIZED;
+  _Thread_Dispatch_enable( cpu_self );
+}
 
-  if ( sync_state == THREAD_BLOCKING_OPERATION_NOTHING_HAPPENED ) {
-    const Thread_queue_Operations *operations;
+void _Thread_queue_Extract_locked(
+  Thread_queue_Control *the_thread_queue,
+  Thread_Control       *the_thread
+)
+{
+  ( *the_thread_queue->operations->extract )( the_thread_queue, the_thread );
 
-    the_thread_queue->sync_state = THREAD_BLOCKING_OPERATION_SYNCHRONIZED;
+  _Thread_Wait_set_queue( the_thread, NULL );
+  _Thread_Wait_restore_default_operations( the_thread );
+  _Thread_Lock_restore_default( the_thread );
+}
 
-    operations = the_thread_queue->operations;
-    _Thread_Wait_set_operations( the_thread, operations );
-    ( *operations->enqueue )( the_thread_queue, the_thread );
+void _Thread_queue_Unblock_critical(
+  Thread_queue_Control *the_thread_queue,
+  Thread_Control       *the_thread,
+  ISR_lock_Context     *lock_context
+)
+{
+  bool success;
+  bool unblock;
 
-    _Thread_queue_Release( the_thread_queue, lock_context );
+  success = _Thread_Wait_flags_try_change_critical(
+    the_thread,
+    THREAD_QUEUE_INTEND_TO_BLOCK,
+    THREAD_QUEUE_READY_AGAIN
+  );
+  if ( success ) {
+    unblock = false;
   } else {
-    /* Cancel a blocking operation due to ISR */
+    _Assert( _Thread_Wait_flags_get( the_thread ) == THREAD_QUEUE_BLOCKED );
+    _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_READY_AGAIN );
+    unblock = true;
+  }
 
-    _Assert(
-      sync_state == THREAD_BLOCKING_OPERATION_TIMEOUT ||
-        sync_state == THREAD_BLOCKING_OPERATION_SATISFIED
-    );
+  if ( unblock ) {
+    Per_CPU_Control *cpu_self;
 
-    _Thread_blocking_operation_Finalize( the_thread_queue, the_thread, lock_context );
+    cpu_self = _Thread_Dispatch_disable_critical();
+    _Thread_queue_Release( the_thread_queue, lock_context );
+
+    _Thread_queue_Unblock( the_thread );
+
+    _Thread_Dispatch_enable( cpu_self );
+  } else {
+    _Thread_queue_Release( the_thread_queue, lock_context );
   }
 }
 
-void _Thread_queue_Extract_with_return_code(
-  Thread_Control *the_thread,
-  uint32_t        return_code
+void _Thread_queue_Extract_critical(
+  Thread_queue_Control *the_thread_queue,
+  Thread_Control       *the_thread,
+  ISR_lock_Context     *lock_context
 )
 {
-  Thread_queue_Control *the_thread_queue;
-  ISR_lock_Control     *lock;
+  _Thread_queue_Extract_locked( the_thread_queue, the_thread );
+  _Thread_queue_Unblock_critical( the_thread_queue, the_thread, lock_context );
+}
+
+void _Thread_queue_Extract( Thread_Control *the_thread )
+{
   ISR_lock_Context      lock_context;
+  ISR_lock_Control     *lock;
+  Thread_queue_Control *the_thread_queue;
 
   lock = _Thread_Lock_acquire( the_thread, &lock_context );
 
   the_thread_queue = the_thread->Wait.queue;
-  if ( the_thread_queue == NULL ) {
-    _Thread_Lock_release( lock, &lock_context );
-    return;
-  }
-
-  _SMP_Assert( lock == &the_thread_queue->Lock );
-
-  ( *the_thread_queue->operations->extract )( the_thread_queue, the_thread );
 
-  the_thread->Wait.return_code = return_code;
+  if ( the_thread_queue != NULL ) {
+    _SMP_Assert( lock == &the_thread_queue->Lock );
 
-  /*
-   * We found a thread to unblock.
-   *
-   * NOTE: This is invoked with interrupts still disabled.
-   */
-  _Thread_blocking_operation_Finalize( the_thread_queue, the_thread, &lock_context );
-}
-
-void _Thread_queue_Extract( Thread_Control *the_thread )
-{
-  _Thread_queue_Extract_with_return_code(
-    the_thread,
-    the_thread->Wait.return_code
-  );
+    _Thread_queue_Extract_critical( the_thread_queue, the_thread, &lock_context );
+  } else {
+    _Thread_Lock_release( lock, &lock_context );
+  }
 }
 
-Thread_Control *_Thread_queue_Dequeue(
-  Thread_queue_Control *the_thread_queue
-)
+Thread_Control *_Thread_queue_Dequeue( Thread_queue_Control *the_thread_queue )
 {
-  Thread_Control                   *the_thread;
-  ISR_lock_Context                  lock_context;
-  Thread_blocking_operation_States  sync_state;
+  ISR_lock_Context  lock_context;
+  Thread_Control   *the_thread;
 
   _Thread_queue_Acquire( the_thread_queue, &lock_context );
 
-  the_thread = ( *the_thread_queue->operations->dequeue )( the_thread_queue );
-  if ( the_thread == NULL ) {
-    /*
-     * We did not find a thread to unblock in the queue.  Maybe the executing
-     * thread is about to block on this thread queue.
-     */
-    sync_state = the_thread_queue->sync_state;
-    if ( (sync_state == THREAD_BLOCKING_OPERATION_TIMEOUT) ||
-         (sync_state == THREAD_BLOCKING_OPERATION_NOTHING_HAPPENED) ) {
-      the_thread_queue->sync_state = THREAD_BLOCKING_OPERATION_SATISFIED;
-      the_thread = _Thread_Executing;
-    } else {
-      _Thread_queue_Release( the_thread_queue, &lock_context );
-      return NULL;
-    }
-  }
+  the_thread = _Thread_queue_First_locked( the_thread_queue );
 
-  /*
-   * We found a thread to unblock.
-   *
-   * NOTE: This is invoked with interrupts still disabled.
-   */
-  _Thread_blocking_operation_Finalize( the_thread_queue, the_thread, &lock_context );
+  if ( the_thread != NULL ) {
+    _SMP_Assert( the_thread->Lock.current == &the_thread_queue->Lock );
+
+    _Thread_queue_Extract_critical( the_thread_queue, the_thread, &lock_context );
+  } else {
+    _Thread_queue_Release( the_thread_queue, &lock_context );
+  }
 
   return the_thread;
 }