diff options
author | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2016-07-25 16:35:37 +0200 |
---|---|---|
committer | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2016-07-27 10:55:30 +0200 |
commit | 1fcac5adc5ed38fb88ce4c6d24b2ca2e27e3cd10 (patch) | |
tree | 7d2fed265befa680902ff146acb3305a360f9116 /cpukit/score/include | |
parent | score: Priority inherit thread queue operations (diff) | |
download | rtems-1fcac5adc5ed38fb88ce4c6d24b2ca2e27e3cd10.tar.bz2 |
score: Turn thread lock into thread wait lock
The _Thread_Lock_acquire() function had a potentially infinite run-time
due to the lack of fairness at atomic operations level.
Update #2412.
Update #2556.
Update #2765.
Diffstat (limited to 'cpukit/score/include')
-rw-r--r-- | cpukit/score/include/rtems/score/schedulerimpl.h | 14 | ||||
-rw-r--r-- | cpukit/score/include/rtems/score/thread.h | 125 | ||||
-rw-r--r-- | cpukit/score/include/rtems/score/threadimpl.h | 518 | ||||
-rw-r--r-- | cpukit/score/include/rtems/score/threadq.h | 97 | ||||
-rw-r--r-- | cpukit/score/include/rtems/score/threadqimpl.h | 148 |
5 files changed, 597 insertions, 305 deletions
diff --git a/cpukit/score/include/rtems/score/schedulerimpl.h b/cpukit/score/include/rtems/score/schedulerimpl.h index db4be99f79..f8c29e215c 100644 --- a/cpukit/score/include/rtems/score/schedulerimpl.h +++ b/cpukit/score/include/rtems/score/schedulerimpl.h @@ -825,6 +825,20 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Node_set_priority( #endif } +RTEMS_INLINE_ROUTINE void _Scheduler_Thread_set_priority( + Thread_Control *the_thread, + Priority_Control new_priority, + bool prepend_it +) +{ + Scheduler_Node *own_node; + + own_node = _Scheduler_Thread_get_own_node( the_thread ); + _Scheduler_Node_set_priority( own_node, new_priority, prepend_it ); + + the_thread->current_priority = new_priority; +} + #if defined(RTEMS_SMP) /** * @brief Gets an idle thread from the scheduler instance. diff --git a/cpukit/score/include/rtems/score/thread.h b/cpukit/score/include/rtems/score/thread.h index 46c222ff52..a52c34f7fd 100644 --- a/cpukit/score/include/rtems/score/thread.h +++ b/cpukit/score/include/rtems/score/thread.h @@ -11,7 +11,7 @@ * COPYRIGHT (c) 1989-2014. * On-Line Applications Research Corporation (OAR). * - * Copyright (c) 2014 embedded brains GmbH. + * Copyright (c) 2014, 2016 embedded brains GmbH. * * The license and distribution terms for this file may be * found in the file LICENSE in this distribution or at @@ -288,16 +288,6 @@ typedef struct { uint32_t return_code; /** - * @brief The current thread queue. - * - * In case this field is @c NULL, then the thread is not blocked on a thread - * queue. This field is protected by the thread lock. - * - * @see _Thread_Lock_set() and _Thread_Wait_set_queue(). - */ - Thread_queue_Queue *queue; - - /** * @brief This field contains several flags used to control the wait class * and state of a thread in case fine-grained locking is used. */ @@ -307,12 +297,54 @@ typedef struct { Thread_Wait_flags flags; #endif +#if defined(RTEMS_SMP) + /** + * @brief Thread wait lock control block. + * + * Parts of the thread wait information are protected by the thread wait + * default lock and additionally a thread queue lock in case the thread + * is enqueued on a thread queue. + * + * The thread wait lock mechanism protects the following thread variables + * - POSIX_API_Control::Attributes, + * - Thread_Control::current_priority, + * - Thread_Control::Wait::Lock::Pending_requests, + * - Thread_Control::Wait::queue, and + * - Thread_Control::Wait::operations. + * + * @see _Thread_Wait_acquire(), _Thread_Wait_release(), _Thread_Wait_claim(), + * _Thread_Wait_restore_default() and _Thread_Wait_tranquilize(). + */ + struct { + /** + * @brief Thread wait default lock. + */ + ISR_lock_Control Default; + + /** + * @brief The pending thread wait lock acquire or tranquilize requests in + * case the thread is enqueued on a thread queue. + */ + Chain_Control Pending_requests; + } Lock; +#endif + + /** + * @brief The current thread queue. + * + * If this field is NULL the thread is not enqueued on a thread queue. This + * field is protected by the thread wait default lock. + * + * @see _Thread_Wait_claim(). + */ + Thread_queue_Queue *queue; + /** * @brief The current thread queue operations. * - * This field is protected by the thread lock. + * This field is protected by the thread lock wait default lock. * - * @see _Thread_Lock_set() and _Thread_Wait_set_operations(). + * @see _Thread_Wait_claim(). */ const Thread_queue_Operations *operations; @@ -639,66 +671,6 @@ typedef struct { void * control; }Thread_Capture_control; -#if defined(RTEMS_SMP) -/** - * @brief Thread lock control. - * - * The thread lock is either the default lock or the lock of the resource on - * which the thread is currently blocked. The generation number takes care - * that the up to date lock is used. Only resources using fine grained locking - * provide their own lock. - * - * The thread lock protects the following thread variables - * - POSIX_API_Control::Attributes, - * - Thread_Control::current_priority, - * - Thread_Control::Wait::queue, and - * - Thread_Control::Wait::operations. - * - * @see _Thread_Lock_acquire(), _Thread_Lock_release(), _Thread_Lock_set() and - * _Thread_Lock_restore_default(). - */ -typedef struct { - /** - * @brief The current thread lock. - * - * This is a plain ticket lock without SMP lock statistics support. This - * enables external libraries to use thread locks since they are independent - * of the actual RTEMS build configuration, e.g. profiling enabled or - * disabled. - */ - union { - /** - * @brief The current thread lock as an atomic unsigned integer pointer value. - */ - Atomic_Uintptr atomic; - - /** - * @brief The current thread lock as a normal pointer. - * - * Only provided for debugging purposes. - */ - SMP_ticket_lock_Control *normal; - } current; - - /** - * @brief The default thread lock in case the thread is not blocked on a - * resource. - */ - SMP_ticket_lock_Control Default; - -#if defined(RTEMS_PROFILING) - /** - * @brief The thread lock statistics. - * - * These statistics are used by the executing thread in case it acquires a - * thread lock. Thus the statistics are an aggregation of acquire and - * release operations of different locks. - */ - SMP_lock_Stats Stats; -#endif -} Thread_Lock_control; -#endif - /** * This structure defines the Thread Control Block (TCB). * @@ -770,13 +742,6 @@ struct _Thread_Control { #endif /*================= end of common block =================*/ -#if defined(RTEMS_SMP) - /** - * @brief Thread lock control. - */ - Thread_Lock_control Lock; -#endif - #if defined(RTEMS_SMP) && defined(RTEMS_PROFILING) /** * @brief Potpourri lock statistics. diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h index 92968a2e8e..ea1c61fa1e 100644 --- a/cpukit/score/include/rtems/score/threadimpl.h +++ b/cpukit/score/include/rtems/score/threadimpl.h @@ -11,7 +11,7 @@ * COPYRIGHT (c) 1989-2008. * On-Line Applications Research Corporation (OAR). * - * Copyright (c) 2014-2015 embedded brains GmbH. + * Copyright (c) 2014, 2016 embedded brains GmbH. * * The license and distribution terms for this file may be * found in the file LICENSE in this distribution or at @@ -933,44 +933,35 @@ RTEMS_INLINE_ROUTINE bool _Thread_Owns_resources( } /** - * @brief Acquires the default thread lock inside a critical section + * @brief Acquires the thread wait default lock inside a critical section * (interrupts disabled). * * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default_critical(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default_critical( +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default_critical( Thread_Control *the_thread, ISR_lock_Context *lock_context ) { - _Assert( _ISR_Get_level() != 0 ); -#if defined(RTEMS_SMP) - _SMP_ticket_lock_Acquire( - &the_thread->Lock.Default, - &_Thread_Executing->Lock.Stats, - &lock_context->Lock_context.Stats_context - ); -#else - (void) the_thread; - (void) lock_context; -#endif + _ISR_lock_Acquire( &the_thread->Wait.Lock.Default, lock_context ); } /** - * @brief Acquires the default thread lock and returns the executing thread. + * @brief Acquires the thread wait default lock and returns the executing + * thread. * * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * * @return The executing thread. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default(). */ -RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing( +RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Wait_acquire_default_for_executing( ISR_lock_Context *lock_context ) { @@ -978,247 +969,401 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing( _ISR_lock_ISR_disable( lock_context ); executing = _Thread_Executing; - _Thread_Lock_acquire_default_critical( executing, lock_context ); + _Thread_Wait_acquire_default_critical( executing, lock_context ); return executing; } /** - * @brief Acquires the default thread lock. + * @brief Acquires the thread wait default lock and disables interrupts. * * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default( +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default( Thread_Control *the_thread, ISR_lock_Context *lock_context ) { _ISR_lock_ISR_disable( lock_context ); - _Thread_Lock_acquire_default_critical( the_thread, lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, lock_context ); } /** - * @brief Releases the thread lock inside a critical section (interrupts - * disabled). + * @brief Releases the thread wait default lock inside a critical section + * (interrupts disabled). * * The previous interrupt status is not restored. * - * @param[in] lock The lock. + * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * acquire. */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_critical( - void *lock, +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default_critical( + Thread_Control *the_thread, ISR_lock_Context *lock_context ) { -#if defined(RTEMS_SMP) - _SMP_ticket_lock_Release( - (SMP_ticket_lock_Control *) lock, - &lock_context->Lock_context.Stats_context - ); -#else - (void) lock; - (void) lock_context; -#endif + _ISR_lock_Release( &the_thread->Wait.Lock.Default, lock_context ); } /** - * @brief Releases the thread lock. + * @brief Releases the thread wait default lock and restores the previous + * interrupt status. * - * @param[in] lock The lock returned by _Thread_Lock_acquire(). - * @param[in] lock_context The lock context used for _Thread_Lock_acquire(). + * @param[in] the_thread The thread. + * @param[in] lock_context The lock context used for the corresponding lock + * acquire. */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release( - void *lock, +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default( + Thread_Control *the_thread, ISR_lock_Context *lock_context ) { - _Thread_Lock_release_critical( lock, lock_context ); + _Thread_Wait_release_default_critical( the_thread, lock_context ); _ISR_lock_ISR_enable( lock_context ); } +#if defined(RTEMS_SMP) +#define THREAD_QUEUE_CONTEXT_OF_REQUEST( node ) \ + RTEMS_CONTAINER_OF( node, Thread_queue_Context, Wait.Gate.Node ) + +RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request_locked( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Chain_Extract_unprotected( &queue_context->Wait.Gate.Node ); + + if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) { + Thread_queue_Context *first; + + first = THREAD_QUEUE_CONTEXT_OF_REQUEST( + _Chain_First( &the_thread->Wait.Lock.Pending_requests ) + ); + + _Thread_queue_Gate_open( &first->Wait.Gate ); + } +} + +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_queue_critical( + SMP_ticket_lock_Control *queue_lock, + Thread_queue_Context *queue_context +) +{ + _SMP_ticket_lock_Acquire( + queue_lock, + &_Thread_Executing->Potpourri_stats, + &queue_context->Lock_context.Lock_context.Stats_context + ); +} + +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_queue_critical( + SMP_ticket_lock_Control *queue_lock, + Thread_queue_Context *queue_context +) +{ + _SMP_ticket_lock_Release( + queue_lock, + &queue_context->Lock_context.Lock_context.Stats_context + ); +} +#endif + /** - * @brief Releases the default thread lock inside a critical section - * (interrupts disabled). - * - * The previous interrupt status is not restored. + * @brief Acquires the thread wait lock inside a critical section (interrupts + * disabled). * * @param[in] the_thread The thread. - * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * @param[in] queue_context The thread queue context for the corresponding + * _Thread_Wait_release_critical(). + * + * @see _Thread_queue_Context_initialize(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default_critical( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_critical( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Thread_Lock_release_critical( #if defined(RTEMS_SMP) - &the_thread->Lock.Default, + Thread_queue_Queue *queue; + + _Thread_Wait_acquire_default_critical( + the_thread, + &queue_context->Lock_context + ); + + queue = the_thread->Wait.queue; + queue_context->Wait.queue = queue; + queue_context->Wait.operations = the_thread->Wait.operations; + + if ( queue != NULL ) { + queue_context->Wait.queue_lock = &queue->Lock; + _Chain_Initialize_node( &queue_context->Wait.Gate.Node ); + _Chain_Append_unprotected( + &the_thread->Wait.Lock.Pending_requests, + &queue_context->Wait.Gate.Node + ); + _Thread_Wait_release_default_critical( + the_thread, + &queue_context->Lock_context + ); + _Thread_Wait_acquire_queue_critical( &queue->Lock, queue_context ); + } else { + queue_context->Wait.queue_lock = NULL; + } #else - NULL, + (void) the_thread; + (void) queue_context; #endif - lock_context - ); } /** - * @brief Releases the default thread lock. + * @brief Acquires the thread wait default lock and disables interrupts. * * @param[in] the_thread The thread. - * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * @param[in] queue_context The thread queue context for the corresponding + * _Thread_Wait_release(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Thread_Lock_release_default_critical( the_thread, lock_context ); - _ISR_lock_ISR_enable( lock_context ); + _Thread_queue_Context_initialize( queue_context ); + _ISR_lock_ISR_disable( &queue_context->Lock_context ); + _Thread_Wait_acquire_critical( the_thread, queue_context ); } /** - * @brief Acquires the thread lock. + * @brief Releases the thread wait lock inside a critical section (interrupts + * disabled). * - * @param[in] the_thread The thread. - * @param[in] lock_context The lock context for _Thread_Lock_release(). + * The previous interrupt status is not restored. * - * @return The lock required by _Thread_Lock_release(). + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire_critical(). */ -RTEMS_INLINE_ROUTINE void *_Thread_Lock_acquire( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_critical( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { #if defined(RTEMS_SMP) - SMP_ticket_lock_Control *lock_0; - - while ( true ) { - SMP_ticket_lock_Control *lock_1; - - _ISR_lock_ISR_disable( lock_context ); - - /* - * We assume that a normal load of pointer is identical to a relaxed atomic - * load. Here, we may read an out-of-date lock. However, only the owner - * of this out-of-date lock is allowed to set a new one. Thus, we read at - * least this new lock ... - */ - lock_0 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr( - &the_thread->Lock.current.atomic, - ATOMIC_ORDER_RELAXED - ); + SMP_ticket_lock_Control *queue_lock; - _SMP_ticket_lock_Acquire( - lock_0, - &_Thread_Executing->Lock.Stats, - &lock_context->Lock_context.Stats_context - ); + queue_lock = queue_context->Wait.queue_lock; - /* - * We must use a load acquire here paired with the store release in - * _Thread_Lock_set_unprotected() to observe corresponding thread wait - * queue and thread wait operations. It is important to do this after the - * lock acquire, since we may have the following scenario. - * - * - We read the default lock and try to acquire it. - * - The thread lock changes to a thread queue lock. - * - The thread lock is restored to the default lock. - * - We acquire the default lock and read it here again. - * - Now, we must read the restored default thread wait queue and thread - * wait operations and this is not synchronized via the default thread - * lock. - */ - lock_1 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr( - &the_thread->Lock.current.atomic, - ATOMIC_ORDER_ACQUIRE + if ( queue_lock != NULL ) { + _Thread_Wait_release_queue_critical( queue_lock, queue_context ); + _Thread_Wait_acquire_default_critical( + the_thread, + &queue_context->Lock_context ); - /* - * ... here, and so on. - */ - if ( lock_0 == lock_1 ) { - return lock_0; - } - - _Thread_Lock_release( lock_0, lock_context ); + _Thread_Wait_remove_request_locked( the_thread, queue_context ); } -#else - _ISR_Local_disable( lock_context->isr_level ); - return NULL; + _Thread_Wait_release_default_critical( + the_thread, + &queue_context->Lock_context + ); +#else + (void) the_thread; + (void) queue_context; #endif } -#if defined(RTEMS_SMP) -/* - * Internal function, use _Thread_Lock_set() or _Thread_Lock_restore_default() - * instead. +/** + * @brief Releases the thread wait lock and restores the previous interrupt + * status. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_set_unprotected( - Thread_Control *the_thread, - SMP_ticket_lock_Control *new_lock +RTEMS_INLINE_ROUTINE void _Thread_Wait_release( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Atomic_Store_uintptr( - &the_thread->Lock.current.atomic, - (uintptr_t) new_lock, - ATOMIC_ORDER_RELEASE - ); + _Thread_Wait_release_critical( the_thread, queue_context ); + _ISR_lock_ISR_enable( &queue_context->Lock_context ); } -#endif /** - * @brief Sets a new thread lock. + * @brief Claims the thread wait queue and operations. * - * The caller must not be the owner of the default thread lock. The caller - * must be the owner of the new lock. + * The caller must not be the owner of the default thread wait lock. The + * caller must be the owner of the corresponding thread queue lock. * * @param[in] the_thread The thread. - * @param[in] new_lock The new thread lock. + * @param[in] queue The new thread queue. + * @param[in] operations The new thread operations. + * + * @see _Thread_Wait_restore_default(). */ -#if defined(RTEMS_SMP) -RTEMS_INLINE_ROUTINE void _Thread_Lock_set( - Thread_Control *the_thread, - SMP_ticket_lock_Control *new_lock +RTEMS_INLINE_ROUTINE void _Thread_Wait_claim( + Thread_Control *the_thread, + Thread_queue_Queue *queue, + const Thread_queue_Operations *operations ) { ISR_lock_Context lock_context; - _Thread_Lock_acquire_default_critical( the_thread, &lock_context ); - _Assert( the_thread->Lock.current.normal == &the_thread->Lock.Default ); - _Thread_Lock_set_unprotected( the_thread, new_lock ); - _Thread_Lock_release_default_critical( the_thread, &lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, &lock_context ); + + _Assert( the_thread->Wait.queue == NULL ); + the_thread->Wait.queue = queue; + the_thread->Wait.operations = operations; + + _Thread_Wait_release_default_critical( the_thread, &lock_context ); } + +/** + * @brief Removes a thread wait lock request. + * + * On SMP configurations, removes a thread wait lock request. + * + * On other configurations, this function does nothing. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ +#if defined(RTEMS_SMP) + ISR_lock_Context lock_context; + + _Thread_Wait_acquire_default( the_thread, &lock_context ); + _Thread_Wait_remove_request_locked( the_thread, queue_context ); + _Thread_Wait_release_default( the_thread, &lock_context ); #else -#define _Thread_Lock_set( the_thread, new_lock ) \ - do { } while ( 0 ) + (void) the_thread; + (void) queue_context; #endif +} /** - * @brief Restores the default thread lock. + * @brief Restores the default thread wait queue and operations. + * + * The caller must be the owner of the current thread wait queue lock. * - * The caller must be the owner of the current thread lock. + * On SMP configurations, the pending requests are updated to use the stale + * thread queue operations. * * @param[in] the_thread The thread. + * + * @see _Thread_Wait_claim(). */ -#if defined(RTEMS_SMP) -RTEMS_INLINE_ROUTINE void _Thread_Lock_restore_default( +RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default( Thread_Control *the_thread ) { - _Thread_Lock_set_unprotected( the_thread, &the_thread->Lock.Default ); +#if defined(RTEMS_SMP) + ISR_lock_Context lock_context; + Chain_Node *node; + const Chain_Node *tail; + + _Thread_Wait_acquire_default_critical( + the_thread, + &lock_context + ); + + node = _Chain_First( &the_thread->Wait.Lock.Pending_requests ); + tail = _Chain_Immutable_tail( &the_thread->Wait.Lock.Pending_requests ); + + while ( node != tail ) { + Thread_queue_Context *queue_context; + + queue_context = THREAD_QUEUE_CONTEXT_OF_REQUEST( node ); + queue_context->Wait.queue = NULL; + queue_context->Wait.operations = &_Thread_queue_Operations_stale_queue; + + node = _Chain_Next( node ); + } +#endif + + the_thread->Wait.queue = NULL; + the_thread->Wait.operations = &_Thread_queue_Operations_default; + +#if defined(RTEMS_SMP) + _Thread_Wait_release_default_critical( + the_thread, + &lock_context + ); +#endif } + +/** + * @brief Tranquilizes the thread after a wait a thread queue. + * + * After the violent blocking procedure this function makes the thread calm and + * peaceful again so that it can carry out its normal work. + * + * On SMP configurations, ensures that all pending thread wait lock requests + * completed before the thread is able to begin a new thread wait procedure. + * + * On other configurations, this function does nothing. + * + * @param[in] the_thread The thread. + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_tranquilize( + Thread_Control *the_thread +) +{ +#if defined(RTEMS_SMP) + Thread_queue_Context queue_context; + + _Thread_queue_Context_initialize( &queue_context ); + _Thread_Wait_acquire_default( the_thread, &queue_context.Lock_context ); + + if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) { + _Thread_queue_Gate_add( + &the_thread->Wait.Lock.Pending_requests, + &queue_context.Wait.Gate + ); + _Thread_Wait_release_default( the_thread, &queue_context.Lock_context ); + _Thread_queue_Gate_wait( &queue_context.Wait.Gate ); + _Thread_Wait_remove_request( the_thread, &queue_context ); + } else { + _Thread_Wait_release_default( the_thread, &queue_context.Lock_context ); + } #else -#define _Thread_Lock_restore_default( the_thread ) \ - do { } while ( 0 ) + (void) the_thread; #endif +} + +/** + * @brief Cancels a thread wait on a thread queue. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_cancel( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Thread_queue_Context_extract( queue_context, the_thread ); + +#if defined(RTEMS_SMP) + if ( queue_context->Wait.queue != NULL ) { +#endif + _Thread_Wait_restore_default( the_thread ); +#if defined(RTEMS_SMP) + } +#endif +} /** * @brief The initial thread wait flags value set by _Thread_Initialize(). @@ -1388,58 +1533,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Wait_flags_try_change_acquire( } /** - * @brief Sets the thread queue. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * @param[in] new_queue The new queue. - * - * @see _Thread_Lock_set(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_set_queue( - Thread_Control *the_thread, - Thread_queue_Queue *new_queue -) -{ - the_thread->Wait.queue = new_queue; -} - -/** - * @brief Sets the thread queue operations. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * @param[in] new_operations The new queue operations. - * - * @see _Thread_Lock_set() and _Thread_Wait_restore_default_operations(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_set_operations( - Thread_Control *the_thread, - const Thread_queue_Operations *new_operations -) -{ - the_thread->Wait.operations = new_operations; -} - -/** - * @brief Restores the default thread queue operations. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * - * @see _Thread_Wait_set_operations(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default_operations( - Thread_Control *the_thread -) -{ - the_thread->Wait.operations = &_Thread_queue_Operations_default; -} - -/** * @brief Returns the object identifier of the object containing the current * thread wait queue. * @@ -1541,6 +1634,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Remove_timer_and_unblock( Thread_queue_Queue *queue ) { + _Thread_Wait_tranquilize( the_thread ); _Thread_Timer_remove( the_thread ); #if defined(RTEMS_MULTIPROCESSING) diff --git a/cpukit/score/include/rtems/score/threadq.h b/cpukit/score/include/rtems/score/threadq.h index 75aab20993..9a178049ea 100644 --- a/cpukit/score/include/rtems/score/threadq.h +++ b/cpukit/score/include/rtems/score/threadq.h @@ -43,6 +43,10 @@ extern "C" { typedef struct _Thread_Control Thread_Control; +typedef struct Thread_queue_Queue Thread_queue_Queue; + +typedef struct Thread_queue_Operations Thread_queue_Operations; + typedef struct Thread_queue_Path Thread_queue_Path; #if defined(RTEMS_MULTIPROCESSING) @@ -53,6 +57,8 @@ typedef struct Thread_queue_Path Thread_queue_Path; * control is actually a thread proxy if and only if * _Objects_Is_local_id( the_proxy->Object.id ) is false. * @param mp_id Object identifier of the object containing the thread queue. + * + * @see _Thread_queue_Context_set_MP_callout(). */ typedef void ( *Thread_queue_MP_callout )( Thread_Control *the_proxy, @@ -60,6 +66,24 @@ typedef void ( *Thread_queue_MP_callout )( ); #endif +#if defined(RTEMS_SMP) +/** + * @brief The thread queue gate is an SMP synchronization means. + * + * The gates are added to a list of requests. A busy wait is performed to make + * sure that preceding requests are carried out. Each predecessor notifies its + * successor about on request completion. + * + * @see _Thread_queue_Gate_add(), _Thread_queue_Gate_wait(), and + * _Thread_queue_Gate_open(). + */ +typedef struct { + Chain_Node Node; + + Atomic_Uint go_ahead; +} Thread_queue_Gate; +#endif + /** * @brief Thread queue context for the thread queue methods. * @@ -105,8 +129,64 @@ typedef struct { #if defined(RTEMS_MULTIPROCESSING) Thread_queue_MP_callout mp_callout; #endif + +#if defined(RTEMS_SMP) + /** + * @brief Data to support thread queue enqueue operations. + */ + struct { + /** + * @brief Gate to synchronize thread wait lock requests. + * + * @see _Thread_Wait_acquire_critical() and _Thread_Wait_tranquilize(). + */ + Thread_queue_Gate Gate; + + /** + * @brief The thread queue lock in case the thread is blocked on a thread + * queue at thread wait lock acquire time. + */ + SMP_ticket_lock_Control *queue_lock; + + /** + * @brief The thread queue after thread wait lock acquire. + * + * In case the thread queue is NULL and the thread queue lock is non-NULL + * in this context, then we have a stale thread queue. This happens in + * case the thread wait default is restored while we wait on the thread + * queue lock, e.g. during a mutex ownership transfer. + * + * @see _Thread_Wait_restore_default(). + */ + Thread_queue_Queue *queue; + + /** + * @brief The thread queue operations after thread wait lock acquire. + */ + const Thread_queue_Operations *operations; + } Wait; +#endif } Thread_queue_Context; +#if defined(RTEMS_SMP) +/** + * @brief A thread queue link from one thread to another specified by the + * thread queue owner and thread wait queue relationships. + */ +typedef struct { + /** + * @brief The owner of this thread queue link. + */ + Thread_Control *owner; + + /** + * @brief The queue context used to acquire the thread wait lock of the + * owner. + */ + Thread_queue_Context Queue_context; +} Thread_queue_Link; +#endif + /** * @brief Thread priority queue. */ @@ -191,7 +271,7 @@ typedef struct _Thread_queue_Heads { sizeof( Thread_queue_Heads ) #endif -typedef struct { +struct Thread_queue_Queue { /** * @brief Lock to protect this thread queue. * @@ -221,13 +301,15 @@ typedef struct { * @brief The thread queue owner. */ Thread_Control *owner; -} Thread_queue_Queue; +}; /** * @brief Thread queue priority change operation. * * @param[in] the_thread The thread. * @param[in] new_priority The new priority value. + * @param[in] prepend_it In case this is true, then the thread is prepended to + * its priority group in its scheduler instance, otherwise it is appended. * @param[in] queue The actual thread queue. * * @see Thread_queue_Operations. @@ -235,6 +317,7 @@ typedef struct { typedef void ( *Thread_queue_Priority_change_operation )( Thread_Control *the_thread, Priority_Control new_priority, + bool prepend_it, Thread_queue_Queue *queue ); @@ -247,8 +330,6 @@ typedef void ( *Thread_queue_Priority_change_operation )( * * @param[in] queue The actual thread queue. * @param[in] the_thread The thread to enqueue on the queue. - * - * @see _Thread_Wait_set_operations(). */ typedef void ( *Thread_queue_Enqueue_operation )( Thread_queue_Queue *queue, @@ -261,8 +342,6 @@ typedef void ( *Thread_queue_Enqueue_operation )( * * @param[in] queue The actual thread queue. * @param[in] the_thread The thread to extract from the thread queue. - * - * @see _Thread_Wait_set_operations(). */ typedef void ( *Thread_queue_Extract_operation )( Thread_queue_Queue *queue, @@ -277,8 +356,6 @@ typedef void ( *Thread_queue_Extract_operation )( * @retval NULL No thread is present on the thread queue. * @retval first The first thread of the thread queue according to the insert * order. This thread remains on the thread queue. - * - * @see _Thread_Wait_set_operations(). */ typedef Thread_Control *( *Thread_queue_First_operation )( Thread_queue_Heads *heads @@ -289,7 +366,7 @@ typedef Thread_Control *( *Thread_queue_First_operation )( * * @see _Thread_wait_Set_operations(). */ -typedef struct { +struct Thread_queue_Operations { /** * @brief Thread queue priority change operation. * @@ -320,7 +397,7 @@ typedef struct { * @brief Thread queue first operation. */ Thread_queue_First_operation first; -} Thread_queue_Operations; +}; /** * This is the structure used to manage sets of tasks which are blocked diff --git a/cpukit/score/include/rtems/score/threadqimpl.h b/cpukit/score/include/rtems/score/threadqimpl.h index 1f17f1a4f1..3d4f6136f6 100644 --- a/cpukit/score/include/rtems/score/threadqimpl.h +++ b/cpukit/score/include/rtems/score/threadqimpl.h @@ -26,6 +26,10 @@ #include <rtems/score/smp.h> #include <rtems/score/thread.h> +#if defined(RTEMS_DEBUG) +#include <string.h> +#endif + #ifdef __cplusplus extern "C" { #endif @@ -46,6 +50,13 @@ extern "C" { * relationships. */ struct Thread_queue_Path { +#if defined(RTEMS_SMP) + /** + * @brief The start of a thread queue path. + */ + Thread_queue_Link Start; +#endif + /** * @brief A potential thread to update the priority via * _Thread_Update_priority(). @@ -84,10 +95,8 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_initialize( ) { #if defined(RTEMS_DEBUG) + memset( queue_context, 0, sizeof( *queue_context ) ); queue_context->expected_thread_dispatch_disable_level = 0xdeadbeef; -#if defined(RTEMS_MULTIPROCESSING) - queue_context->mp_callout = NULL; -#endif #else (void) queue_context; #endif @@ -187,6 +196,135 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_set_MP_callout( } while ( 0 ) #endif +/** + * @brief Gets the thread wait queue of the thread queue context. + * + * On SMP configurations, the value is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \ + ( queue_context )->Wait.queue +#else +#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \ + ( the_thread )->Wait.queue +#endif + +/** + * @brief Gets the thread wait operations of the thread queue context. + * + * On SMP configurations, the value is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \ + ( queue_context )->Wait.operations +#else +#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \ + ( the_thread )->Wait.operations +#endif + +/** + * @brief Thread priority change by means of the thread queue context. + * + * On SMP configurations, the used data is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + * @param new_priority The new thread priority. + * @param prepend_it Prepend it to its priority group or not. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_priority_change( \ + queue_context, \ + the_thread, \ + new_priority, \ + prepend_it \ + ) \ + ( *( queue_context )->Wait.operations->priority_change )( \ + the_thread, \ + new_priority, \ + prepend_it, \ + ( queue_context )->Wait.queue \ + ) +#else +#define _Thread_queue_Context_priority_change( \ + queue_context, \ + the_thread, \ + new_priority, \ + prepend_it \ + ) \ + ( *( the_thread )->Wait.operations->priority_change )( \ + the_thread, \ + new_priority, \ + prepend_it, \ + ( the_thread )->Wait.queue \ + ) +#endif + +/** + * @brief Thread queue extract by means of the thread queue context. + * + * On SMP configurations, the used data is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_extract( \ + queue_context, \ + the_thread \ + ) \ + ( *( queue_context )->Wait.operations->extract )( \ + ( queue_context )->Wait.queue, \ + the_thread \ + ) +#else +#define _Thread_queue_Context_extract( \ + queue_context, \ + the_thread \ + ) \ + ( *( the_thread )->Wait.operations->extract )( \ + ( the_thread )->Wait.queue, \ + the_thread \ + ) +#endif + +#if defined(RTEMS_SMP) +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_add( + Chain_Control *chain, + Thread_queue_Gate *gate +) +{ + _Atomic_Store_uint( &gate->go_ahead, 0, ATOMIC_ORDER_RELAXED ); + _Chain_Append_unprotected( chain, &gate->Node ); +} + +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_open( + Thread_queue_Gate *gate +) +{ + _Atomic_Store_uint( &gate->go_ahead, 1, ATOMIC_ORDER_RELAXED ); +} + +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_wait( + Thread_queue_Gate *gate +) +{ + while ( _Atomic_Load_uint( &gate->go_ahead, ATOMIC_ORDER_RELAXED ) == 0 ) { + /* Wait */ + } +} +#endif + RTEMS_INLINE_ROUTINE void _Thread_queue_Heads_initialize( Thread_queue_Heads *heads ) @@ -911,6 +1049,10 @@ extern const Thread_queue_Operations _Thread_queue_Operations_priority; extern const Thread_queue_Operations _Thread_queue_Operations_priority_inherit; +#if defined(RTEMS_SMP) +extern const Thread_queue_Operations _Thread_queue_Operations_stale_queue; +#endif + /**@}*/ #ifdef __cplusplus |