From 1fcac5adc5ed38fb88ce4c6d24b2ca2e27e3cd10 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Mon, 25 Jul 2016 16:35:37 +0200 Subject: score: Turn thread lock into thread wait lock The _Thread_Lock_acquire() function had a potentially infinite run-time due to the lack of fairness at atomic operations level. Update #2412. Update #2556. Update #2765. --- cpukit/libmisc/monitor/mon-task.c | 7 +- cpukit/posix/src/pthreadgetschedparam.c | 8 +- cpukit/rtems/include/rtems/rtems/ratemonimpl.h | 4 +- cpukit/rtems/src/eventreceive.c | 4 +- cpukit/rtems/src/eventseize.c | 6 +- cpukit/rtems/src/eventsurrender.c | 6 +- cpukit/rtems/src/systemeventreceive.c | 4 +- cpukit/rtems/src/tasksetscheduler.c | 18 +- cpukit/score/include/rtems/score/schedulerimpl.h | 14 + cpukit/score/include/rtems/score/thread.h | 125 ++---- cpukit/score/include/rtems/score/threadimpl.h | 518 +++++++++++++---------- cpukit/score/include/rtems/score/threadq.h | 97 ++++- cpukit/score/include/rtems/score/threadqimpl.h | 148 ++++++- cpukit/score/src/threadchangepriority.c | 30 +- cpukit/score/src/threadinitialize.c | 10 +- cpukit/score/src/threadqenqueue.c | 91 +++- cpukit/score/src/threadqops.c | 63 ++- cpukit/score/src/threadrestart.c | 3 +- cpukit/score/src/threadtimeout.c | 30 +- testsuites/sptests/spthreadq01/init.c | 7 +- 20 files changed, 769 insertions(+), 424 deletions(-) diff --git a/cpukit/libmisc/monitor/mon-task.c b/cpukit/libmisc/monitor/mon-task.c index 96891e26de..eedba3e1d4 100644 --- a/cpukit/libmisc/monitor/mon-task.c +++ b/cpukit/libmisc/monitor/mon-task.c @@ -20,17 +20,16 @@ rtems_monitor_task_wait_info( Thread_Control *rtems_thread ) { - ISR_lock_Context lock_context; - void *lock; + Thread_queue_Context queue_context; - lock = _Thread_Lock_acquire( rtems_thread, &lock_context ); + _Thread_Wait_acquire( rtems_thread, &queue_context ); canonical_task->state = rtems_thread->current_state; canonical_task->wait_id = _Thread_Wait_get_id( rtems_thread ); canonical_task->wait_queue = rtems_thread->Wait.queue; canonical_task->wait_operations = rtems_thread->Wait.operations; - _Thread_Lock_release( lock, &lock_context ); + _Thread_Wait_release( rtems_thread, &queue_context ); } void diff --git a/cpukit/posix/src/pthreadgetschedparam.c b/cpukit/posix/src/pthreadgetschedparam.c index a0a4c6e15b..1b7c731587 100644 --- a/cpukit/posix/src/pthreadgetschedparam.c +++ b/cpukit/posix/src/pthreadgetschedparam.c @@ -36,7 +36,7 @@ int pthread_getschedparam( ) { Thread_Control *the_thread; - ISR_lock_Context lock_context; + Thread_queue_Context queue_context; POSIX_API_Control *api; const Scheduler_Control *scheduler; Priority_Control priority; @@ -45,7 +45,7 @@ int pthread_getschedparam( return EINVAL; } - the_thread = _Thread_Get( thread, &lock_context ); + the_thread = _Thread_Get( thread, &queue_context.Lock_context ); if ( the_thread == NULL ) { return ESRCH; @@ -53,7 +53,7 @@ int pthread_getschedparam( api = the_thread->API_Extensions[ THREAD_API_POSIX ]; - _Thread_Lock_acquire_default_critical( the_thread, &lock_context ); + _Thread_Wait_acquire_critical( the_thread, &queue_context ); *policy = api->Attributes.schedpolicy; *param = api->Attributes.schedparam; @@ -61,7 +61,7 @@ int pthread_getschedparam( scheduler = _Scheduler_Get_own( the_thread ); priority = the_thread->real_priority; - _Thread_Lock_release_default( the_thread, &lock_context ); + _Thread_Wait_release( the_thread, &queue_context ); param->sched_priority = _POSIX_Priority_From_core( scheduler, priority ); return 0; diff --git a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h index 61ebb5a0df..9963cab612 100644 --- a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h +++ b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h @@ -73,7 +73,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Acquire_critical( ISR_lock_Context *lock_context ) { - _Thread_Lock_acquire_default_critical( the_thread, lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, lock_context ); } RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release( @@ -81,7 +81,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release( ISR_lock_Context *lock_context ) { - _Thread_Lock_release_default( the_thread, lock_context ); + _Thread_Wait_release_default( the_thread, lock_context ); } RTEMS_INLINE_ROUTINE Rate_monotonic_Control *_Rate_monotonic_Get( diff --git a/cpukit/rtems/src/eventreceive.c b/cpukit/rtems/src/eventreceive.c index e03ff279eb..193960b7b2 100644 --- a/cpukit/rtems/src/eventreceive.c +++ b/cpukit/rtems/src/eventreceive.c @@ -38,7 +38,7 @@ rtems_status_code rtems_event_receive( RTEMS_API_Control *api; Event_Control *event; - executing = _Thread_Lock_acquire_default_for_executing( &lock_context ); + executing = _Thread_Wait_acquire_default_for_executing( &lock_context ); api = executing->API_Extensions[ THREAD_API_RTEMS ]; event = &api->Event; @@ -56,7 +56,7 @@ rtems_status_code rtems_event_receive( ); } else { *event_out = event->pending_events; - _Thread_Lock_release_default( executing, &lock_context ); + _Thread_Wait_release_default( executing, &lock_context ); sc = RTEMS_SUCCESSFUL; } } else { diff --git a/cpukit/rtems/src/eventseize.c b/cpukit/rtems/src/eventseize.c index 7f5903df89..696481c2f5 100644 --- a/cpukit/rtems/src/eventseize.c +++ b/cpukit/rtems/src/eventseize.c @@ -50,13 +50,13 @@ rtems_status_code _Event_Seize( (seized_events == event_in || _Options_Is_any( option_set )) ) { event->pending_events = _Event_sets_Clear( pending_events, seized_events ); - _Thread_Lock_release_default( executing, lock_context ); + _Thread_Wait_release_default( executing, lock_context ); *event_out = seized_events; return RTEMS_SUCCESSFUL; } if ( _Options_Is_no_wait( option_set ) ) { - _Thread_Lock_release_default( executing, lock_context ); + _Thread_Wait_release_default( executing, lock_context ); *event_out = seized_events; return RTEMS_UNSATISFIED; } @@ -78,7 +78,7 @@ rtems_status_code _Event_Seize( _Thread_Wait_flags_set( executing, intend_to_block ); cpu_self = _Thread_Dispatch_disable_critical( lock_context ); - _Thread_Lock_release_default( executing, lock_context ); + _Thread_Wait_release_default( executing, lock_context ); if ( ticks ) { _Thread_Timer_insert_relative( diff --git a/cpukit/rtems/src/eventsurrender.c b/cpukit/rtems/src/eventsurrender.c index a9bef5916c..c805b0e89d 100644 --- a/cpukit/rtems/src/eventsurrender.c +++ b/cpukit/rtems/src/eventsurrender.c @@ -77,7 +77,7 @@ rtems_status_code _Event_Surrender( rtems_event_set seized_events; bool unblock; - _Thread_Lock_acquire_default_critical( the_thread, lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, lock_context ); _Event_sets_Post( event_in, &event->pending_events ); pending_events = event->pending_events; @@ -116,14 +116,14 @@ rtems_status_code _Event_Surrender( Per_CPU_Control *cpu_self; cpu_self = _Thread_Dispatch_disable_critical( lock_context ); - _Thread_Lock_release_default( the_thread, lock_context ); + _Thread_Wait_release_default( the_thread, lock_context ); _Thread_Timer_remove( the_thread ); _Thread_Unblock( the_thread ); _Thread_Dispatch_enable( cpu_self ); } else { - _Thread_Lock_release_default( the_thread, lock_context ); + _Thread_Wait_release_default( the_thread, lock_context ); } return RTEMS_SUCCESSFUL; diff --git a/cpukit/rtems/src/systemeventreceive.c b/cpukit/rtems/src/systemeventreceive.c index a2215fa7ec..93a5694eec 100644 --- a/cpukit/rtems/src/systemeventreceive.c +++ b/cpukit/rtems/src/systemeventreceive.c @@ -44,7 +44,7 @@ rtems_status_code rtems_event_system_receive( RTEMS_API_Control *api; Event_Control *event; - executing = _Thread_Lock_acquire_default_for_executing( &lock_context ); + executing = _Thread_Wait_acquire_default_for_executing( &lock_context ); api = executing->API_Extensions[ THREAD_API_RTEMS ]; event = &api->System_event; @@ -62,7 +62,7 @@ rtems_status_code rtems_event_system_receive( ); } else { *event_out = event->pending_events; - _Thread_Lock_release_default( executing, &lock_context ); + _Thread_Wait_release_default( executing, &lock_context ); sc = RTEMS_SUCCESSFUL; } } else { diff --git a/cpukit/rtems/src/tasksetscheduler.c b/cpukit/rtems/src/tasksetscheduler.c index 175f235f02..0ff74d9c97 100644 --- a/cpukit/rtems/src/tasksetscheduler.c +++ b/cpukit/rtems/src/tasksetscheduler.c @@ -28,10 +28,9 @@ rtems_status_code rtems_task_set_scheduler( { const Scheduler_Control *scheduler; Thread_Control *the_thread; - ISR_lock_Context lock_context; - ISR_lock_Context state_lock_context; + Thread_queue_Context queue_context; + ISR_lock_Context state_context; Per_CPU_Control *cpu_self; - void *lock; bool valid; Priority_Control core_priority; Status_Control status; @@ -45,7 +44,7 @@ rtems_status_code rtems_task_set_scheduler( return RTEMS_INVALID_PRIORITY; } - the_thread = _Thread_Get( task_id, &lock_context ); + the_thread = _Thread_Get( task_id, &queue_context.Lock_context ); if ( the_thread == NULL ) { #if defined(RTEMS_MULTIPROCESSING) @@ -57,16 +56,15 @@ rtems_status_code rtems_task_set_scheduler( return RTEMS_INVALID_ID; } - cpu_self = _Thread_Dispatch_disable_critical( &lock_context ); - _ISR_lock_ISR_enable( &lock_context ); + cpu_self = _Thread_Dispatch_disable_critical( &queue_context.Lock_context ); - lock = _Thread_Lock_acquire( the_thread, &lock_context ); - _Thread_State_acquire_critical( the_thread, &state_lock_context ); + _Thread_Wait_acquire_critical( the_thread, &queue_context ); + _Thread_State_acquire_critical( the_thread, &state_context ); status = _Scheduler_Set( scheduler, the_thread, core_priority ); - _Thread_State_release_critical( the_thread, &state_lock_context ); - _Thread_Lock_release( lock, &lock_context ); + _Thread_State_release_critical( the_thread, &state_context ); + _Thread_Wait_release( the_thread, &queue_context ); _Thread_Dispatch_enable( cpu_self ); return _Status_Get( status ); } diff --git a/cpukit/score/include/rtems/score/schedulerimpl.h b/cpukit/score/include/rtems/score/schedulerimpl.h index db4be99f79..f8c29e215c 100644 --- a/cpukit/score/include/rtems/score/schedulerimpl.h +++ b/cpukit/score/include/rtems/score/schedulerimpl.h @@ -825,6 +825,20 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Node_set_priority( #endif } +RTEMS_INLINE_ROUTINE void _Scheduler_Thread_set_priority( + Thread_Control *the_thread, + Priority_Control new_priority, + bool prepend_it +) +{ + Scheduler_Node *own_node; + + own_node = _Scheduler_Thread_get_own_node( the_thread ); + _Scheduler_Node_set_priority( own_node, new_priority, prepend_it ); + + the_thread->current_priority = new_priority; +} + #if defined(RTEMS_SMP) /** * @brief Gets an idle thread from the scheduler instance. diff --git a/cpukit/score/include/rtems/score/thread.h b/cpukit/score/include/rtems/score/thread.h index 46c222ff52..a52c34f7fd 100644 --- a/cpukit/score/include/rtems/score/thread.h +++ b/cpukit/score/include/rtems/score/thread.h @@ -11,7 +11,7 @@ * COPYRIGHT (c) 1989-2014. * On-Line Applications Research Corporation (OAR). * - * Copyright (c) 2014 embedded brains GmbH. + * Copyright (c) 2014, 2016 embedded brains GmbH. * * The license and distribution terms for this file may be * found in the file LICENSE in this distribution or at @@ -287,16 +287,6 @@ typedef struct { */ uint32_t return_code; - /** - * @brief The current thread queue. - * - * In case this field is @c NULL, then the thread is not blocked on a thread - * queue. This field is protected by the thread lock. - * - * @see _Thread_Lock_set() and _Thread_Wait_set_queue(). - */ - Thread_queue_Queue *queue; - /** * @brief This field contains several flags used to control the wait class * and state of a thread in case fine-grained locking is used. @@ -307,12 +297,54 @@ typedef struct { Thread_Wait_flags flags; #endif +#if defined(RTEMS_SMP) + /** + * @brief Thread wait lock control block. + * + * Parts of the thread wait information are protected by the thread wait + * default lock and additionally a thread queue lock in case the thread + * is enqueued on a thread queue. + * + * The thread wait lock mechanism protects the following thread variables + * - POSIX_API_Control::Attributes, + * - Thread_Control::current_priority, + * - Thread_Control::Wait::Lock::Pending_requests, + * - Thread_Control::Wait::queue, and + * - Thread_Control::Wait::operations. + * + * @see _Thread_Wait_acquire(), _Thread_Wait_release(), _Thread_Wait_claim(), + * _Thread_Wait_restore_default() and _Thread_Wait_tranquilize(). + */ + struct { + /** + * @brief Thread wait default lock. + */ + ISR_lock_Control Default; + + /** + * @brief The pending thread wait lock acquire or tranquilize requests in + * case the thread is enqueued on a thread queue. + */ + Chain_Control Pending_requests; + } Lock; +#endif + + /** + * @brief The current thread queue. + * + * If this field is NULL the thread is not enqueued on a thread queue. This + * field is protected by the thread wait default lock. + * + * @see _Thread_Wait_claim(). + */ + Thread_queue_Queue *queue; + /** * @brief The current thread queue operations. * - * This field is protected by the thread lock. + * This field is protected by the thread lock wait default lock. * - * @see _Thread_Lock_set() and _Thread_Wait_set_operations(). + * @see _Thread_Wait_claim(). */ const Thread_queue_Operations *operations; @@ -639,66 +671,6 @@ typedef struct { void * control; }Thread_Capture_control; -#if defined(RTEMS_SMP) -/** - * @brief Thread lock control. - * - * The thread lock is either the default lock or the lock of the resource on - * which the thread is currently blocked. The generation number takes care - * that the up to date lock is used. Only resources using fine grained locking - * provide their own lock. - * - * The thread lock protects the following thread variables - * - POSIX_API_Control::Attributes, - * - Thread_Control::current_priority, - * - Thread_Control::Wait::queue, and - * - Thread_Control::Wait::operations. - * - * @see _Thread_Lock_acquire(), _Thread_Lock_release(), _Thread_Lock_set() and - * _Thread_Lock_restore_default(). - */ -typedef struct { - /** - * @brief The current thread lock. - * - * This is a plain ticket lock without SMP lock statistics support. This - * enables external libraries to use thread locks since they are independent - * of the actual RTEMS build configuration, e.g. profiling enabled or - * disabled. - */ - union { - /** - * @brief The current thread lock as an atomic unsigned integer pointer value. - */ - Atomic_Uintptr atomic; - - /** - * @brief The current thread lock as a normal pointer. - * - * Only provided for debugging purposes. - */ - SMP_ticket_lock_Control *normal; - } current; - - /** - * @brief The default thread lock in case the thread is not blocked on a - * resource. - */ - SMP_ticket_lock_Control Default; - -#if defined(RTEMS_PROFILING) - /** - * @brief The thread lock statistics. - * - * These statistics are used by the executing thread in case it acquires a - * thread lock. Thus the statistics are an aggregation of acquire and - * release operations of different locks. - */ - SMP_lock_Stats Stats; -#endif -} Thread_Lock_control; -#endif - /** * This structure defines the Thread Control Block (TCB). * @@ -770,13 +742,6 @@ struct _Thread_Control { #endif /*================= end of common block =================*/ -#if defined(RTEMS_SMP) - /** - * @brief Thread lock control. - */ - Thread_Lock_control Lock; -#endif - #if defined(RTEMS_SMP) && defined(RTEMS_PROFILING) /** * @brief Potpourri lock statistics. diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h index 92968a2e8e..ea1c61fa1e 100644 --- a/cpukit/score/include/rtems/score/threadimpl.h +++ b/cpukit/score/include/rtems/score/threadimpl.h @@ -11,7 +11,7 @@ * COPYRIGHT (c) 1989-2008. * On-Line Applications Research Corporation (OAR). * - * Copyright (c) 2014-2015 embedded brains GmbH. + * Copyright (c) 2014, 2016 embedded brains GmbH. * * The license and distribution terms for this file may be * found in the file LICENSE in this distribution or at @@ -933,44 +933,35 @@ RTEMS_INLINE_ROUTINE bool _Thread_Owns_resources( } /** - * @brief Acquires the default thread lock inside a critical section + * @brief Acquires the thread wait default lock inside a critical section * (interrupts disabled). * * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default_critical(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default_critical( +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default_critical( Thread_Control *the_thread, ISR_lock_Context *lock_context ) { - _Assert( _ISR_Get_level() != 0 ); -#if defined(RTEMS_SMP) - _SMP_ticket_lock_Acquire( - &the_thread->Lock.Default, - &_Thread_Executing->Lock.Stats, - &lock_context->Lock_context.Stats_context - ); -#else - (void) the_thread; - (void) lock_context; -#endif + _ISR_lock_Acquire( &the_thread->Wait.Lock.Default, lock_context ); } /** - * @brief Acquires the default thread lock and returns the executing thread. + * @brief Acquires the thread wait default lock and returns the executing + * thread. * * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * * @return The executing thread. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default(). */ -RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing( +RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Wait_acquire_default_for_executing( ISR_lock_Context *lock_context ) { @@ -978,247 +969,401 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing( _ISR_lock_ISR_disable( lock_context ); executing = _Thread_Executing; - _Thread_Lock_acquire_default_critical( executing, lock_context ); + _Thread_Wait_acquire_default_critical( executing, lock_context ); return executing; } /** - * @brief Acquires the default thread lock. + * @brief Acquires the thread wait default lock and disables interrupts. * * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * release. + * release. * - * @see _Thread_Lock_release_default(). + * @see _Thread_Wait_release_default(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default( +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default( Thread_Control *the_thread, ISR_lock_Context *lock_context ) { _ISR_lock_ISR_disable( lock_context ); - _Thread_Lock_acquire_default_critical( the_thread, lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, lock_context ); } /** - * @brief Releases the thread lock inside a critical section (interrupts - * disabled). + * @brief Releases the thread wait default lock inside a critical section + * (interrupts disabled). * * The previous interrupt status is not restored. * - * @param[in] lock The lock. + * @param[in] the_thread The thread. * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * acquire. */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_critical( - void *lock, +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default_critical( + Thread_Control *the_thread, ISR_lock_Context *lock_context ) { -#if defined(RTEMS_SMP) - _SMP_ticket_lock_Release( - (SMP_ticket_lock_Control *) lock, - &lock_context->Lock_context.Stats_context - ); -#else - (void) lock; - (void) lock_context; -#endif + _ISR_lock_Release( &the_thread->Wait.Lock.Default, lock_context ); } /** - * @brief Releases the thread lock. + * @brief Releases the thread wait default lock and restores the previous + * interrupt status. * - * @param[in] lock The lock returned by _Thread_Lock_acquire(). - * @param[in] lock_context The lock context used for _Thread_Lock_acquire(). + * @param[in] the_thread The thread. + * @param[in] lock_context The lock context used for the corresponding lock + * acquire. */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release( - void *lock, +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default( + Thread_Control *the_thread, ISR_lock_Context *lock_context ) { - _Thread_Lock_release_critical( lock, lock_context ); + _Thread_Wait_release_default_critical( the_thread, lock_context ); _ISR_lock_ISR_enable( lock_context ); } +#if defined(RTEMS_SMP) +#define THREAD_QUEUE_CONTEXT_OF_REQUEST( node ) \ + RTEMS_CONTAINER_OF( node, Thread_queue_Context, Wait.Gate.Node ) + +RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request_locked( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Chain_Extract_unprotected( &queue_context->Wait.Gate.Node ); + + if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) { + Thread_queue_Context *first; + + first = THREAD_QUEUE_CONTEXT_OF_REQUEST( + _Chain_First( &the_thread->Wait.Lock.Pending_requests ) + ); + + _Thread_queue_Gate_open( &first->Wait.Gate ); + } +} + +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_queue_critical( + SMP_ticket_lock_Control *queue_lock, + Thread_queue_Context *queue_context +) +{ + _SMP_ticket_lock_Acquire( + queue_lock, + &_Thread_Executing->Potpourri_stats, + &queue_context->Lock_context.Lock_context.Stats_context + ); +} + +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_queue_critical( + SMP_ticket_lock_Control *queue_lock, + Thread_queue_Context *queue_context +) +{ + _SMP_ticket_lock_Release( + queue_lock, + &queue_context->Lock_context.Lock_context.Stats_context + ); +} +#endif + /** - * @brief Releases the default thread lock inside a critical section - * (interrupts disabled). - * - * The previous interrupt status is not restored. + * @brief Acquires the thread wait lock inside a critical section (interrupts + * disabled). * * @param[in] the_thread The thread. - * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * @param[in] queue_context The thread queue context for the corresponding + * _Thread_Wait_release_critical(). + * + * @see _Thread_queue_Context_initialize(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default_critical( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_critical( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Thread_Lock_release_critical( #if defined(RTEMS_SMP) - &the_thread->Lock.Default, + Thread_queue_Queue *queue; + + _Thread_Wait_acquire_default_critical( + the_thread, + &queue_context->Lock_context + ); + + queue = the_thread->Wait.queue; + queue_context->Wait.queue = queue; + queue_context->Wait.operations = the_thread->Wait.operations; + + if ( queue != NULL ) { + queue_context->Wait.queue_lock = &queue->Lock; + _Chain_Initialize_node( &queue_context->Wait.Gate.Node ); + _Chain_Append_unprotected( + &the_thread->Wait.Lock.Pending_requests, + &queue_context->Wait.Gate.Node + ); + _Thread_Wait_release_default_critical( + the_thread, + &queue_context->Lock_context + ); + _Thread_Wait_acquire_queue_critical( &queue->Lock, queue_context ); + } else { + queue_context->Wait.queue_lock = NULL; + } #else - NULL, + (void) the_thread; + (void) queue_context; #endif - lock_context - ); } /** - * @brief Releases the default thread lock. + * @brief Acquires the thread wait default lock and disables interrupts. * * @param[in] the_thread The thread. - * @param[in] lock_context The lock context used for the corresponding lock - * acquire. + * @param[in] queue_context The thread queue context for the corresponding + * _Thread_Wait_release(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Thread_Lock_release_default_critical( the_thread, lock_context ); - _ISR_lock_ISR_enable( lock_context ); + _Thread_queue_Context_initialize( queue_context ); + _ISR_lock_ISR_disable( &queue_context->Lock_context ); + _Thread_Wait_acquire_critical( the_thread, queue_context ); } /** - * @brief Acquires the thread lock. + * @brief Releases the thread wait lock inside a critical section (interrupts + * disabled). * - * @param[in] the_thread The thread. - * @param[in] lock_context The lock context for _Thread_Lock_release(). + * The previous interrupt status is not restored. * - * @return The lock required by _Thread_Lock_release(). + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire_critical(). */ -RTEMS_INLINE_ROUTINE void *_Thread_Lock_acquire( - Thread_Control *the_thread, - ISR_lock_Context *lock_context +RTEMS_INLINE_ROUTINE void _Thread_Wait_release_critical( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { #if defined(RTEMS_SMP) - SMP_ticket_lock_Control *lock_0; - - while ( true ) { - SMP_ticket_lock_Control *lock_1; - - _ISR_lock_ISR_disable( lock_context ); - - /* - * We assume that a normal load of pointer is identical to a relaxed atomic - * load. Here, we may read an out-of-date lock. However, only the owner - * of this out-of-date lock is allowed to set a new one. Thus, we read at - * least this new lock ... - */ - lock_0 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr( - &the_thread->Lock.current.atomic, - ATOMIC_ORDER_RELAXED - ); + SMP_ticket_lock_Control *queue_lock; - _SMP_ticket_lock_Acquire( - lock_0, - &_Thread_Executing->Lock.Stats, - &lock_context->Lock_context.Stats_context - ); + queue_lock = queue_context->Wait.queue_lock; - /* - * We must use a load acquire here paired with the store release in - * _Thread_Lock_set_unprotected() to observe corresponding thread wait - * queue and thread wait operations. It is important to do this after the - * lock acquire, since we may have the following scenario. - * - * - We read the default lock and try to acquire it. - * - The thread lock changes to a thread queue lock. - * - The thread lock is restored to the default lock. - * - We acquire the default lock and read it here again. - * - Now, we must read the restored default thread wait queue and thread - * wait operations and this is not synchronized via the default thread - * lock. - */ - lock_1 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr( - &the_thread->Lock.current.atomic, - ATOMIC_ORDER_ACQUIRE + if ( queue_lock != NULL ) { + _Thread_Wait_release_queue_critical( queue_lock, queue_context ); + _Thread_Wait_acquire_default_critical( + the_thread, + &queue_context->Lock_context ); - /* - * ... here, and so on. - */ - if ( lock_0 == lock_1 ) { - return lock_0; - } - - _Thread_Lock_release( lock_0, lock_context ); + _Thread_Wait_remove_request_locked( the_thread, queue_context ); } -#else - _ISR_Local_disable( lock_context->isr_level ); - return NULL; + _Thread_Wait_release_default_critical( + the_thread, + &queue_context->Lock_context + ); +#else + (void) the_thread; + (void) queue_context; #endif } -#if defined(RTEMS_SMP) -/* - * Internal function, use _Thread_Lock_set() or _Thread_Lock_restore_default() - * instead. +/** + * @brief Releases the thread wait lock and restores the previous interrupt + * status. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). */ -RTEMS_INLINE_ROUTINE void _Thread_Lock_set_unprotected( - Thread_Control *the_thread, - SMP_ticket_lock_Control *new_lock +RTEMS_INLINE_ROUTINE void _Thread_Wait_release( + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - _Atomic_Store_uintptr( - &the_thread->Lock.current.atomic, - (uintptr_t) new_lock, - ATOMIC_ORDER_RELEASE - ); + _Thread_Wait_release_critical( the_thread, queue_context ); + _ISR_lock_ISR_enable( &queue_context->Lock_context ); } -#endif /** - * @brief Sets a new thread lock. + * @brief Claims the thread wait queue and operations. * - * The caller must not be the owner of the default thread lock. The caller - * must be the owner of the new lock. + * The caller must not be the owner of the default thread wait lock. The + * caller must be the owner of the corresponding thread queue lock. * * @param[in] the_thread The thread. - * @param[in] new_lock The new thread lock. + * @param[in] queue The new thread queue. + * @param[in] operations The new thread operations. + * + * @see _Thread_Wait_restore_default(). */ -#if defined(RTEMS_SMP) -RTEMS_INLINE_ROUTINE void _Thread_Lock_set( - Thread_Control *the_thread, - SMP_ticket_lock_Control *new_lock +RTEMS_INLINE_ROUTINE void _Thread_Wait_claim( + Thread_Control *the_thread, + Thread_queue_Queue *queue, + const Thread_queue_Operations *operations ) { ISR_lock_Context lock_context; - _Thread_Lock_acquire_default_critical( the_thread, &lock_context ); - _Assert( the_thread->Lock.current.normal == &the_thread->Lock.Default ); - _Thread_Lock_set_unprotected( the_thread, new_lock ); - _Thread_Lock_release_default_critical( the_thread, &lock_context ); + _Thread_Wait_acquire_default_critical( the_thread, &lock_context ); + + _Assert( the_thread->Wait.queue == NULL ); + the_thread->Wait.queue = queue; + the_thread->Wait.operations = operations; + + _Thread_Wait_release_default_critical( the_thread, &lock_context ); } + +/** + * @brief Removes a thread wait lock request. + * + * On SMP configurations, removes a thread wait lock request. + * + * On other configurations, this function does nothing. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ +#if defined(RTEMS_SMP) + ISR_lock_Context lock_context; + + _Thread_Wait_acquire_default( the_thread, &lock_context ); + _Thread_Wait_remove_request_locked( the_thread, queue_context ); + _Thread_Wait_release_default( the_thread, &lock_context ); #else -#define _Thread_Lock_set( the_thread, new_lock ) \ - do { } while ( 0 ) + (void) the_thread; + (void) queue_context; #endif +} /** - * @brief Restores the default thread lock. + * @brief Restores the default thread wait queue and operations. + * + * The caller must be the owner of the current thread wait queue lock. * - * The caller must be the owner of the current thread lock. + * On SMP configurations, the pending requests are updated to use the stale + * thread queue operations. * * @param[in] the_thread The thread. + * + * @see _Thread_Wait_claim(). */ -#if defined(RTEMS_SMP) -RTEMS_INLINE_ROUTINE void _Thread_Lock_restore_default( +RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default( Thread_Control *the_thread ) { - _Thread_Lock_set_unprotected( the_thread, &the_thread->Lock.Default ); +#if defined(RTEMS_SMP) + ISR_lock_Context lock_context; + Chain_Node *node; + const Chain_Node *tail; + + _Thread_Wait_acquire_default_critical( + the_thread, + &lock_context + ); + + node = _Chain_First( &the_thread->Wait.Lock.Pending_requests ); + tail = _Chain_Immutable_tail( &the_thread->Wait.Lock.Pending_requests ); + + while ( node != tail ) { + Thread_queue_Context *queue_context; + + queue_context = THREAD_QUEUE_CONTEXT_OF_REQUEST( node ); + queue_context->Wait.queue = NULL; + queue_context->Wait.operations = &_Thread_queue_Operations_stale_queue; + + node = _Chain_Next( node ); + } +#endif + + the_thread->Wait.queue = NULL; + the_thread->Wait.operations = &_Thread_queue_Operations_default; + +#if defined(RTEMS_SMP) + _Thread_Wait_release_default_critical( + the_thread, + &lock_context + ); +#endif } + +/** + * @brief Tranquilizes the thread after a wait a thread queue. + * + * After the violent blocking procedure this function makes the thread calm and + * peaceful again so that it can carry out its normal work. + * + * On SMP configurations, ensures that all pending thread wait lock requests + * completed before the thread is able to begin a new thread wait procedure. + * + * On other configurations, this function does nothing. + * + * @param[in] the_thread The thread. + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_tranquilize( + Thread_Control *the_thread +) +{ +#if defined(RTEMS_SMP) + Thread_queue_Context queue_context; + + _Thread_queue_Context_initialize( &queue_context ); + _Thread_Wait_acquire_default( the_thread, &queue_context.Lock_context ); + + if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) { + _Thread_queue_Gate_add( + &the_thread->Wait.Lock.Pending_requests, + &queue_context.Wait.Gate + ); + _Thread_Wait_release_default( the_thread, &queue_context.Lock_context ); + _Thread_queue_Gate_wait( &queue_context.Wait.Gate ); + _Thread_Wait_remove_request( the_thread, &queue_context ); + } else { + _Thread_Wait_release_default( the_thread, &queue_context.Lock_context ); + } #else -#define _Thread_Lock_restore_default( the_thread ) \ - do { } while ( 0 ) + (void) the_thread; #endif +} + +/** + * @brief Cancels a thread wait on a thread queue. + * + * @param[in] the_thread The thread. + * @param[in] queue_context The thread queue context used for corresponding + * _Thread_Wait_acquire(). + */ +RTEMS_INLINE_ROUTINE void _Thread_Wait_cancel( + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Thread_queue_Context_extract( queue_context, the_thread ); + +#if defined(RTEMS_SMP) + if ( queue_context->Wait.queue != NULL ) { +#endif + _Thread_Wait_restore_default( the_thread ); +#if defined(RTEMS_SMP) + } +#endif +} /** * @brief The initial thread wait flags value set by _Thread_Initialize(). @@ -1387,58 +1532,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Wait_flags_try_change_acquire( return success; } -/** - * @brief Sets the thread queue. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * @param[in] new_queue The new queue. - * - * @see _Thread_Lock_set(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_set_queue( - Thread_Control *the_thread, - Thread_queue_Queue *new_queue -) -{ - the_thread->Wait.queue = new_queue; -} - -/** - * @brief Sets the thread queue operations. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * @param[in] new_operations The new queue operations. - * - * @see _Thread_Lock_set() and _Thread_Wait_restore_default_operations(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_set_operations( - Thread_Control *the_thread, - const Thread_queue_Operations *new_operations -) -{ - the_thread->Wait.operations = new_operations; -} - -/** - * @brief Restores the default thread queue operations. - * - * The caller must be the owner of the thread lock. - * - * @param[in] the_thread The thread. - * - * @see _Thread_Wait_set_operations(). - */ -RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default_operations( - Thread_Control *the_thread -) -{ - the_thread->Wait.operations = &_Thread_queue_Operations_default; -} - /** * @brief Returns the object identifier of the object containing the current * thread wait queue. @@ -1541,6 +1634,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Remove_timer_and_unblock( Thread_queue_Queue *queue ) { + _Thread_Wait_tranquilize( the_thread ); _Thread_Timer_remove( the_thread ); #if defined(RTEMS_MULTIPROCESSING) diff --git a/cpukit/score/include/rtems/score/threadq.h b/cpukit/score/include/rtems/score/threadq.h index 75aab20993..9a178049ea 100644 --- a/cpukit/score/include/rtems/score/threadq.h +++ b/cpukit/score/include/rtems/score/threadq.h @@ -43,6 +43,10 @@ extern "C" { typedef struct _Thread_Control Thread_Control; +typedef struct Thread_queue_Queue Thread_queue_Queue; + +typedef struct Thread_queue_Operations Thread_queue_Operations; + typedef struct Thread_queue_Path Thread_queue_Path; #if defined(RTEMS_MULTIPROCESSING) @@ -53,6 +57,8 @@ typedef struct Thread_queue_Path Thread_queue_Path; * control is actually a thread proxy if and only if * _Objects_Is_local_id( the_proxy->Object.id ) is false. * @param mp_id Object identifier of the object containing the thread queue. + * + * @see _Thread_queue_Context_set_MP_callout(). */ typedef void ( *Thread_queue_MP_callout )( Thread_Control *the_proxy, @@ -60,6 +66,24 @@ typedef void ( *Thread_queue_MP_callout )( ); #endif +#if defined(RTEMS_SMP) +/** + * @brief The thread queue gate is an SMP synchronization means. + * + * The gates are added to a list of requests. A busy wait is performed to make + * sure that preceding requests are carried out. Each predecessor notifies its + * successor about on request completion. + * + * @see _Thread_queue_Gate_add(), _Thread_queue_Gate_wait(), and + * _Thread_queue_Gate_open(). + */ +typedef struct { + Chain_Node Node; + + Atomic_Uint go_ahead; +} Thread_queue_Gate; +#endif + /** * @brief Thread queue context for the thread queue methods. * @@ -105,8 +129,64 @@ typedef struct { #if defined(RTEMS_MULTIPROCESSING) Thread_queue_MP_callout mp_callout; #endif + +#if defined(RTEMS_SMP) + /** + * @brief Data to support thread queue enqueue operations. + */ + struct { + /** + * @brief Gate to synchronize thread wait lock requests. + * + * @see _Thread_Wait_acquire_critical() and _Thread_Wait_tranquilize(). + */ + Thread_queue_Gate Gate; + + /** + * @brief The thread queue lock in case the thread is blocked on a thread + * queue at thread wait lock acquire time. + */ + SMP_ticket_lock_Control *queue_lock; + + /** + * @brief The thread queue after thread wait lock acquire. + * + * In case the thread queue is NULL and the thread queue lock is non-NULL + * in this context, then we have a stale thread queue. This happens in + * case the thread wait default is restored while we wait on the thread + * queue lock, e.g. during a mutex ownership transfer. + * + * @see _Thread_Wait_restore_default(). + */ + Thread_queue_Queue *queue; + + /** + * @brief The thread queue operations after thread wait lock acquire. + */ + const Thread_queue_Operations *operations; + } Wait; +#endif } Thread_queue_Context; +#if defined(RTEMS_SMP) +/** + * @brief A thread queue link from one thread to another specified by the + * thread queue owner and thread wait queue relationships. + */ +typedef struct { + /** + * @brief The owner of this thread queue link. + */ + Thread_Control *owner; + + /** + * @brief The queue context used to acquire the thread wait lock of the + * owner. + */ + Thread_queue_Context Queue_context; +} Thread_queue_Link; +#endif + /** * @brief Thread priority queue. */ @@ -191,7 +271,7 @@ typedef struct _Thread_queue_Heads { sizeof( Thread_queue_Heads ) #endif -typedef struct { +struct Thread_queue_Queue { /** * @brief Lock to protect this thread queue. * @@ -221,13 +301,15 @@ typedef struct { * @brief The thread queue owner. */ Thread_Control *owner; -} Thread_queue_Queue; +}; /** * @brief Thread queue priority change operation. * * @param[in] the_thread The thread. * @param[in] new_priority The new priority value. + * @param[in] prepend_it In case this is true, then the thread is prepended to + * its priority group in its scheduler instance, otherwise it is appended. * @param[in] queue The actual thread queue. * * @see Thread_queue_Operations. @@ -235,6 +317,7 @@ typedef struct { typedef void ( *Thread_queue_Priority_change_operation )( Thread_Control *the_thread, Priority_Control new_priority, + bool prepend_it, Thread_queue_Queue *queue ); @@ -247,8 +330,6 @@ typedef void ( *Thread_queue_Priority_change_operation )( * * @param[in] queue The actual thread queue. * @param[in] the_thread The thread to enqueue on the queue. - * - * @see _Thread_Wait_set_operations(). */ typedef void ( *Thread_queue_Enqueue_operation )( Thread_queue_Queue *queue, @@ -261,8 +342,6 @@ typedef void ( *Thread_queue_Enqueue_operation )( * * @param[in] queue The actual thread queue. * @param[in] the_thread The thread to extract from the thread queue. - * - * @see _Thread_Wait_set_operations(). */ typedef void ( *Thread_queue_Extract_operation )( Thread_queue_Queue *queue, @@ -277,8 +356,6 @@ typedef void ( *Thread_queue_Extract_operation )( * @retval NULL No thread is present on the thread queue. * @retval first The first thread of the thread queue according to the insert * order. This thread remains on the thread queue. - * - * @see _Thread_Wait_set_operations(). */ typedef Thread_Control *( *Thread_queue_First_operation )( Thread_queue_Heads *heads @@ -289,7 +366,7 @@ typedef Thread_Control *( *Thread_queue_First_operation )( * * @see _Thread_wait_Set_operations(). */ -typedef struct { +struct Thread_queue_Operations { /** * @brief Thread queue priority change operation. * @@ -320,7 +397,7 @@ typedef struct { * @brief Thread queue first operation. */ Thread_queue_First_operation first; -} Thread_queue_Operations; +}; /** * This is the structure used to manage sets of tasks which are blocked diff --git a/cpukit/score/include/rtems/score/threadqimpl.h b/cpukit/score/include/rtems/score/threadqimpl.h index 1f17f1a4f1..3d4f6136f6 100644 --- a/cpukit/score/include/rtems/score/threadqimpl.h +++ b/cpukit/score/include/rtems/score/threadqimpl.h @@ -26,6 +26,10 @@ #include #include +#if defined(RTEMS_DEBUG) +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -46,6 +50,13 @@ extern "C" { * relationships. */ struct Thread_queue_Path { +#if defined(RTEMS_SMP) + /** + * @brief The start of a thread queue path. + */ + Thread_queue_Link Start; +#endif + /** * @brief A potential thread to update the priority via * _Thread_Update_priority(). @@ -84,10 +95,8 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_initialize( ) { #if defined(RTEMS_DEBUG) + memset( queue_context, 0, sizeof( *queue_context ) ); queue_context->expected_thread_dispatch_disable_level = 0xdeadbeef; -#if defined(RTEMS_MULTIPROCESSING) - queue_context->mp_callout = NULL; -#endif #else (void) queue_context; #endif @@ -187,6 +196,135 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_set_MP_callout( } while ( 0 ) #endif +/** + * @brief Gets the thread wait queue of the thread queue context. + * + * On SMP configurations, the value is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \ + ( queue_context )->Wait.queue +#else +#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \ + ( the_thread )->Wait.queue +#endif + +/** + * @brief Gets the thread wait operations of the thread queue context. + * + * On SMP configurations, the value is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \ + ( queue_context )->Wait.operations +#else +#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \ + ( the_thread )->Wait.operations +#endif + +/** + * @brief Thread priority change by means of the thread queue context. + * + * On SMP configurations, the used data is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + * @param new_priority The new thread priority. + * @param prepend_it Prepend it to its priority group or not. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_priority_change( \ + queue_context, \ + the_thread, \ + new_priority, \ + prepend_it \ + ) \ + ( *( queue_context )->Wait.operations->priority_change )( \ + the_thread, \ + new_priority, \ + prepend_it, \ + ( queue_context )->Wait.queue \ + ) +#else +#define _Thread_queue_Context_priority_change( \ + queue_context, \ + the_thread, \ + new_priority, \ + prepend_it \ + ) \ + ( *( the_thread )->Wait.operations->priority_change )( \ + the_thread, \ + new_priority, \ + prepend_it, \ + ( the_thread )->Wait.queue \ + ) +#endif + +/** + * @brief Thread queue extract by means of the thread queue context. + * + * On SMP configurations, the used data is stored in the thread queue context, + * otherwise in the thread itself. + * + * @param queue_context The thread queue context. + * @param the_thread The thread. + */ +#if defined(RTEMS_SMP) +#define _Thread_queue_Context_extract( \ + queue_context, \ + the_thread \ + ) \ + ( *( queue_context )->Wait.operations->extract )( \ + ( queue_context )->Wait.queue, \ + the_thread \ + ) +#else +#define _Thread_queue_Context_extract( \ + queue_context, \ + the_thread \ + ) \ + ( *( the_thread )->Wait.operations->extract )( \ + ( the_thread )->Wait.queue, \ + the_thread \ + ) +#endif + +#if defined(RTEMS_SMP) +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_add( + Chain_Control *chain, + Thread_queue_Gate *gate +) +{ + _Atomic_Store_uint( &gate->go_ahead, 0, ATOMIC_ORDER_RELAXED ); + _Chain_Append_unprotected( chain, &gate->Node ); +} + +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_open( + Thread_queue_Gate *gate +) +{ + _Atomic_Store_uint( &gate->go_ahead, 1, ATOMIC_ORDER_RELAXED ); +} + +RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_wait( + Thread_queue_Gate *gate +) +{ + while ( _Atomic_Load_uint( &gate->go_ahead, ATOMIC_ORDER_RELAXED ) == 0 ) { + /* Wait */ + } +} +#endif + RTEMS_INLINE_ROUTINE void _Thread_queue_Heads_initialize( Thread_queue_Heads *heads ) @@ -911,6 +1049,10 @@ extern const Thread_queue_Operations _Thread_queue_Operations_priority; extern const Thread_queue_Operations _Thread_queue_Operations_priority_inherit; +#if defined(RTEMS_SMP) +extern const Thread_queue_Operations _Thread_queue_Operations_stale_queue; +#endif + /**@}*/ #ifdef __cplusplus diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c index e3cf4a165a..c569530d74 100644 --- a/cpukit/score/src/threadchangepriority.c +++ b/cpukit/score/src/threadchangepriority.c @@ -27,7 +27,8 @@ static Thread_Control *_Thread_Apply_priority_locked( Priority_Control new_priority, void *arg, Thread_Change_priority_filter filter, - bool prepend_it + bool prepend_it, + Thread_queue_Context *queue_context ) { /* @@ -45,17 +46,11 @@ static Thread_Control *_Thread_Apply_priority_locked( * we are not REALLY changing priority. */ if ( ( *filter )( the_thread, &new_priority, arg ) ) { - Scheduler_Node *own_node; - - own_node = _Scheduler_Thread_get_own_node( the_thread ); - _Scheduler_Node_set_priority( own_node, new_priority, prepend_it ); - - the_thread->current_priority = new_priority; - - ( *the_thread->Wait.operations->priority_change )( + _Thread_queue_Context_priority_change( + queue_context, the_thread, new_priority, - the_thread->Wait.queue + prepend_it ); } else { the_thread = NULL; @@ -72,19 +67,20 @@ Thread_Control *_Thread_Apply_priority( bool prepend_it ) { - ISR_lock_Context lock_context; - ISR_lock_Control *lock; + Thread_queue_Context queue_context; + Thread_Control *the_thread_to_update; - lock = _Thread_Lock_acquire( the_thread, &lock_context ); - the_thread = _Thread_Apply_priority_locked( + _Thread_Wait_acquire( the_thread, &queue_context ); + the_thread_to_update = _Thread_Apply_priority_locked( the_thread, new_priority, arg, filter, - prepend_it + prepend_it, + &queue_context ); - _Thread_Lock_release( lock, &lock_context ); - return the_thread; + _Thread_Wait_release( the_thread, &queue_context ); + return the_thread_to_update; } void _Thread_Update_priority( Thread_Control *the_thread ) diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c index 940537f135..209be9ebb1 100644 --- a/cpukit/score/src/threadinitialize.c +++ b/cpukit/score/src/threadinitialize.c @@ -182,13 +182,11 @@ bool _Thread_Initialize( the_thread->Scheduler.control = scheduler; the_thread->Scheduler.own_node = scheduler_node; _Resource_Node_initialize( &the_thread->Resource_node ); - _Atomic_Store_uintptr( - &the_thread->Lock.current.atomic, - (uintptr_t) &the_thread->Lock.Default, - ATOMIC_ORDER_RELAXED + _ISR_lock_Initialize( + &the_thread->Wait.Lock.Default, + "Thread Wait Default Lock" ); - _SMP_ticket_lock_Initialize( &the_thread->Lock.Default ); - _SMP_lock_Stats_initialize( &the_thread->Lock.Stats, "Thread Lock" ); + _Chain_Initialize_empty( &the_thread->Wait.Lock.Pending_requests ); _SMP_lock_Stats_initialize( &the_thread->Potpourri_stats, "Thread Potpourri" ); #endif diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c index 818073c2b1..19c345b6c5 100644 --- a/cpukit/score/src/threadqenqueue.c +++ b/cpukit/score/src/threadqenqueue.c @@ -34,6 +34,51 @@ #define THREAD_QUEUE_READY_AGAIN \ (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN) +static void _Thread_queue_Path_release( Thread_queue_Path *path ) +{ +#if defined(RTEMS_SMP) + Thread_queue_Link *link; + + link = &path->Start; + + if ( link->owner != NULL ) { + _Thread_Wait_release_critical( link->owner, &link->Queue_context ); + } +#else + (void) path; +#endif +} + +static void _Thread_queue_Path_acquire( + Thread_Control *the_thread, + Thread_queue_Queue *queue, + Thread_queue_Path *path +) +{ +#if defined(RTEMS_SMP) + Thread_Control *owner; + Thread_queue_Link *link; + + owner = queue->owner; + + if ( owner == NULL ) { + return; + } + + link = &path->Start; + link->owner = owner; + + _Thread_Wait_acquire_default_critical( + owner, + &link->Queue_context.Lock_context + ); +#else + (void) the_thread; + (void) queue; + (void) path; +#endif +} + void _Thread_queue_Enqueue_critical( Thread_queue_Queue *queue, const Thread_queue_Operations *operations, @@ -52,14 +97,13 @@ void _Thread_queue_Enqueue_critical( } #endif - _Thread_Lock_set( the_thread, &queue->Lock ); - - the_thread->Wait.return_code = STATUS_SUCCESSFUL; - _Thread_Wait_set_queue( the_thread, queue ); - _Thread_Wait_set_operations( the_thread, operations ); + _Thread_Wait_claim( the_thread, queue, operations ); + _Thread_queue_Path_acquire( the_thread, queue, &path ); ( *operations->enqueue )( queue, the_thread, &path ); + _Thread_queue_Path_release( &path ); + the_thread->Wait.return_code = STATUS_SUCCESSFUL; _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK ); cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context ); _Thread_queue_Queue_release( queue, &queue_context->Lock_context ); @@ -173,9 +217,7 @@ bool _Thread_queue_Do_extract_locked( unblock = true; } - _Thread_Wait_set_queue( the_thread, NULL ); - _Thread_Wait_restore_default_operations( the_thread ); - _Thread_Lock_restore_default( the_thread ); + _Thread_Wait_restore_default( the_thread ); return unblock; } @@ -227,30 +269,35 @@ void _Thread_queue_Extract_critical( void _Thread_queue_Extract( Thread_Control *the_thread ) { - Thread_queue_Context queue_context; - void *lock; - Thread_queue_Queue *queue; + Thread_queue_Context queue_context; _Thread_queue_Context_initialize( &queue_context ); - lock = _Thread_Lock_acquire( the_thread, &queue_context.Lock_context ); + _Thread_Wait_acquire( the_thread, &queue_context ); - queue = the_thread->Wait.queue; - - if ( queue != NULL ) { - _SMP_Assert( lock == &queue->Lock ); + if ( + _Thread_queue_Context_get_queue( &queue_context, the_thread ) != NULL + ) { + bool unblock; + _Thread_Wait_remove_request( the_thread, &queue_context ); _Thread_queue_Context_set_MP_callout( &queue_context, _Thread_queue_MP_callout_do_nothing ); - _Thread_queue_Extract_critical( - queue, - the_thread->Wait.operations, + unblock = _Thread_queue_Extract_locked( + _Thread_queue_Context_get_queue( &queue_context, the_thread ), + _Thread_queue_Context_get_operations( &queue_context, the_thread ), the_thread, - &queue_context + &queue_context.Lock_context + ); + _Thread_queue_Unblock_critical( + unblock, + _Thread_queue_Context_get_queue( &queue_context, the_thread ), + the_thread, + &queue_context.Lock_context ); } else { - _Thread_Lock_release( lock, &queue_context.Lock_context ); + _Thread_Wait_release( the_thread, &queue_context ); } } @@ -273,8 +320,6 @@ Thread_Control *_Thread_queue_Do_dequeue( the_thread = _Thread_queue_First_locked( the_thread_queue, operations ); if ( the_thread != NULL ) { - _SMP_Assert( the_thread->Lock.current.normal == &the_thread_queue->Queue.Lock ); - _Thread_queue_Extract_critical( &the_thread_queue->Queue, operations, diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c index c288ceeca9..8059446578 100644 --- a/cpukit/score/src/threadqops.c +++ b/cpukit/score/src/threadqops.c @@ -22,13 +22,16 @@ #include #include -static void _Thread_queue_Do_nothing_priority_change( +static void _Thread_queue_Default_priority_change( Thread_Control *the_thread, Priority_Control new_priority, + bool prepend_it, Thread_queue_Queue *queue ) { - /* Do nothing */ + (void) queue; + + _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it ); } static void _Thread_queue_Do_nothing_extract( @@ -39,6 +42,31 @@ static void _Thread_queue_Do_nothing_extract( /* Do nothing */ } +#if defined(RTEMS_SMP) +static void _Thread_queue_Stale_queue_priority_change( + Thread_Control *the_thread, + Priority_Control new_priority, + bool prepend_it, + Thread_queue_Queue *queue +) +{ + ISR_lock_Context lock_context; + + (void) queue; + + /* + * This operation is used to change the priority in case we have a thread + * queue context with a stale thread queue. We own the thread queue lock of + * the former thread queue. In addition, we need the thread wait default + * lock, see _Thread_Wait_restore_default(). + */ + + _Thread_Wait_acquire_default_critical( the_thread, &lock_context ); + _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it ); + _Thread_Wait_release_default_critical( the_thread, &lock_context ); +} +#endif + static Thread_queue_Heads *_Thread_queue_Queue_enqueue( Thread_queue_Queue *queue, Thread_Control *the_thread, @@ -190,6 +218,7 @@ static bool _Thread_queue_Priority_less( static void _Thread_queue_Priority_priority_change( Thread_Control *the_thread, Priority_Control new_priority, + bool prepend_it, Thread_queue_Queue *queue ) { @@ -198,6 +227,8 @@ static void _Thread_queue_Priority_priority_change( _Assert( heads != NULL ); + _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it ); + priority_queue = _Thread_queue_Priority_queue( heads, the_thread ); _RBTree_Extract( @@ -353,22 +384,16 @@ static void _Thread_queue_Priority_inherit_enqueue( #endif if ( priority < owner->current_priority ) { - Scheduler_Node *own_node; - path->update_priority = owner; owner->priority_restore_hint = true; _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); - own_node = _Scheduler_Thread_get_own_node( owner ); - _Scheduler_Node_set_priority( own_node, priority, false ); - - owner->current_priority = priority; - - ( *owner->Wait.operations->priority_change )( + _Thread_queue_Context_priority_change( + &path->Start.Queue_context, owner, priority, - owner->Wait.queue + false ); } else { path->update_priority = NULL; @@ -385,24 +410,21 @@ void _Thread_queue_Boost_priority( if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) { const Scheduler_Control *scheduler; - Scheduler_Node *own_node; Priority_Control boost_priority; the_thread->priority_restore_hint = true; _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); scheduler = _Scheduler_Get_own( the_thread ); - own_node = _Scheduler_Thread_get_own_node( the_thread ); boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR ); - _Scheduler_Node_set_priority( own_node, boost_priority, false ); - the_thread->current_priority = boost_priority; + _Scheduler_Thread_set_priority( the_thread, boost_priority, false ); } } #endif const Thread_queue_Operations _Thread_queue_Operations_default = { - .priority_change = _Thread_queue_Do_nothing_priority_change, + .priority_change = _Thread_queue_Default_priority_change, .extract = _Thread_queue_Do_nothing_extract /* * The default operations are only used in _Thread_Change_priority() and @@ -412,7 +434,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = { }; const Thread_queue_Operations _Thread_queue_Operations_FIFO = { - .priority_change = _Thread_queue_Do_nothing_priority_change, + .priority_change = _Thread_queue_Default_priority_change, .enqueue = _Thread_queue_FIFO_enqueue, .extract = _Thread_queue_FIFO_extract, .first = _Thread_queue_FIFO_first @@ -431,3 +453,10 @@ const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = { .extract = _Thread_queue_Priority_extract, .first = _Thread_queue_Priority_first }; + +#if defined(RTEMS_SMP) +const Thread_queue_Operations _Thread_queue_Operations_stale_queue = { + .priority_change = _Thread_queue_Stale_queue_priority_change, + .extract = _Thread_queue_Do_nothing_extract +}; +#endif diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c index 757f552699..81ba73d704 100644 --- a/cpukit/score/src/threadrestart.c +++ b/cpukit/score/src/threadrestart.c @@ -215,8 +215,7 @@ static void _Thread_Free( Thread_Control *the_thread ) _Workspace_Free( the_thread->Start.tls_area ); #if defined(RTEMS_SMP) - _SMP_ticket_lock_Destroy( &the_thread->Lock.Default ); - _SMP_lock_Stats_destroy( &the_thread->Lock.Stats ); + _ISR_lock_Destroy( &the_thread->Wait.Lock.Default ); _SMP_lock_Stats_destroy( &the_thread->Potpourri_stats ); #endif diff --git a/cpukit/score/src/threadtimeout.c b/cpukit/score/src/threadtimeout.c index a2ba61f9e7..b6b6cc4ac2 100644 --- a/cpukit/score/src/threadtimeout.c +++ b/cpukit/score/src/threadtimeout.c @@ -22,28 +22,15 @@ #include #include -static void _Thread_Do_timeout( Thread_Control *the_thread ) -{ - the_thread->Wait.return_code = STATUS_TIMEOUT; - ( *the_thread->Wait.operations->extract )( - the_thread->Wait.queue, - the_thread - ); - _Thread_Wait_set_queue( the_thread, NULL ); - _Thread_Wait_restore_default_operations( the_thread ); - _Thread_Lock_restore_default( the_thread ); -} - void _Thread_Timeout( Watchdog_Control *watchdog ) { - Thread_Control *the_thread; - void *thread_lock; - ISR_lock_Context lock_context; - Thread_Wait_flags wait_flags; - bool unblock; + Thread_Control *the_thread; + Thread_queue_Context queue_context; + Thread_Wait_flags wait_flags; + bool unblock; the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog ); - thread_lock = _Thread_Lock_acquire( the_thread, &lock_context ); + _Thread_Wait_acquire( the_thread, &queue_context ); wait_flags = _Thread_Wait_flags_get( the_thread ); @@ -52,7 +39,9 @@ void _Thread_Timeout( Watchdog_Control *watchdog ) Thread_Wait_flags ready_again; bool success; - _Thread_Do_timeout( the_thread ); + _Thread_Wait_cancel( the_thread, &queue_context ); + + the_thread->Wait.return_code = STATUS_TIMEOUT; wait_class = wait_flags & THREAD_WAIT_CLASS_MASK; ready_again = wait_class | THREAD_WAIT_STATE_READY_AGAIN; @@ -76,9 +65,10 @@ void _Thread_Timeout( Watchdog_Control *watchdog ) unblock = false; } - _Thread_Lock_release( thread_lock, &lock_context ); + _Thread_Wait_release( the_thread, &queue_context ); if ( unblock ) { + _Thread_Wait_tranquilize( the_thread ); _Thread_Unblock( the_thread ); #if defined(RTEMS_MULTIPROCESSING) diff --git a/testsuites/sptests/spthreadq01/init.c b/testsuites/sptests/spthreadq01/init.c index 85df6bd15c..afaa6527e0 100644 --- a/testsuites/sptests/spthreadq01/init.c +++ b/testsuites/sptests/spthreadq01/init.c @@ -68,13 +68,12 @@ static void wake_up_master(test_context *ctx) static rtems_id get_wait_id(test_context *ctx) { - ISR_lock_Context lock_context; - void *lock; + Thread_queue_Context queue_context; rtems_id id; - lock = _Thread_Lock_acquire(ctx->master, &lock_context); + _Thread_Wait_acquire(ctx->master, &queue_context); id = _Thread_Wait_get_id(ctx->master); - _Thread_Lock_release(lock, &lock_context); + _Thread_Wait_release(ctx->master, &queue_context); return id; } -- cgit v1.2.3