diff options
Diffstat (limited to 'cpukit/score/src')
26 files changed, 1169 insertions, 591 deletions
diff --git a/cpukit/score/src/mutex.c b/cpukit/score/src/mutex.c index bfa36ff994..daa90a528f 100644 --- a/cpukit/score/src/mutex.c +++ b/cpukit/score/src/mutex.c @@ -128,33 +128,20 @@ static void _Mutex_Release_critical( ) { Thread_queue_Heads *heads; - bool keep_priority; + heads = mutex->Queue.Queue.heads; mutex->Queue.Queue.owner = NULL; - --executing->resource_count; - /* - * Ensure that the owner resource count is visible to all other - * processors and that we read the latest priority restore - * hint. - */ - _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); - - heads = mutex->Queue.Queue.heads; - keep_priority = _Thread_Owns_resources( executing ) - || !executing->priority_restore_hint; - - if ( __predict_true( heads == NULL && keep_priority ) ) { + if ( __predict_true( heads == NULL ) ) { _Mutex_Queue_release( mutex, queue_context ); } else { _Thread_queue_Surrender( &mutex->Queue.Queue, - MUTEX_TQ_OPERATIONS, heads, executing, - keep_priority, - queue_context + queue_context, + MUTEX_TQ_OPERATIONS ); } } diff --git a/cpukit/score/src/schedulercbs.c b/cpukit/score/src/schedulercbs.c index 98ec0eb29e..f114d4fa02 100644 --- a/cpukit/score/src/schedulercbs.c +++ b/cpukit/score/src/schedulercbs.c @@ -19,24 +19,28 @@ #endif #include <rtems/score/schedulercbsimpl.h> -#include <rtems/score/threadimpl.h> -#include <rtems/score/wkspace.h> void _Scheduler_CBS_Budget_callout( Thread_Control *the_thread ) { - Priority_Control new_priority; - Priority_Control unused; - Scheduler_CBS_Node *node; - Scheduler_CBS_Server_id server_id; + Scheduler_CBS_Node *node; + Scheduler_CBS_Server_id server_id; + Thread_queue_Context queue_context; + + node = _Scheduler_CBS_Thread_get_node( the_thread ); /* Put violating task to background until the end of period. */ - new_priority = the_thread->Start.initial_priority; - _Thread_Set_priority( the_thread, new_priority, &unused, true ); + _Thread_queue_Context_clear_priority_updates( &queue_context ); + _Scheduler_CBS_Cancel_job( + NULL, + the_thread, + node->deadline_node, + &queue_context + ); + _Thread_Priority_update( &queue_context ); /* Invoke callback function if any. */ - node = _Scheduler_CBS_Thread_get_node( the_thread ); if ( node->cbs_server->cbs_budget_overrun ) { _Scheduler_CBS_Get_server_id( node->cbs_server->task_id, diff --git a/cpukit/score/src/schedulercbsnodeinit.c b/cpukit/score/src/schedulercbsnodeinit.c index 53800693c0..89b6f8e2df 100644 --- a/cpukit/score/src/schedulercbsnodeinit.c +++ b/cpukit/score/src/schedulercbsnodeinit.c @@ -33,4 +33,5 @@ void _Scheduler_CBS_Node_initialize( the_node = _Scheduler_CBS_Node_downcast( node ); the_node->cbs_server = NULL; + the_node->deadline_node = NULL; } diff --git a/cpukit/score/src/schedulercbsreleasejob.c b/cpukit/score/src/schedulercbsreleasejob.c index d2169af899..186f95c779 100644 --- a/cpukit/score/src/schedulercbsreleasejob.c +++ b/cpukit/score/src/schedulercbsreleasejob.c @@ -21,10 +21,12 @@ #include <rtems/score/schedulercbsimpl.h> -Thread_Control *_Scheduler_CBS_Release_job( +void _Scheduler_CBS_Release_job( const Scheduler_Control *scheduler, Thread_Control *the_thread, - uint64_t deadline + Priority_Node *priority_node, + uint64_t deadline, + Thread_queue_Context *queue_context ) { Scheduler_CBS_Node *node; @@ -38,5 +40,37 @@ Thread_Control *_Scheduler_CBS_Release_job( the_thread->cpu_time_budget = serv_info->parameters.budget; } - return _Scheduler_EDF_Release_job( scheduler, the_thread, deadline ); + node->deadline_node = priority_node; + + _Scheduler_EDF_Release_job( + scheduler, + the_thread, + priority_node, + deadline, + queue_context + ); +} + +void _Scheduler_CBS_Cancel_job( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Priority_Node *priority_node, + Thread_queue_Context *queue_context +) +{ + Scheduler_CBS_Node *node; + + node = _Scheduler_CBS_Thread_get_node( the_thread ); + + if ( node->deadline_node != NULL ) { + _Assert( node->deadline_node == priority_node ); + node->deadline_node = NULL; + + _Scheduler_EDF_Cancel_job( + scheduler, + the_thread, + priority_node, + queue_context + ); + } } diff --git a/cpukit/score/src/schedulercbsunblock.c b/cpukit/score/src/schedulercbsunblock.c index 0c1e48ebed..c09f471afa 100644 --- a/cpukit/score/src/schedulercbsunblock.c +++ b/cpukit/score/src/schedulercbsunblock.c @@ -30,13 +30,11 @@ Scheduler_Void_or_thread _Scheduler_CBS_Unblock( Thread_Control *the_thread ) { - Scheduler_EDF_Context *context; - Scheduler_CBS_Node *node; - Scheduler_CBS_Server *serv_info; - Priority_Control priority; - bool prepend_it; + Scheduler_CBS_Node *node; + Scheduler_CBS_Server *serv_info; + Priority_Control priority; + bool prepend_it; - context = _Scheduler_EDF_Get_context( scheduler ); node = _Scheduler_CBS_Thread_get_node( the_thread ); serv_info = node->cbs_server; priority = _Scheduler_Node_get_priority( &node->Base.Base, &prepend_it ); @@ -55,40 +53,19 @@ Scheduler_Void_or_thread _Scheduler_CBS_Unblock( Priority_Control budget_left = priority - _Watchdog_Ticks_since_boot; if ( deadline * budget_left > budget * deadline_left ) { - /* Put late unblocked task to background until the end of period. */ - - priority = node->Base.background_priority; - the_thread->real_priority = priority; + Thread_queue_Context queue_context; - if ( - _Thread_Priority_less_than( - _Thread_Get_priority( the_thread ), - priority - ) || !_Thread_Owns_resources( the_thread ) - ) { - the_thread->current_priority = priority; - } + /* Put late unblocked task to background until the end of period. */ + _Thread_queue_Context_clear_priority_updates( &queue_context ); + _Scheduler_CBS_Cancel_job( + scheduler, + the_thread, + node->deadline_node, + &queue_context + ); } } - node->Base.current_priority = priority; - _Scheduler_EDF_Enqueue( context, &node->Base, priority ); - - /* - * If the thread that was unblocked is more important than the heir, - * then we have a new heir. This may or may not result in a - * context switch. - * - * Normal case: - * If the current thread is preemptible, then we need to do - * a context switch. - * Pseudo-ISR case: - * Even if the thread isn't preemptible, if the new heir is - * a pseudo-ISR system task, we need to do a context switch. - */ - if ( priority < _Thread_Get_priority( _Thread_Heir ) ) { - _Scheduler_Update_heir( the_thread, priority == PRIORITY_PSEUDO_ISR ); - } - + _Scheduler_EDF_Unblock( scheduler, the_thread ); SCHEDULER_RETURN_VOID_OR_NULL; } diff --git a/cpukit/score/src/schedulerdefaultnodeinit.c b/cpukit/score/src/schedulerdefaultnodeinit.c index 10e71f8a05..53aed5274f 100644 --- a/cpukit/score/src/schedulerdefaultnodeinit.c +++ b/cpukit/score/src/schedulerdefaultnodeinit.c @@ -28,7 +28,5 @@ void _Scheduler_default_Node_initialize( Priority_Control priority ) { - (void) scheduler; - - _Scheduler_Node_do_initialize( node, the_thread, priority ); + _Scheduler_Node_do_initialize( scheduler, node, the_thread, priority ); } diff --git a/cpukit/score/src/schedulerdefaultreleasejob.c b/cpukit/score/src/schedulerdefaultreleasejob.c index 7272fc1946..490d58b05f 100644 --- a/cpukit/score/src/schedulerdefaultreleasejob.c +++ b/cpukit/score/src/schedulerdefaultreleasejob.c @@ -21,26 +21,30 @@ #include <rtems/score/scheduler.h> -Thread_Control *_Scheduler_default_Release_job( +void _Scheduler_default_Release_job( const Scheduler_Control *scheduler, Thread_Control *the_thread, - uint64_t deadline + Priority_Node *priority_node, + uint64_t deadline, + Thread_queue_Context *queue_context ) { (void) scheduler; (void) the_thread; + (void) priority_node; (void) deadline; - - return NULL; + (void) queue_context; } -Thread_Control *_Scheduler_default_Cancel_job( +void _Scheduler_default_Cancel_job( const Scheduler_Control *scheduler, - Thread_Control *the_thread + Thread_Control *the_thread, + Priority_Node *priority_node, + Thread_queue_Context *queue_context ) { (void) scheduler; (void) the_thread; - - return NULL; + (void) priority_node; + (void) queue_context; } diff --git a/cpukit/score/src/scheduleredfchangepriority.c b/cpukit/score/src/scheduleredfchangepriority.c index 9a73128e4c..8940b1d54b 100644 --- a/cpukit/score/src/scheduleredfchangepriority.c +++ b/cpukit/score/src/scheduleredfchangepriority.c @@ -54,16 +54,12 @@ Scheduler_Void_or_thread _Scheduler_EDF_Update_priority( node = _Scheduler_EDF_Thread_get_node( the_thread ); priority = _Scheduler_Node_get_priority( &node->Base, &prepend_it ); - if ( priority == node->current_priority ) { + if ( priority == node->priority ) { /* Nothing to do */ SCHEDULER_RETURN_VOID_OR_NULL; } - if ( ( priority & SCHEDULER_EDF_PRIO_MSB ) != 0 ) { - node->background_priority = priority; - } - - node->current_priority = priority; + node->priority = priority; context = _Scheduler_EDF_Get_context( scheduler ); _Scheduler_EDF_Extract( context, node ); diff --git a/cpukit/score/src/scheduleredfnodeinit.c b/cpukit/score/src/scheduleredfnodeinit.c index d290bd74fd..94f8facbe3 100644 --- a/cpukit/score/src/scheduleredfnodeinit.c +++ b/cpukit/score/src/scheduleredfnodeinit.c @@ -29,11 +29,9 @@ void _Scheduler_EDF_Node_initialize( { Scheduler_EDF_Node *the_node; - (void) scheduler; - - _Scheduler_Node_do_initialize( node, the_thread, priority ); + _Scheduler_Node_do_initialize( scheduler, node, the_thread, priority ); the_node = _Scheduler_EDF_Node_downcast( node ); - the_node->thread = the_thread; _RBTree_Initialize_node( &the_node->Node ); + the_node->priority = priority; } diff --git a/cpukit/score/src/scheduleredfreleasejob.c b/cpukit/score/src/scheduleredfreleasejob.c index 4c74c48699..c19d9b9d24 100644 --- a/cpukit/score/src/scheduleredfreleasejob.c +++ b/cpukit/score/src/scheduleredfreleasejob.c @@ -20,75 +20,47 @@ #include <rtems/score/scheduleredfimpl.h> -static bool _Scheduler_EDF_Release_job_filter( - Thread_Control *the_thread, - Priority_Control *new_priority_p, - void *arg +void _Scheduler_EDF_Release_job( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Priority_Node *priority_node, + uint64_t deadline, + Thread_queue_Context *queue_context ) { - Scheduler_EDF_Node *node; - Priority_Control current_priority; - Priority_Control new_priority; + (void) scheduler; - node = _Scheduler_EDF_Thread_get_node( the_thread ); + _Thread_Wait_acquire_critical( the_thread, queue_context ); - current_priority = _Thread_Get_priority( the_thread ); - new_priority = *new_priority_p; + _Priority_Node_set_priority( priority_node, deadline ); - node->current_priority = new_priority; - the_thread->real_priority = new_priority; + if ( _Priority_Node_is_active( priority_node ) ) { + _Thread_Priority_changed( + the_thread, + priority_node, + false, + queue_context + ); + } else { + _Thread_Priority_add( the_thread, priority_node, queue_context ); + } - return _Thread_Priority_less_than( current_priority, new_priority ) - || !_Thread_Owns_resources( the_thread ); + _Thread_Wait_release_critical( the_thread, queue_context ); } -Thread_Control *_Scheduler_EDF_Release_job( +void _Scheduler_EDF_Cancel_job( const Scheduler_Control *scheduler, Thread_Control *the_thread, - uint64_t deadline -) -{ - return _Thread_Apply_priority( - the_thread, - deadline, - NULL, - _Scheduler_EDF_Release_job_filter, - true - ); -} - -static bool _Scheduler_EDF_Cancel_job_filter( - Thread_Control *the_thread, - Priority_Control *new_priority_p, - void *arg + Priority_Node *priority_node, + Thread_queue_Context *queue_context ) { - Scheduler_EDF_Node *node; - Priority_Control current_priority; - Priority_Control new_priority; + (void) scheduler; - node = _Scheduler_EDF_Thread_get_node( the_thread ); + _Thread_Wait_acquire_critical( the_thread, queue_context ); - current_priority = _Thread_Get_priority( the_thread ); - new_priority = node->background_priority; + _Thread_Priority_remove( the_thread, priority_node, queue_context ); + _Priority_Node_set_inactive( priority_node ); - node->current_priority = new_priority; - the_thread->real_priority = new_priority; - - return _Thread_Priority_less_than( current_priority, new_priority ) - || !_Thread_Owns_resources( the_thread ); -} - -Thread_Control *_Scheduler_EDF_Cancel_job( - const Scheduler_Control *scheduler, - Thread_Control *the_thread -) -{ - return _Thread_Apply_priority( - the_thread, - 0, - NULL, - _Scheduler_EDF_Cancel_job_filter, - true - ); + _Thread_Wait_release_critical( the_thread, queue_context ); } diff --git a/cpukit/score/src/scheduleredfunblock.c b/cpukit/score/src/scheduleredfunblock.c index 9b156eca46..a5cc4b6a37 100644 --- a/cpukit/score/src/scheduleredfunblock.c +++ b/cpukit/score/src/scheduleredfunblock.c @@ -37,7 +37,7 @@ Scheduler_Void_or_thread _Scheduler_EDF_Unblock( priority = _Scheduler_Node_get_priority( &node->Base, &prepend_it ); (void) prepend_it; - node->current_priority = priority; + node->priority = priority; _Scheduler_EDF_Enqueue( context, node, priority ); /* diff --git a/cpukit/score/src/scheduleredfyield.c b/cpukit/score/src/scheduleredfyield.c index 06c1b46895..3e64e5ca05 100644 --- a/cpukit/score/src/scheduleredfyield.c +++ b/cpukit/score/src/scheduleredfyield.c @@ -33,7 +33,7 @@ Scheduler_Void_or_thread _Scheduler_EDF_Yield( node = _Scheduler_EDF_Thread_get_node( the_thread ); _Scheduler_EDF_Extract( context, node ); - _Scheduler_EDF_Enqueue( context, node, node->current_priority ); + _Scheduler_EDF_Enqueue( context, node, node->priority ); _Scheduler_EDF_Schedule_body( scheduler, the_thread, true ); SCHEDULER_RETURN_VOID_OR_NULL; diff --git a/cpukit/score/src/schedulerpriority.c b/cpukit/score/src/schedulerpriority.c index 11cee92424..ddfd973e0a 100644 --- a/cpukit/score/src/schedulerpriority.c +++ b/cpukit/score/src/schedulerpriority.c @@ -43,7 +43,7 @@ void _Scheduler_priority_Node_initialize( Scheduler_priority_Context *context; Scheduler_priority_Node *the_node; - _Scheduler_Node_do_initialize( node, the_thread, priority ); + _Scheduler_Node_do_initialize( scheduler, node, the_thread, priority ); context = _Scheduler_priority_Get_context( scheduler ); the_node = _Scheduler_priority_Node_downcast( node ); diff --git a/cpukit/score/src/schedulerprioritysmp.c b/cpukit/score/src/schedulerprioritysmp.c index 07e7af4268..e624a6acf4 100644 --- a/cpukit/score/src/schedulerprioritysmp.c +++ b/cpukit/score/src/schedulerprioritysmp.c @@ -57,7 +57,12 @@ void _Scheduler_priority_SMP_Node_initialize( Scheduler_priority_SMP_Node *the_node; the_node = _Scheduler_priority_SMP_Node_downcast( node ); - _Scheduler_SMP_Node_initialize( &the_node->Base, the_thread, priority ); + _Scheduler_SMP_Node_initialize( + scheduler, + &the_node->Base, + the_thread, + priority + ); context = _Scheduler_Get_context( scheduler ); self = _Scheduler_priority_SMP_Get_self( context ); diff --git a/cpukit/score/src/schedulersimplesmp.c b/cpukit/score/src/schedulersimplesmp.c index 8f86ea87da..9606896a94 100644 --- a/cpukit/score/src/schedulersimplesmp.c +++ b/cpukit/score/src/schedulersimplesmp.c @@ -52,7 +52,7 @@ void _Scheduler_simple_SMP_Node_initialize( Scheduler_SMP_Node *the_node; the_node = _Scheduler_SMP_Node_downcast( node ); - _Scheduler_SMP_Node_initialize( the_node, the_thread, priority ); + _Scheduler_SMP_Node_initialize( scheduler, the_node, the_thread, priority ); } static void _Scheduler_simple_SMP_Do_update( diff --git a/cpukit/score/src/schedulerstrongapa.c b/cpukit/score/src/schedulerstrongapa.c index 5d7c7f7a1d..fc6d0125e7 100644 --- a/cpukit/score/src/schedulerstrongapa.c +++ b/cpukit/score/src/schedulerstrongapa.c @@ -183,7 +183,12 @@ void _Scheduler_strong_APA_Node_initialize( Scheduler_strong_APA_Node *the_node; the_node = _Scheduler_strong_APA_Node_downcast( node ); - _Scheduler_SMP_Node_initialize( &the_node->Base, the_thread, priority ); + _Scheduler_SMP_Node_initialize( + scheduler, + &the_node->Base, + the_thread, + priority + ); context = _Scheduler_Get_context( scheduler ); self = _Scheduler_strong_APA_Get_self( context ); diff --git a/cpukit/score/src/thread.c b/cpukit/score/src/thread.c index c569ae5bab..864b21bf6f 100644 --- a/cpukit/score/src/thread.c +++ b/cpukit/score/src/thread.c @@ -32,9 +32,7 @@ THREAD_OFFSET_ASSERT( Object ); THREAD_OFFSET_ASSERT( Join_queue ); THREAD_OFFSET_ASSERT( current_state ); -THREAD_OFFSET_ASSERT( current_priority ); -THREAD_OFFSET_ASSERT( real_priority ); -THREAD_OFFSET_ASSERT( priority_restore_hint ); +THREAD_OFFSET_ASSERT( Real_priority ); THREAD_OFFSET_ASSERT( resource_count ); THREAD_OFFSET_ASSERT( Scheduler ); THREAD_OFFSET_ASSERT( Wait ); diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c index 3429e1a88d..c10c712710 100644 --- a/cpukit/score/src/threadchangepriority.c +++ b/cpukit/score/src/threadchangepriority.c @@ -10,6 +10,8 @@ * COPYRIGHT (c) 1989-2014. * On-Line Applications Research Corporation (OAR). * + * Copyright (c) 2013, 2016 embedded brains GmbH + * * The license and distribution terms for this file may be * found in the file LICENSE in this distribution or at * http://www.rtems.org/license/LICENSE. @@ -20,145 +22,321 @@ #endif #include <rtems/score/threadimpl.h> +#include <rtems/score/assert.h> #include <rtems/score/schedulerimpl.h> -static Thread_Control *_Thread_Apply_priority_locked( +static void _Thread_Set_scheduler_node_priority( + Priority_Aggregation *priority_aggregation, + bool prepend_it +) +{ + _Scheduler_Node_set_priority( + SCHEDULER_NODE_OF_WAIT_PRIORITY_NODE( priority_aggregation ), + _Priority_Get_priority( priority_aggregation ), + prepend_it + ); +} + +#if defined(RTEMS_SMP) +static void _Thread_Priority_action_add( + Priority_Aggregation *priority_aggregation, + Priority_Actions *priority_actions, + void *arg +) +{ + _Thread_Set_scheduler_node_priority( priority_aggregation, false ); + _Priority_Set_action_type( priority_aggregation, PRIORITY_ACTION_ADD ); + _Priority_Actions_add( priority_actions, priority_aggregation ); +} + +static void _Thread_Priority_action_remove( + Priority_Aggregation *priority_aggregation, + Priority_Actions *priority_actions, + void *arg +) +{ + _Thread_Set_scheduler_node_priority( priority_aggregation, true ); + _Priority_Set_action_type( priority_aggregation, PRIORITY_ACTION_REMOVE ); + _Priority_Actions_add( priority_actions, priority_aggregation ); +} +#endif + +static void _Thread_Priority_action_change( + Priority_Aggregation *priority_aggregation, + bool prepend_it, + Priority_Actions *priority_actions, + void *arg +) +{ + _Thread_Set_scheduler_node_priority( priority_aggregation, prepend_it ); +#if defined(RTEMS_SMP) || defined(RTEMS_DEBUG) + _Priority_Set_action_type( priority_aggregation, PRIORITY_ACTION_CHANGE ); +#endif + _Priority_Actions_add( priority_actions, priority_aggregation ); +} + +static void _Thread_Priority_do_perform_actions( Thread_Control *the_thread, - Priority_Control new_priority, - void *arg, - Thread_Change_priority_filter filter, + Thread_queue_Queue *queue, + const Thread_queue_Operations *operations, bool prepend_it, Thread_queue_Context *queue_context ) { - /* - * For simplicity set the priority restore hint unconditionally since this is - * an average case optimization. Otherwise complicated atomic operations - * would be necessary. Synchronize with a potential read of the resource - * count in the filter function. See also _CORE_mutex_Surrender(), - * _Thread_Set_priority_filter() and _Thread_Restore_priority_filter(). - */ - the_thread->priority_restore_hint = true; - _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); + Priority_Aggregation *priority_aggregation; - /* - * Do not bother recomputing all the priority related information if - * we are not REALLY changing priority. - */ - if ( ( *filter )( the_thread, &new_priority, arg ) ) { - _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it ); + _Assert( !_Priority_Actions_is_empty( &queue_context->Priority.Actions ) ); + priority_aggregation = _Priority_Actions_move( &queue_context->Priority.Actions ); - ( *the_thread->Wait.operations->priority_change )( - the_thread->Wait.queue, - the_thread, - new_priority + do { + Priority_Aggregation *next_aggregation; + Priority_Node *priority_action_node; + Priority_Action_type priority_action_type; + + next_aggregation = _Priority_Get_next_action( priority_aggregation ); + + priority_action_node = priority_aggregation->Action.node; + priority_action_type = priority_aggregation->Action.type; + + switch ( priority_action_type ) { + case PRIORITY_ACTION_ADD: +#if defined(RTEMS_SMP) + _Priority_Insert( + priority_aggregation, + priority_action_node, + &queue_context->Priority.Actions, + _Thread_Priority_action_add, + _Thread_Priority_action_change, + NULL + ); +#else + _Priority_Non_empty_insert( + priority_aggregation, + priority_action_node, + &queue_context->Priority.Actions, + _Thread_Priority_action_change, + NULL + ); +#endif + break; + case PRIORITY_ACTION_REMOVE: +#if defined(RTEMS_SMP) + _Priority_Extract( + priority_aggregation, + priority_action_node, + &queue_context->Priority.Actions, + _Thread_Priority_action_remove, + _Thread_Priority_action_change, + NULL + ); +#else + _Priority_Extract_non_empty( + priority_aggregation, + priority_action_node, + &queue_context->Priority.Actions, + _Thread_Priority_action_change, + NULL + ); +#endif + break; + default: + _Assert( priority_action_type == PRIORITY_ACTION_CHANGE ); + _Priority_Changed( + priority_aggregation, + priority_action_node, + prepend_it, + &queue_context->Priority.Actions, + _Thread_Priority_action_change, + NULL + ); + break; + } + + priority_aggregation = next_aggregation; + } while ( _Priority_Actions_is_valid( priority_aggregation ) ); + + if ( !_Priority_Actions_is_empty( &queue_context->Priority.Actions ) ) { + _Thread_queue_Context_add_priority_update( queue_context, the_thread ); + ( *operations->priority_actions )( + queue, + &queue_context->Priority.Actions ); - } else { - the_thread = NULL; } +} + +void _Thread_Priority_perform_actions( + Thread_Control *start_of_path, + Thread_queue_Context *queue_context +) +{ +#if defined(RTEMS_SMP) + Thread_queue_Link *link; +#endif + Thread_Control *the_thread; + size_t update_count; + + _Assert( start_of_path != NULL ); - return the_thread; +#if defined(RTEMS_SMP) + link = &queue_context->Path.Start; +#endif + the_thread = start_of_path; + update_count = _Thread_queue_Context_save_priority_updates( queue_context ); + + while ( true ) { + Thread_queue_Queue *queue; + +#if defined(RTEMS_SMP) + _Assert( link->owner == the_thread ); + queue = link->Lock_context.Wait.queue; +#else + queue = the_thread->Wait.queue; +#endif + + _Thread_Priority_do_perform_actions( + the_thread, + queue, + the_thread->Wait.operations, + false, + queue_context + ); + + if ( _Priority_Actions_is_empty( &queue_context->Priority.Actions ) ) { + return; + } + + _Assert( queue != NULL ); + the_thread = queue->owner; + _Assert( the_thread != NULL ); + +#if defined(RTEMS_SMP) + link = THREAD_QUEUE_LINK_OF_PATH_NODE( _Chain_Next( &link->Path_node ) ); +#endif + + /* + * In case the priority action list is non-empty, then the current thread + * is enqueued on a thread queue. There is no need to notify the scheduler + * about a priority change, since it will pick up the new priority once it + * is unblocked. Restore the previous set of threads bound to update the + * priority. + */ + _Thread_queue_Context_restore_priority_updates( + queue_context, + update_count + ); + } } -Thread_Control *_Thread_Apply_priority( - Thread_Control *the_thread, - Priority_Control new_priority, - void *arg, - Thread_Change_priority_filter filter, - bool prepend_it +static void _Thread_Priority_apply( + Thread_Control *the_thread, + Priority_Node *priority_action_node, + Thread_queue_Context *queue_context, + bool prepend_it, + Priority_Action_type priority_action_type ) { - Thread_queue_Context queue_context; - Thread_Control *the_thread_to_update; + Scheduler_Node *own_node; + Thread_queue_Queue *queue; - _Thread_Wait_acquire( the_thread, &queue_context ); - the_thread_to_update = _Thread_Apply_priority_locked( + own_node = _Thread_Scheduler_get_own_node( the_thread ); + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &own_node->Wait.Priority, + priority_action_node, + priority_action_type + ); + queue = the_thread->Wait.queue; + _Thread_Priority_do_perform_actions( the_thread, - new_priority, - arg, - filter, + queue, + the_thread->Wait.operations, prepend_it, - &queue_context + queue_context ); - _Thread_Wait_release( the_thread, &queue_context ); - return the_thread_to_update; -} - -void _Thread_Update_priority( Thread_Control *the_thread ) -{ - if ( the_thread != NULL ) { - ISR_lock_Context lock_context; - _Thread_State_acquire( the_thread, &lock_context ); - _Scheduler_Update_priority( the_thread ); - _Thread_State_release( the_thread, &lock_context ); + if ( !_Priority_Actions_is_empty( &queue_context->Priority.Actions ) ) { + _Thread_queue_Path_acquire_critical( queue, the_thread, queue_context ); + _Thread_Priority_perform_actions( queue->owner, queue_context ); + _Thread_queue_Path_release_critical( queue_context ); } } -void _Thread_Change_priority( - Thread_Control *the_thread, - Priority_Control new_priority, - void *arg, - Thread_Change_priority_filter filter, - bool prepend_it +void _Thread_Priority_add( + Thread_Control *the_thread, + Priority_Node *priority_node, + Thread_queue_Context *queue_context ) { - the_thread = _Thread_Apply_priority( + _Thread_Priority_apply( the_thread, - new_priority, - arg, - filter, - prepend_it + priority_node, + queue_context, + false, + PRIORITY_ACTION_ADD ); - _Thread_Update_priority( the_thread ); } -static bool _Thread_Raise_priority_filter( - Thread_Control *the_thread, - Priority_Control *new_priority, - void *arg +void _Thread_Priority_remove( + Thread_Control *the_thread, + Priority_Node *priority_node, + Thread_queue_Context *queue_context ) { - return _Thread_Priority_less_than( - _Thread_Get_priority( the_thread ), - *new_priority + _Thread_Priority_apply( + the_thread, + priority_node, + queue_context, + true, + PRIORITY_ACTION_REMOVE ); } -void _Thread_Raise_priority( - Thread_Control *the_thread, - Priority_Control new_priority +void _Thread_Priority_changed( + Thread_Control *the_thread, + Priority_Node *priority_node, + bool prepend_it, + Thread_queue_Context *queue_context ) { - _Thread_Change_priority( + _Thread_Priority_apply( the_thread, - new_priority, - NULL, - _Thread_Raise_priority_filter, - false + priority_node, + queue_context, + prepend_it, + PRIORITY_ACTION_CHANGE ); } -static bool _Thread_Restore_priority_filter( - Thread_Control *the_thread, - Priority_Control *new_priority, - void *arg +void _Thread_Priority_replace( + Thread_Control *the_thread, + Priority_Node *victim_node, + Priority_Node *replacement_node ) { - *new_priority = the_thread->real_priority; - - the_thread->priority_restore_hint = false; + Scheduler_Node *own_node; - return *new_priority != _Thread_Get_priority( the_thread ); + own_node = _Thread_Scheduler_get_own_node( the_thread ); + _Priority_Replace( &own_node->Wait.Priority, victim_node, replacement_node ); } -void _Thread_Restore_priority( Thread_Control *the_thread ) +void _Thread_Priority_update( Thread_queue_Context *queue_context ) { - _Thread_Change_priority( - the_thread, - 0, - NULL, - _Thread_Restore_priority_filter, - true - ); + size_t i; + size_t n; + + n = queue_context->Priority.update_count; + + /* + * Update the priority of all threads of the set. Do not care to clear the + * set, since the thread queue context will soon get destroyed anyway. + */ + for ( i = 0; i < n ; ++i ) { + Thread_Control *the_thread; + ISR_lock_Context lock_context; + + the_thread = queue_context->Priority.update[ i ]; + _Thread_State_acquire( the_thread, &lock_context ); + _Scheduler_Update_priority( the_thread ); + _Thread_State_release( the_thread, &lock_context ); + } } diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c index 90f58e646d..76b1dff6e4 100644 --- a/cpukit/score/src/threadinitialize.c +++ b/cpukit/score/src/threadinitialize.c @@ -198,12 +198,15 @@ bool _Thread_Initialize( the_thread->current_state = STATES_DORMANT; the_thread->Wait.operations = &_Thread_queue_Operations_default; - the_thread->current_priority = priority; - the_thread->real_priority = priority; the_thread->Start.initial_priority = priority; RTEMS_STATIC_ASSERT( THREAD_WAIT_FLAGS_INITIAL == 0, Wait_flags ); + _Priority_Node_initialize( &the_thread->Real_priority, priority ); + _Priority_Initialize_one( + &scheduler_node->Wait.Priority, + &the_thread->Real_priority + ); _Scheduler_Node_initialize( scheduler, scheduler_node, the_thread, priority ); scheduler_node_initialized = true; diff --git a/cpukit/score/src/threadmp.c b/cpukit/score/src/threadmp.c index cbb6c1c6b1..49e1c27979 100644 --- a/cpukit/score/src/threadmp.c +++ b/cpukit/score/src/threadmp.c @@ -80,6 +80,7 @@ void _Thread_MP_Handler_initialization ( #endif proxy->Scheduler.node = &proxy->Scheduler_node; _Scheduler_Node_do_initialize( + _Scheduler_Get_by_CPU_index( 0 ), &proxy->Scheduler_node, (Thread_Control *) proxy, 0 @@ -150,7 +151,7 @@ Thread_Control *_Thread_MP_Allocate_proxy ( the_proxy->receive_packet = receive_packet; the_proxy->Object.id = source_tid; - the_proxy->current_priority = receive_packet->source_priority; + the_proxy->Real_priority.priority = receive_packet->source_priority; the_proxy->current_state = _States_Set( STATES_DORMANT, the_state ); the_proxy->Wait.count = executing->Wait.count; diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c index f16dff0005..2864c0924a 100644 --- a/cpukit/score/src/threadqenqueue.c +++ b/cpukit/score/src/threadqenqueue.c @@ -162,17 +162,16 @@ static void _Thread_queue_Link_remove( Thread_queue_Link *link ) } #endif -#define THREAD_QUEUE_LINK_OF_PATH_NODE( node ) \ - RTEMS_CONTAINER_OF( node, Thread_queue_Link, Path_node ); - -static void _Thread_queue_Path_release( Thread_queue_Path *path ) +void _Thread_queue_Path_release_critical( + Thread_queue_Context *queue_context +) { #if defined(RTEMS_SMP) Chain_Node *head; Chain_Node *node; - head = _Chain_Head( &path->Links ); - node = _Chain_Last( &path->Links ); + head = _Chain_Head( &queue_context->Path.Links ); + node = _Chain_Last( &queue_context->Path.Links ); if ( head != node ) { Thread_queue_Link *link; @@ -215,18 +214,17 @@ static void _Thread_queue_Path_release( Thread_queue_Path *path ) } } #else - (void) path; + (void) queue_context; #endif } -static bool _Thread_queue_Path_acquire( - Thread_Control *the_thread, - Thread_queue_Queue *queue, - Thread_queue_Path *path +bool _Thread_queue_Path_acquire_critical( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { Thread_Control *owner; - #if defined(RTEMS_SMP) Thread_queue_Link *link; Thread_queue_Queue *target; @@ -239,7 +237,7 @@ static bool _Thread_queue_Path_acquire( * this would result in an unrecoverable deadlock of the overall system. */ - _Chain_Initialize_empty( &path->Links ); + _Chain_Initialize_empty( &queue_context->Path.Links ); owner = queue->owner; @@ -251,13 +249,15 @@ static bool _Thread_queue_Path_acquire( return false; } - _RBTree_Initialize_node( &path->Start.Registry_node ); - _Chain_Initialize_node( &path->Start.Path_node ); - _Chain_Initialize_node( &path->Start.Lock_context.Wait.Gate.Node ); - link = &path->Start; + _RBTree_Initialize_node( &queue_context->Path.Start.Registry_node ); + _Chain_Initialize_node( &queue_context->Path.Start.Path_node ); + _Chain_Initialize_node( + &queue_context->Path.Start.Lock_context.Wait.Gate.Node + ); + link = &queue_context->Path.Start; do { - _Chain_Append_unprotected( &path->Links, &link->Path_node ); + _Chain_Append_unprotected( &queue_context->Path.Links, &link->Path_node ); link->owner = owner; _Thread_Wait_acquire_default_critical( @@ -293,7 +293,6 @@ static bool _Thread_queue_Path_acquire( } } else { link->Lock_context.Wait.queue = NULL; - _Thread_queue_Path_release( path ); return false; } } else { @@ -345,9 +344,8 @@ void _Thread_queue_Enqueue_critical( Thread_queue_Context *queue_context ) { - Thread_queue_Path path; - Per_CPU_Control *cpu_self; - bool success; + Per_CPU_Control *cpu_self; + bool success; #if defined(RTEMS_MULTIPROCESSING) if ( _Thread_MP_Is_receive( the_thread ) && the_thread->receive_packet ) { @@ -357,7 +355,8 @@ void _Thread_queue_Enqueue_critical( _Thread_Wait_claim( the_thread, queue, operations ); - if ( !_Thread_queue_Path_acquire( the_thread, queue, &path ) ) { + if ( !_Thread_queue_Path_acquire_critical( queue, the_thread, queue_context ) ) { + _Thread_queue_Path_release_critical( queue_context ); _Thread_Wait_restore_default( the_thread ); _Thread_queue_Queue_release( queue, &queue_context->Lock_context.Lock_context ); _Thread_Wait_tranquilize( the_thread ); @@ -365,9 +364,10 @@ void _Thread_queue_Enqueue_critical( return; } - ( *operations->enqueue )( queue, the_thread, &path ); + _Thread_queue_Context_clear_priority_updates( queue_context ); + ( *operations->enqueue )( queue, the_thread, queue_context ); - _Thread_queue_Path_release( &path ); + _Thread_queue_Path_release_critical( queue_context ); the_thread->Wait.return_code = STATUS_SUCCESSFUL; _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK ); @@ -437,7 +437,7 @@ void _Thread_queue_Enqueue_critical( _Thread_Remove_timer_and_unblock( the_thread, queue ); } - _Thread_Update_priority( path.update_priority ); + _Thread_Priority_update( queue_context ); _Thread_Dispatch_enable( cpu_self ); } @@ -488,20 +488,17 @@ static bool _Thread_queue_Make_ready_again( Thread_Control *the_thread ) return unblock; } -bool _Thread_queue_Do_extract_locked( +bool _Thread_queue_Extract_locked( Thread_queue_Queue *queue, const Thread_queue_Operations *operations, - Thread_Control *the_thread -#if defined(RTEMS_MULTIPROCESSING) - , - const Thread_queue_Context *queue_context -#endif + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { #if defined(RTEMS_MULTIPROCESSING) _Thread_queue_MP_set_callout( the_thread, queue_context ); #endif - ( *operations->extract )( queue, the_thread ); + ( *operations->extract )( queue, the_thread, queue_context ); return _Thread_queue_Make_ready_again( the_thread ); } @@ -587,46 +584,51 @@ void _Thread_queue_Extract( Thread_Control *the_thread ) void _Thread_queue_Surrender( Thread_queue_Queue *queue, - const Thread_queue_Operations *operations, Thread_queue_Heads *heads, Thread_Control *previous_owner, - bool keep_priority, - Thread_queue_Context *queue_context + Thread_queue_Context *queue_context, + const Thread_queue_Operations *operations ) { - if ( heads != NULL ) { - Thread_Control *new_owner; - bool unblock; + Thread_Control *new_owner; + bool unblock; + Per_CPU_Control *cpu_self; - new_owner = ( *operations->surrender )( queue, heads, previous_owner ); - queue->owner = new_owner; + _Assert( heads != NULL ); + + _Thread_queue_Context_clear_priority_updates( queue_context ); + new_owner = ( *operations->surrender )( + queue, + heads, + previous_owner, + queue_context + ); + queue->owner = new_owner; #if defined(RTEMS_MULTIPROCESSING) - if ( !_Thread_queue_MP_set_callout( new_owner, queue_context ) ) + if ( !_Thread_queue_MP_set_callout( new_owner, queue_context ) ) #endif - { - ++new_owner->resource_count; - } + { + ++new_owner->resource_count; + } - unblock = _Thread_queue_Make_ready_again( new_owner ); + unblock = _Thread_queue_Make_ready_again( new_owner ); - _Thread_queue_Unblock_critical( - unblock, - queue, - new_owner, - &queue_context->Lock_context.Lock_context - ); - } else { - _Thread_queue_Queue_release( queue, &queue_context->Lock_context.Lock_context ); - } + cpu_self = _Thread_Dispatch_disable_critical( + &queue_context->Lock_context.Lock_context + ); + _Thread_queue_Queue_release( + queue, + &queue_context->Lock_context.Lock_context + ); - if ( !keep_priority ) { - Per_CPU_Control *cpu_self; + _Thread_Priority_update( queue_context ); - cpu_self = _Thread_Dispatch_disable(); - _Thread_Restore_priority( previous_owner ); - _Thread_Dispatch_enable( cpu_self ); + if ( unblock ) { + _Thread_Remove_timer_and_unblock( new_owner, queue ); } + + _Thread_Dispatch_enable( cpu_self ); } Thread_Control *_Thread_queue_Do_dequeue( diff --git a/cpukit/score/src/threadqflush.c b/cpukit/score/src/threadqflush.c index fb1323073d..9ea82af24a 100644 --- a/cpukit/score/src/threadqflush.c +++ b/cpukit/score/src/threadqflush.c @@ -99,8 +99,11 @@ size_t _Thread_queue_Flush_critical( if ( do_unblock ) { Scheduler_Node *scheduler_node; - scheduler_node = _Scheduler_Thread_get_own_node( first ); - _Chain_Append_unprotected( &unblock, &scheduler_node->Wait.Node.Chain ); + scheduler_node = _Thread_Scheduler_get_own_node( first ); + _Chain_Append_unprotected( + &unblock, + &scheduler_node->Wait.Priority.Node.Node.Chain + ); } ++flushed; @@ -123,7 +126,7 @@ size_t _Thread_queue_Flush_critical( Chain_Node *next; next = _Chain_Next( node ); - scheduler_node = SCHEDULER_NODE_OF_WAIT_CHAIN_NODE( node ); + scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY_NODE( node ); the_thread = _Scheduler_Node_get_owner( scheduler_node ); _Thread_Remove_timer_and_unblock( the_thread, queue ); diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c index e20241d494..602d6bbffb 100644 --- a/cpukit/score/src/threadqops.c +++ b/cpukit/score/src/threadqops.c @@ -22,31 +22,58 @@ #include <rtems/score/rbtreeimpl.h> #include <rtems/score/schedulerimpl.h> -static void _Thread_queue_Do_nothing_priority_change( +#define THREAD_QUEUE_CONTEXT_OF_PRIORITY_ACTIONS( priority_actions ) \ + RTEMS_CONTAINER_OF( \ + priority_actions, \ + Thread_queue_Context, \ + Priority.Actions \ + ) + +#define THREAD_QUEUE_PRIORITY_QUEUE_OF_PRIORITY_AGGREGATION( \ + priority_aggregation \ +) \ + RTEMS_CONTAINER_OF( \ + priority_aggregation, \ + Thread_queue_Priority_queue, \ + Queue \ + ) + +static void _Thread_queue_Do_nothing_priority_actions( Thread_queue_Queue *queue, - Thread_Control *the_thread, - Priority_Control new_priority + Priority_Actions *priority_actions ) { (void) queue; - (void) the_thread; - (void) new_priority; + _Priority_Actions_initialize_empty( priority_actions ); } static void _Thread_queue_Do_nothing_extract( - Thread_queue_Queue *queue, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { (void) queue; (void) the_thread; + (void) queue_context; } -static Thread_queue_Heads *_Thread_queue_Queue_enqueue( - Thread_queue_Queue *queue, - Thread_Control *the_thread, - void ( *initialize )( Thread_queue_Heads *, Thread_Control * ), - void ( *enqueue )( Thread_queue_Heads *, Thread_Control * ) +static void _Thread_queue_Queue_enqueue( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + void ( *initialize )( + Thread_queue_Queue *, + Thread_Control *, + Thread_queue_Context *, + Thread_queue_Heads * + ), + void ( *enqueue )( + Thread_queue_Queue *, + Thread_Control *, + Thread_queue_Context *, + Thread_queue_Heads * + ) ) { Thread_queue_Heads *heads; @@ -63,20 +90,26 @@ static Thread_queue_Heads *_Thread_queue_Queue_enqueue( heads = spare_heads; queue->heads = heads; _Chain_Prepend_unprotected( &heads->Free_chain, &spare_heads->Free_node ); - ( *initialize )( heads, the_thread ); + ( *initialize )( queue, the_thread, queue_context, heads ); } else { _Chain_Prepend_unprotected( &heads->Free_chain, &spare_heads->Free_node ); - ( *enqueue )( heads, the_thread ); + ( *enqueue )( queue, the_thread, queue_context, heads ); } - - return heads; } static void _Thread_queue_Queue_extract( - Thread_queue_Queue *queue, - Thread_queue_Heads *heads, - Thread_Control *the_thread, - void ( *extract )( Thread_queue_Heads *, Thread_Control * ) + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *current_or_previous_owner, + Thread_queue_Context *queue_context, + Thread_Control *the_thread, + void ( *extract )( + Thread_queue_Queue *, + Thread_queue_Heads *, + Thread_Control *, + Thread_queue_Context *, + Thread_Control * + ) ) { _Assert( heads != NULL ); @@ -91,76 +124,95 @@ static void _Thread_queue_Queue_extract( queue->heads = NULL; } - ( *extract )( heads, the_thread ); + ( *extract )( + queue, + heads, + current_or_previous_owner, + queue_context, + the_thread + ); } static void _Thread_queue_FIFO_do_initialize( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads ) { Scheduler_Node *scheduler_node; - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); - _Chain_Initialize_node( &scheduler_node->Wait.Node.Chain ); + _Chain_Initialize_node( &scheduler_node->Wait.Priority.Node.Node.Chain ); _Chain_Initialize_one( &heads->Heads.Fifo, - &scheduler_node->Wait.Node.Chain + &scheduler_node->Wait.Priority.Node.Node.Chain ); } static void _Thread_queue_FIFO_do_enqueue( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads ) { Scheduler_Node *scheduler_node; - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); - _Chain_Initialize_node( &scheduler_node->Wait.Node.Chain ); + _Chain_Initialize_node( &scheduler_node->Wait.Priority.Node.Node.Chain ); _Chain_Append_unprotected( &heads->Heads.Fifo, - &scheduler_node->Wait.Node.Chain + &scheduler_node->Wait.Priority.Node.Node.Chain ); } static void _Thread_queue_FIFO_do_extract( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *current_or_previous_owner, + Thread_queue_Context *queue_context, + Thread_Control *the_thread ) { Scheduler_Node *scheduler_node; - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); - _Chain_Extract_unprotected( &scheduler_node->Wait.Node.Chain ); + (void) current_or_previous_owner; + (void) queue_context; + + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + + _Chain_Extract_unprotected( &scheduler_node->Wait.Priority.Node.Node.Chain ); } static void _Thread_queue_FIFO_enqueue( - Thread_queue_Queue *queue, - Thread_Control *the_thread, - Thread_queue_Path *path + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - path->update_priority = NULL; - _Thread_queue_Queue_enqueue( queue, the_thread, + queue_context, _Thread_queue_FIFO_do_initialize, _Thread_queue_FIFO_do_enqueue ); } static void _Thread_queue_FIFO_extract( - Thread_queue_Queue *queue, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { _Thread_queue_Queue_extract( queue, queue->heads, + NULL, + queue_context, the_thread, _Thread_queue_FIFO_do_extract ); @@ -177,15 +229,16 @@ static Thread_Control *_Thread_queue_FIFO_first( fifo = &heads->Heads.Fifo; _Assert( !_Chain_Is_empty( fifo ) ); first = _Chain_First( fifo ); - scheduler_node = SCHEDULER_NODE_OF_WAIT_CHAIN_NODE( first ); + scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY_NODE( first ); return _Scheduler_Node_get_owner( scheduler_node ); } static Thread_Control *_Thread_queue_FIFO_surrender( - Thread_queue_Queue *queue, - Thread_queue_Heads *heads, - Thread_Control *previous_owner + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *previous_owner, + Thread_queue_Context *queue_context ) { Thread_Control *first; @@ -194,6 +247,8 @@ static Thread_Control *_Thread_queue_FIFO_surrender( _Thread_queue_Queue_extract( queue, heads, + NULL, + queue_context, first, _Thread_queue_FIFO_do_extract ); @@ -203,165 +258,162 @@ static Thread_Control *_Thread_queue_FIFO_surrender( static Thread_queue_Priority_queue *_Thread_queue_Priority_queue( Thread_queue_Heads *heads, - const Thread_Control *the_thread + const Scheduler_Node *scheduler_node ) { #if defined(RTEMS_SMP) - return &heads->Priority[ - _Scheduler_Get_index( _Scheduler_Get_own( the_thread ) ) - ]; -#else - (void) the_thread; + const Scheduler_Control *scheduler; + scheduler = _Priority_Get_scheduler( &scheduler_node->Wait.Priority ); + return &heads->Priority[ _Scheduler_Get_index( scheduler ) ]; +#else + (void) scheduler_node; return &heads->Heads.Priority; #endif } -static bool _Thread_queue_Priority_less( - const void *left, - const RBTree_Node *right -) -{ - const Priority_Control *the_left; - const Scheduler_Node *scheduler_node; - const Thread_Control *the_right; - - the_left = left; - scheduler_node = SCHEDULER_NODE_OF_WAIT_RBTREE_NODE( right ); - the_right = _Scheduler_Node_get_owner( scheduler_node ); - - return *the_left < _Thread_Get_priority( the_right ); -} - -static void _Thread_queue_Priority_priority_change( +static void _Thread_queue_Priority_priority_actions( Thread_queue_Queue *queue, - Thread_Control *the_thread, - Priority_Control new_priority + Priority_Actions *priority_actions ) { - Thread_queue_Heads *heads; - Thread_queue_Priority_queue *priority_queue; - Scheduler_Node *scheduler_node; + Thread_queue_Heads *heads; + Priority_Aggregation *priority_aggregation; heads = queue->heads; _Assert( heads != NULL ); - priority_queue = _Thread_queue_Priority_queue( heads, the_thread ); - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); + _Assert( !_Priority_Actions_is_empty( priority_actions ) ); + priority_aggregation = _Priority_Actions_move( priority_actions ); - _RBTree_Extract( - &priority_queue->Queue, - &scheduler_node->Wait.Node.RBTree - ); - _RBTree_Insert_inline( - &priority_queue->Queue, - &scheduler_node->Wait.Node.RBTree, - &new_priority, - _Thread_queue_Priority_less - ); + do { + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + + scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY( priority_aggregation ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); + + _Assert( priority_aggregation->Action.type == PRIORITY_ACTION_CHANGE ); + _Priority_Plain_changed( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node + ); + + priority_aggregation = _Priority_Get_next_action( priority_aggregation ); + } while ( _Priority_Actions_is_valid( priority_aggregation ) ); } static void _Thread_queue_Priority_do_initialize( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads ) { - Thread_queue_Priority_queue *priority_queue; Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; - priority_queue = _Thread_queue_Priority_queue( heads, the_thread ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); #if defined(RTEMS_SMP) _Chain_Initialize_one( &heads->Heads.Fifo, &priority_queue->Node ); #endif - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); - - _RBTree_Initialize_node( &scheduler_node->Wait.Node.RBTree ); - _RBTree_Initialize_one( + _Priority_Initialize_one( &priority_queue->Queue, - &scheduler_node->Wait.Node.RBTree + &scheduler_node->Wait.Priority.Node ); } static void _Thread_queue_Priority_do_enqueue( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads ) { - Thread_queue_Priority_queue *priority_queue; Scheduler_Node *scheduler_node; - Priority_Control current_priority; + Thread_queue_Priority_queue *priority_queue; - priority_queue = _Thread_queue_Priority_queue( heads, the_thread ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); #if defined(RTEMS_SMP) - if ( _RBTree_Is_empty( &priority_queue->Queue ) ) { + if ( _Priority_Is_empty( &priority_queue->Queue ) ) { _Chain_Append_unprotected( &heads->Heads.Fifo, &priority_queue->Node ); + _Priority_Initialize_one( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node + ); + return; } #endif - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); - current_priority = _Thread_Get_priority( the_thread ); - - _RBTree_Initialize_node( &scheduler_node->Wait.Node.RBTree ); - _RBTree_Insert_inline( + _Priority_Plain_insert( &priority_queue->Queue, - &scheduler_node->Wait.Node.RBTree, - ¤t_priority, - _Thread_queue_Priority_less + &scheduler_node->Wait.Priority.Node, + _Priority_Get_priority( &scheduler_node->Wait.Priority ) ); } static void _Thread_queue_Priority_do_extract( - Thread_queue_Heads *heads, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *current_or_previous_owner, + Thread_queue_Context *queue_context, + Thread_Control *the_thread ) { - Thread_queue_Priority_queue *priority_queue; Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + + (void) current_or_previous_owner; + (void) queue_context; - priority_queue = _Thread_queue_Priority_queue( heads, the_thread ); - scheduler_node = _Scheduler_Thread_get_own_node( the_thread ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); - _RBTree_Extract( + _Priority_Plain_extract( &priority_queue->Queue, - &scheduler_node->Wait.Node.RBTree + &scheduler_node->Wait.Priority.Node ); #if defined(RTEMS_SMP) _Chain_Extract_unprotected( &priority_queue->Node ); - if ( !_RBTree_Is_empty( &priority_queue->Queue ) ) { + if ( !_Priority_Is_empty( &priority_queue->Queue ) ) { _Chain_Append_unprotected( &heads->Heads.Fifo, &priority_queue->Node ); } #endif } static void _Thread_queue_Priority_enqueue( - Thread_queue_Queue *queue, - Thread_Control *the_thread, - Thread_queue_Path *path + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { - path->update_priority = NULL; - _Thread_queue_Queue_enqueue( queue, the_thread, + queue_context, _Thread_queue_Priority_do_initialize, _Thread_queue_Priority_do_enqueue ); } static void _Thread_queue_Priority_extract( - Thread_queue_Queue *queue, - Thread_Control *the_thread + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context ) { _Thread_queue_Queue_extract( queue, queue->heads, + NULL, + queue_context, the_thread, _Thread_queue_Priority_do_extract ); @@ -372,7 +424,7 @@ static Thread_Control *_Thread_queue_Priority_first( ) { Thread_queue_Priority_queue *priority_queue; - RBTree_Node *first; + Priority_Node *first; Scheduler_Node *scheduler_node; #if defined(RTEMS_SMP) @@ -383,17 +435,18 @@ static Thread_Control *_Thread_queue_Priority_first( priority_queue = &heads->Heads.Priority; #endif - _Assert( !_RBTree_Is_empty( &priority_queue->Queue ) ); - first = _RBTree_Minimum( &priority_queue->Queue ); - scheduler_node = SCHEDULER_NODE_OF_WAIT_RBTREE_NODE( first ); + _Assert( !_Priority_Is_empty( &priority_queue->Queue ) ); + first = _Priority_Get_minimum_node( &priority_queue->Queue ); + scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY_NODE( first ); return _Scheduler_Node_get_owner( scheduler_node ); } static Thread_Control *_Thread_queue_Priority_surrender( - Thread_queue_Queue *queue, - Thread_queue_Heads *heads, - Thread_Control *previous_owner + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *previous_owner, + Thread_queue_Context *queue_context ) { Thread_Control *first; @@ -402,6 +455,8 @@ static Thread_Control *_Thread_queue_Priority_surrender( _Thread_queue_Queue_extract( queue, heads, + NULL, + queue_context, first, _Thread_queue_Priority_do_extract ); @@ -409,104 +464,524 @@ static Thread_Control *_Thread_queue_Priority_surrender( return first; } -static void _Thread_queue_Priority_inherit_enqueue( - Thread_queue_Queue *queue, - Thread_Control *the_thread, - Thread_queue_Path *path +static void _Thread_queue_Priority_inherit_do_actions_change( + Priority_Aggregation *priority_aggregation, + bool prepend_it, + Priority_Actions *priority_actions, + void *arg ) { - Thread_queue_Heads *heads; - Thread_Control *owner; - Priority_Control priority; + Thread_queue_Priority_queue *priority_queue; + Scheduler_Node *scheduler_node; - heads = _Thread_queue_Queue_enqueue( - queue, - the_thread, - _Thread_queue_Priority_do_initialize, - _Thread_queue_Priority_do_enqueue + priority_queue = THREAD_QUEUE_PRIORITY_QUEUE_OF_PRIORITY_AGGREGATION( + priority_aggregation ); + scheduler_node = priority_queue->scheduler_node; - owner = queue->owner; + _Priority_Set_action( + &scheduler_node->Wait.Priority, + &priority_aggregation->Node, + PRIORITY_ACTION_CHANGE + ); + _Priority_Actions_add( priority_actions, &scheduler_node->Wait.Priority ); +} -#if defined(RTEMS_SMP) - if ( _Chain_Has_only_one_node( &heads->Heads.Fifo ) ) { - priority = the_thread->current_priority; - } else { - priority = _Scheduler_Map_priority( - _Scheduler_Get_own( the_thread ), - PRIORITY_PSEUDO_ISR +static void _Thread_queue_Priority_inherit_priority_actions( + Thread_queue_Queue *queue, + Priority_Actions *priority_actions +) +{ + Thread_queue_Heads *heads; + Priority_Aggregation *priority_aggregation; + + heads = queue->heads; + _Assert( heads != NULL ); + + _Assert( !_Priority_Actions_is_empty( priority_actions ) ); + priority_aggregation = _Priority_Actions_move( priority_actions ); + + do { + Priority_Aggregation *next_aggregation; + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + + next_aggregation = _Priority_Get_next_action( priority_aggregation ); + + scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY( priority_aggregation ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); + + _Assert( priority_aggregation->Action.type == PRIORITY_ACTION_CHANGE ); + _Priority_Changed( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node, + false, + priority_actions, + _Thread_queue_Priority_inherit_do_actions_change, + NULL ); + + priority_aggregation = next_aggregation; + } while ( _Priority_Actions_is_valid( priority_aggregation ) ); +} + +static void _Thread_queue_Boost_priority( + Thread_queue_Heads *heads, + Thread_Control *the_thread, + Thread_Control *owner, + Thread_queue_Context *queue_context +) +{ +#if defined(RTEMS_SMP) + const Scheduler_Control *scheduler; + const Scheduler_Control *scheduler_of_owner; + Scheduler_Node *scheduler_node_of_owner; + Priority_Control boost_priority; + + if ( _Priority_Node_is_active( &heads->Boost_priority ) ) { + return; } + + scheduler = _Scheduler_Get_own( the_thread ); + scheduler_of_owner = _Scheduler_Get_own( owner ); + + if ( scheduler == scheduler_of_owner ) { + return; + } + + scheduler_node_of_owner = _Thread_Scheduler_get_own_node( owner ); + + boost_priority = _Scheduler_Map_priority( + scheduler_of_owner, + PRIORITY_PSEUDO_ISR + ); + + _Priority_Node_initialize( &heads->Boost_priority, boost_priority ); + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &scheduler_node_of_owner->Wait.Priority, + &heads->Boost_priority, + PRIORITY_ACTION_ADD + ); + _Thread_Priority_perform_actions( owner, queue_context ); #else (void) heads; + (void) the_thread; + (void) owner; + (void) queue_context; +#endif +} + +static void _Thread_queue_Priority_inherit_do_initialize( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads +) +{ + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + Thread_Control *owner; + Scheduler_Node *scheduler_node_of_owner; + + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); - priority = the_thread->current_priority; +#if defined(RTEMS_SMP) + _Chain_Initialize_one( &heads->Heads.Fifo, &priority_queue->Node ); #endif - if ( priority < owner->current_priority ) { - path->update_priority = owner; + _Priority_Initialize_one( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node + ); - owner->priority_restore_hint = true; - _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); + owner = queue->owner; + scheduler_node_of_owner = _Thread_Scheduler_get_own_node( owner ); + priority_queue->scheduler_node = scheduler_node_of_owner; + + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &scheduler_node_of_owner->Wait.Priority, + &priority_queue->Queue.Node, + PRIORITY_ACTION_ADD + ); + _Thread_Priority_perform_actions( owner, queue_context ); + _Thread_queue_Boost_priority( heads, the_thread, owner, queue_context ); +} + +static void _Thread_queue_Priority_inherit_do_enqueue_change( + Priority_Aggregation *priority_aggregation, + bool prepend_it, + Priority_Actions *priority_actions, + void *arg +) +{ + Thread_queue_Queue *queue; + Thread_Control *owner; + Scheduler_Node *scheduler_node_of_owner; + Thread_queue_Context *queue_context; + + queue = arg; + owner = queue->owner; + scheduler_node_of_owner = _Thread_Scheduler_get_own_node( owner ); + queue_context = THREAD_QUEUE_CONTEXT_OF_PRIORITY_ACTIONS( priority_actions ); + + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &scheduler_node_of_owner->Wait.Priority, + &priority_aggregation->Node, + PRIORITY_ACTION_CHANGE + ); + _Thread_Priority_perform_actions( owner, queue_context ); +} + +static void _Thread_queue_Priority_inherit_do_enqueue( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context, + Thread_queue_Heads *heads +) +{ + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; - _Scheduler_Thread_set_priority( owner, priority, false ); + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); - ( *owner->Wait.operations->priority_change )( - owner->Wait.queue, +#if defined(RTEMS_SMP) + if ( _Priority_Is_empty( &priority_queue->Queue ) ) { + Thread_Control *owner; + Scheduler_Node *scheduler_node_of_owner; + + _Chain_Append_unprotected( &heads->Heads.Fifo, &priority_queue->Node ); + _Priority_Initialize_one( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node + ); + + owner = queue->owner; + scheduler_node_of_owner = _Thread_Scheduler_get_own_node( owner ); + priority_queue->scheduler_node = scheduler_node_of_owner; + + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &scheduler_node_of_owner->Wait.Priority, + &priority_queue->Queue.Node, + PRIORITY_ACTION_ADD + ); + _Thread_Priority_perform_actions( owner, queue_context ); + _Thread_queue_Boost_priority( heads, the_thread, owner, queue_context ); + return; + } +#endif + + _Priority_Non_empty_insert( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node, + &queue_context->Priority.Actions, + _Thread_queue_Priority_inherit_do_enqueue_change, + queue + ); + _Thread_queue_Boost_priority( + heads, + the_thread, + queue->owner, + queue_context + ); +} + +static void _Thread_queue_Priority_inherit_enqueue( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Thread_queue_Queue_enqueue( + queue, + the_thread, + queue_context, + _Thread_queue_Priority_inherit_do_initialize, + _Thread_queue_Priority_inherit_do_enqueue + ); +} + +static void _Thread_queue_Priority_inherit_do_extract_action( + Priority_Actions *priority_actions, + Thread_Control *owner, + Priority_Node *priority_action_node, + Priority_Action_type priority_action_type +) +{ + Thread_queue_Context *queue_context; + Scheduler_Node *scheduler_node_of_owner; + + queue_context = THREAD_QUEUE_CONTEXT_OF_PRIORITY_ACTIONS( priority_actions ); + scheduler_node_of_owner = _Thread_Scheduler_get_own_node( owner ); + + _Priority_Actions_initialize_one( + &queue_context->Priority.Actions, + &scheduler_node_of_owner->Wait.Priority, + priority_action_node, + priority_action_type + ); + _Thread_Priority_perform_actions( owner, queue_context ); +} + +static void _Thread_queue_Priority_inherit_do_extract_remove( + Priority_Aggregation *priority_aggregation, + Priority_Actions *priority_actions, + void *arg +) +{ + _Thread_queue_Priority_inherit_do_extract_action( + priority_actions, + arg, + &priority_aggregation->Node, + PRIORITY_ACTION_REMOVE + ); +} + +static void _Thread_queue_Priority_inherit_do_extract_change( + Priority_Aggregation *priority_aggregation, + bool prepend_it, + Priority_Actions *priority_actions, + void *arg +) +{ + _Thread_queue_Priority_inherit_do_extract_action( + priority_actions, + arg, + &priority_aggregation->Node, + PRIORITY_ACTION_CHANGE + ); +} + +static void _Thread_queue_Priority_inherit_do_extract( + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *owner, + Thread_queue_Context *queue_context, + Thread_Control *the_thread +) +{ + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); + + _Thread_queue_Path_acquire_critical( queue, the_thread, queue_context ); + + _Priority_Extract( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node, + &queue_context->Priority.Actions, + _Thread_queue_Priority_inherit_do_extract_remove, + _Thread_queue_Priority_inherit_do_extract_change, + owner + ); + +#if defined(RTEMS_SMP) + _Chain_Extract_unprotected( &priority_queue->Node ); + + if ( !_Priority_Is_empty( &priority_queue->Queue ) ) { + _Chain_Append_unprotected( &heads->Heads.Fifo, &priority_queue->Node ); + } + + if ( + _Priority_Node_is_active( &heads->Boost_priority ) + && ( _Chain_Is_empty( &heads->Heads.Fifo ) + || _Chain_Has_only_one_node( &heads->Heads.Fifo ) ) + ) { + _Thread_queue_Priority_inherit_do_extract_action( + &queue_context->Priority.Actions, owner, - priority + &heads->Boost_priority, + PRIORITY_ACTION_REMOVE ); - } else { - path->update_priority = NULL; + _Priority_Node_set_inactive( &heads->Boost_priority ); } +#endif + + _Thread_queue_Path_release_critical( queue_context ); } -static void _Thread_queue_Boost_priority( - Thread_queue_Heads *heads, - Thread_Control *the_thread +static void _Thread_queue_Priority_inherit_extract( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + _Thread_queue_Queue_extract( + queue, + queue->heads, + queue->owner, + queue_context, + the_thread, + _Thread_queue_Priority_inherit_do_extract + ); +} + +static void _Thread_queue_Priority_inherit_do_surrender_change( + Priority_Aggregation *priority_aggregation, + bool prepend_it, + Priority_Actions *priority_actions, + void *arg ) { + _Thread_queue_Context_add_priority_update( + THREAD_QUEUE_CONTEXT_OF_PRIORITY_ACTIONS( priority_actions ), + arg + ); + _Scheduler_Node_set_priority( + SCHEDULER_NODE_OF_WAIT_PRIORITY( priority_aggregation ), + _Priority_Get_priority( priority_aggregation ), + prepend_it + ); +} + +static void _Thread_queue_Priority_add( + Thread_Control *the_thread, + Priority_Aggregation *priority_aggregation, + Priority_Node *priority_node, + Thread_queue_Context *queue_context +) +{ + _Priority_Non_empty_insert( + priority_aggregation, + priority_node, + &queue_context->Priority.Actions, + _Thread_queue_Priority_inherit_do_surrender_change, + the_thread + ); +} + +static void _Thread_queue_Priority_remove( + Thread_Control *the_thread, + Scheduler_Node *scheduler_node, + Priority_Node *priority_node, + Thread_queue_Context *queue_context +) +{ + _Priority_Extract_non_empty( + &scheduler_node->Wait.Priority, + priority_node, + &queue_context->Priority.Actions, + _Thread_queue_Priority_inherit_do_surrender_change, + the_thread + ); +} + +static void _Thread_queue_Priority_inherit_do_surrender( + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *previous_owner, + Thread_queue_Context *queue_context, + Thread_Control *the_thread +) +{ + Scheduler_Node *scheduler_node; + Thread_queue_Priority_queue *priority_queue; + ISR_lock_Context lock_context; + + scheduler_node = _Thread_Scheduler_get_own_node( the_thread ); + priority_queue = _Thread_queue_Priority_queue( heads, scheduler_node ); + + _Thread_Wait_acquire_default_critical( previous_owner, &lock_context ); + #if defined(RTEMS_SMP) - if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) { - const Scheduler_Control *scheduler; - Priority_Control boost_priority; + if ( _Priority_Node_is_active( &heads->Boost_priority ) ) { + _Thread_queue_Priority_remove( + previous_owner, + _Thread_Scheduler_get_own_node( previous_owner ), + &heads->Boost_priority, + queue_context + ); + _Priority_Node_set_inactive( &heads->Boost_priority ); + } +#endif - the_thread->priority_restore_hint = true; - _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); + _Thread_queue_Priority_remove( + previous_owner, + priority_queue->scheduler_node, + &priority_queue->Queue.Node, + queue_context + ); - scheduler = _Scheduler_Get_own( the_thread ); - boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR ); + _Thread_Wait_release_default_critical( previous_owner, &lock_context ); - _Scheduler_Thread_set_priority( the_thread, boost_priority, false ); + _Priority_Extract( + &priority_queue->Queue, + &scheduler_node->Wait.Priority.Node, + NULL, + _Priority_Remove_nothing, + _Priority_Change_nothing, + previous_owner + ); + + if ( !_Priority_Is_empty( &priority_queue->Queue ) ) { + priority_queue->scheduler_node = scheduler_node; + _Thread_queue_Priority_add( + the_thread, + &scheduler_node->Wait.Priority, + &priority_queue->Queue.Node, + queue_context + ); + } + +#if defined(RTEMS_SMP) + _Chain_Extract_unprotected( &priority_queue->Node ); + + if ( !_Priority_Is_empty( &priority_queue->Queue ) ) { + _Chain_Append_unprotected( &heads->Heads.Fifo, &priority_queue->Node ); + } + + if ( + !_Chain_Is_empty( &heads->Heads.Fifo) + && !_Chain_Has_only_one_node( &heads->Heads.Fifo) + ) { + Priority_Control boost_priority; + + boost_priority = _Scheduler_Map_priority( + _Scheduler_Get_own( the_thread ), + PRIORITY_PSEUDO_ISR + ); + _Priority_Node_initialize( &heads->Boost_priority, boost_priority ); + _Thread_queue_Priority_add( + the_thread, + &scheduler_node->Wait.Priority, + &heads->Boost_priority, + queue_context + ); } -#else - (void) heads; - (void) the_thread; #endif } static Thread_Control *_Thread_queue_Priority_inherit_surrender( - Thread_queue_Queue *queue, - Thread_queue_Heads *heads, - Thread_Control *previous_owner + Thread_queue_Queue *queue, + Thread_queue_Heads *heads, + Thread_Control *previous_owner, + Thread_queue_Context *queue_context ) { Thread_Control *first; first = _Thread_queue_Priority_first( heads ); - _Thread_queue_Boost_priority( heads, first ); _Thread_queue_Queue_extract( queue, heads, + previous_owner, + queue_context, first, - _Thread_queue_Priority_do_extract + _Thread_queue_Priority_inherit_do_surrender ); return first; } const Thread_queue_Operations _Thread_queue_Operations_default = { - .priority_change = _Thread_queue_Do_nothing_priority_change, + .priority_actions = _Thread_queue_Do_nothing_priority_actions, .extract = _Thread_queue_Do_nothing_extract /* * The default operations are only used in _Thread_Change_priority() and @@ -516,7 +991,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = { }; const Thread_queue_Operations _Thread_queue_Operations_FIFO = { - .priority_change = _Thread_queue_Do_nothing_priority_change, + .priority_actions = _Thread_queue_Do_nothing_priority_actions, .enqueue = _Thread_queue_FIFO_enqueue, .extract = _Thread_queue_FIFO_extract, .surrender = _Thread_queue_FIFO_surrender, @@ -524,7 +999,7 @@ const Thread_queue_Operations _Thread_queue_Operations_FIFO = { }; const Thread_queue_Operations _Thread_queue_Operations_priority = { - .priority_change = _Thread_queue_Priority_priority_change, + .priority_actions = _Thread_queue_Priority_priority_actions, .enqueue = _Thread_queue_Priority_enqueue, .extract = _Thread_queue_Priority_extract, .surrender = _Thread_queue_Priority_surrender, @@ -532,9 +1007,9 @@ const Thread_queue_Operations _Thread_queue_Operations_priority = { }; const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = { - .priority_change = _Thread_queue_Priority_priority_change, + .priority_actions = _Thread_queue_Priority_inherit_priority_actions, .enqueue = _Thread_queue_Priority_inherit_enqueue, - .extract = _Thread_queue_Priority_extract, + .extract = _Thread_queue_Priority_inherit_extract, .surrender = _Thread_queue_Priority_inherit_surrender, .first = _Thread_queue_Priority_first }; diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c index 149882795c..69534001aa 100644 --- a/cpukit/score/src/threadrestart.c +++ b/cpukit/score/src/threadrestart.c @@ -50,40 +50,28 @@ static Thread_Zombie_control _Thread_Zombies = { .Lock = ISR_LOCK_INITIALIZER( "thread zombies" ) }; -static bool _Thread_Raise_real_priority_filter( +static void _Thread_Raise_real_priority( Thread_Control *the_thread, - Priority_Control *new_priority_ptr, - void *arg + Priority_Control priority ) { - Priority_Control real_priority; - Priority_Control new_priority; - Priority_Control current_priority; - - real_priority = the_thread->real_priority; - new_priority = *new_priority_ptr; - current_priority = _Thread_Get_priority( the_thread ); - - new_priority = _Thread_Priority_highest( real_priority, new_priority ); - *new_priority_ptr = new_priority; + Thread_queue_Context queue_context; - the_thread->real_priority = new_priority; + _Thread_queue_Context_clear_priority_updates( &queue_context ); + _Thread_Wait_acquire( the_thread, &queue_context ); - return _Thread_Priority_less_than( current_priority, new_priority ); -} + if ( priority < the_thread->Real_priority.priority ) { + _Thread_Priority_change( + the_thread, + &the_thread->Real_priority, + priority, + false, + &queue_context + ); + } -static void _Thread_Raise_real_priority( - Thread_Control *the_thread, - Priority_Control priority -) -{ - _Thread_Change_priority( - the_thread, - priority, - NULL, - _Thread_Raise_real_priority_filter, - false - ); + _Thread_Wait_release( the_thread, &queue_context ); + _Thread_Priority_update( &queue_context ); } typedef struct { @@ -182,7 +170,7 @@ static void _Thread_Free( Thread_Control *the_thread ) _ISR_lock_Destroy( &the_thread->Keys.Lock ); _Scheduler_Node_destroy( _Scheduler_Get( the_thread ), - _Scheduler_Thread_get_own_node( the_thread ) + _Thread_Scheduler_get_own_node( the_thread ) ); _ISR_lock_Destroy( &the_thread->Timer.Lock ); @@ -622,8 +610,8 @@ void _Thread_Restart_self( ISR_lock_Context *lock_context ) { - Per_CPU_Control *cpu_self; - Priority_Control unused; + Per_CPU_Control *cpu_self; + Thread_queue_Context queue_context; _Assert( _Watchdog_Get_state( &executing->Timer.Watchdog ) == WATCHDOG_INACTIVE @@ -646,13 +634,18 @@ void _Thread_Restart_self( cpu_self = _Thread_Dispatch_disable_critical( lock_context ); _Thread_State_release( executing, lock_context ); - _Thread_Set_priority( + _Thread_queue_Context_clear_priority_updates( &queue_context ); + _Thread_Wait_acquire_default( executing, lock_context ); + _Thread_Priority_change( executing, + &executing->Real_priority, executing->Start.initial_priority, - &unused, - true + false, + &queue_context ); + _Thread_Wait_release_default( executing, lock_context ); + _Thread_Priority_update( &queue_context ); _Thread_Dispatch_enable( cpu_self ); RTEMS_UNREACHABLE(); } diff --git a/cpukit/score/src/threadsetpriority.c b/cpukit/score/src/threadsetpriority.c deleted file mode 100644 index d6b8319970..0000000000 --- a/cpukit/score/src/threadsetpriority.c +++ /dev/null @@ -1,59 +0,0 @@ -/** - * @file - * - * @brief Set Thread Priority - * @ingroup ScoreThread - */ - -/* - * COPYRIGHT (c) 1989-2011. - * On-Line Applications Research Corporation (OAR). - * - * The license and distribution terms for this file may be - * found in the file LICENSE in this distribution or at - * http://www.rtems.org/license/LICENSE. - */ - -#if HAVE_CONFIG_H -#include "config.h" -#endif - -#include <rtems/score/threadimpl.h> - -static bool _Thread_Set_priority_filter( - Thread_Control *the_thread, - Priority_Control *new_priority_ptr, - void *arg -) -{ - Priority_Control current_priority; - Priority_Control new_priority; - Priority_Control *old_priority_ptr; - - current_priority = _Thread_Get_priority( the_thread ); - new_priority = *new_priority_ptr; - - old_priority_ptr = arg; - *old_priority_ptr = current_priority; - - the_thread->real_priority = new_priority; - - return _Thread_Priority_less_than( current_priority, new_priority ) - || !_Thread_Owns_resources( the_thread ); -} - -void _Thread_Set_priority( - Thread_Control *the_thread, - Priority_Control new_priority, - Priority_Control *old_priority, - bool prepend_it -) -{ - _Thread_Change_priority( - the_thread, - new_priority, - old_priority, - _Thread_Set_priority_filter, - prepend_it - ); -} diff --git a/cpukit/score/src/threadtimeout.c b/cpukit/score/src/threadtimeout.c index b6b6cc4ac2..8297c55c4b 100644 --- a/cpukit/score/src/threadtimeout.c +++ b/cpukit/score/src/threadtimeout.c @@ -30,6 +30,8 @@ void _Thread_Timeout( Watchdog_Control *watchdog ) bool unblock; the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog ); + + _Thread_queue_Context_clear_priority_updates( &queue_context ); _Thread_Wait_acquire( the_thread, &queue_context ); wait_flags = _Thread_Wait_flags_get( the_thread ); @@ -66,6 +68,7 @@ void _Thread_Timeout( Watchdog_Control *watchdog ) } _Thread_Wait_release( the_thread, &queue_context ); + _Thread_Priority_update( &queue_context ); if ( unblock ) { _Thread_Wait_tranquilize( the_thread ); |