diff options
author | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2015-05-05 13:05:54 +0200 |
---|---|---|
committer | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2015-05-19 12:00:47 +0200 |
commit | 900d337f960cb7cc53f5c93c29a503e5ced2c31f (patch) | |
tree | 1d1f49724e5cfcef1974d5dc4251486b0fddd2ef /cpukit/score/src | |
parent | score: Fine grained locking for mutexes (diff) | |
download | rtems-900d337f960cb7cc53f5c93c29a503e5ced2c31f.tar.bz2 |
score: Rework _Thread_Change_priority()
Move the writes to Thread_Control::current_priority and
Thread_Control::real_priority into _Thread_Change_priority() under the
protection of the thread lock. Add a filter function to
_Thread_Change_priority() to enable specialized variants.
Avoid race conditions during a thread priority restore with the new
Thread_Control::priority_restore_hint for an important average case
optimizations used by priority inheritance mutexes.
Update #2273.
Diffstat (limited to 'cpukit/score/src')
-rw-r--r-- | cpukit/score/src/coremutex.c | 19 | ||||
-rw-r--r-- | cpukit/score/src/coremutexseize.c | 8 | ||||
-rw-r--r-- | cpukit/score/src/coremutexsurrender.c | 31 | ||||
-rw-r--r-- | cpukit/score/src/schedulercbs.c | 6 | ||||
-rw-r--r-- | cpukit/score/src/schedulercbsreleasejob.c | 4 | ||||
-rw-r--r-- | cpukit/score/src/scheduleredfreleasejob.c | 4 | ||||
-rw-r--r-- | cpukit/score/src/thread.c | 1 | ||||
-rw-r--r-- | cpukit/score/src/threadchangepriority.c | 71 | ||||
-rw-r--r-- | cpukit/score/src/threadinitialize.c | 3 | ||||
-rw-r--r-- | cpukit/score/src/threadrestart.c | 46 | ||||
-rw-r--r-- | cpukit/score/src/threadsetpriority.c | 37 |
11 files changed, 171 insertions, 59 deletions
diff --git a/cpukit/score/src/coremutex.c b/cpukit/score/src/coremutex.c index 58c6c42b06..6bb535a8ec 100644 --- a/cpukit/score/src/coremutex.c +++ b/cpukit/score/src/coremutex.c @@ -50,15 +50,20 @@ CORE_mutex_Status _CORE_mutex_Initialize( Priority_Control ceiling = the_mutex->Attributes.priority_ceiling; Per_CPU_Control *cpu_self; - /* - * The mutex initialization is only protected by the allocator lock in - * general. Disable thread dispatching before the priority check to - * prevent interference with priority inheritance. - */ + /* The mutex initialization is only protected by the allocator lock */ cpu_self = _Thread_Dispatch_disable(); + /* + * The test to check for a ceiling violation is a bit arbitrary. In case + * this thread is the owner of a priority inheritance mutex, then it may + * get a higher priority later or anytime on SMP configurations. + */ if ( is_priority_ceiling && executing->current_priority < ceiling ) { - _Thread_Enable_dispatch(); + /* + * There is no need to undo the previous work since this error aborts + * the object creation. + */ + _Thread_Dispatch_enable( cpu_self ); return CORE_MUTEX_STATUS_CEILING_VIOLATED; } @@ -71,7 +76,7 @@ CORE_mutex_Status _CORE_mutex_Initialize( executing->resource_count++; if ( is_priority_ceiling ) { - _Thread_Change_priority( executing, ceiling, false ); + _Thread_Raise_priority( executing, ceiling ); } _Thread_Dispatch_enable( cpu_self ); diff --git a/cpukit/score/src/coremutexseize.c b/cpukit/score/src/coremutexseize.c index 94c480d054..d5f85c5fad 100644 --- a/cpukit/score/src/coremutexseize.c +++ b/cpukit/score/src/coremutexseize.c @@ -21,7 +21,6 @@ #include <rtems/system.h> #include <rtems/score/isr.h> #include <rtems/score/coremuteximpl.h> -#include <rtems/score/schedulerimpl.h> #include <rtems/score/statesimpl.h> #include <rtems/score/thread.h> @@ -76,12 +75,7 @@ void _CORE_mutex_Seize_interrupt_blocking( _Thread_queue_Release( &the_mutex->Wait_queue, lock_context ); #endif - _Scheduler_Change_priority_if_higher( - _Scheduler_Get( holder ), - holder, - executing->current_priority, - false - ); + _Thread_Raise_priority( holder, executing->current_priority ); #if !defined(RTEMS_SMP) _Thread_queue_Acquire( &the_mutex->Wait_queue, lock_context ); diff --git a/cpukit/score/src/coremutexsurrender.c b/cpukit/score/src/coremutexsurrender.c index 3f0cd8619c..d5dde1e8e6 100644 --- a/cpukit/score/src/coremutexsurrender.c +++ b/cpukit/score/src/coremutexsurrender.c @@ -207,14 +207,10 @@ CORE_mutex_Status _CORE_mutex_Surrender( case CORE_MUTEX_DISCIPLINES_PRIORITY_CEILING: _CORE_mutex_Push_priority( the_mutex, the_thread ); the_thread->resource_count++; - if (the_mutex->Attributes.priority_ceiling < - the_thread->current_priority){ - _Thread_Change_priority( - the_thread, - the_mutex->Attributes.priority_ceiling, - false - ); - } + _Thread_Raise_priority( + the_thread, + the_mutex->Attributes.priority_ceiling + ); break; } } @@ -246,13 +242,20 @@ CORE_mutex_Status _CORE_mutex_Surrender( * inherited priority must be lowered if this is the last * mutex (i.e. resource) this task has. */ - if ( !_Thread_Owns_resources( holder ) && - holder->real_priority != holder->current_priority ) { - Per_CPU_Control *cpu_self; + if ( !_Thread_Owns_resources( holder ) ) { + /* + * Ensure that the holder resource count is visible to all other processors + * and that we read the latest priority restore hint. + */ + _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); - cpu_self = _Thread_Dispatch_disable(); - _Thread_Change_priority( holder, holder->real_priority, true ); - _Thread_Dispatch_enable( cpu_self ); + if ( holder->priority_restore_hint ) { + Per_CPU_Control *cpu_self; + + cpu_self = _Thread_Dispatch_disable(); + _Thread_Restore_priority( holder ); + _Thread_Dispatch_enable( cpu_self ); + } } return CORE_MUTEX_STATUS_SUCCESSFUL; diff --git a/cpukit/score/src/schedulercbs.c b/cpukit/score/src/schedulercbs.c index 44221cdbdf..98ec0eb29e 100644 --- a/cpukit/score/src/schedulercbs.c +++ b/cpukit/score/src/schedulercbs.c @@ -27,15 +27,13 @@ void _Scheduler_CBS_Budget_callout( ) { Priority_Control new_priority; + Priority_Control unused; Scheduler_CBS_Node *node; Scheduler_CBS_Server_id server_id; /* Put violating task to background until the end of period. */ new_priority = the_thread->Start.initial_priority; - if ( the_thread->real_priority != new_priority ) - the_thread->real_priority = new_priority; - if ( the_thread->current_priority != new_priority ) - _Thread_Change_priority(the_thread, new_priority, true); + _Thread_Set_priority( the_thread, new_priority, &unused, true ); /* Invoke callback function if any. */ node = _Scheduler_CBS_Thread_get_node( the_thread ); diff --git a/cpukit/score/src/schedulercbsreleasejob.c b/cpukit/score/src/schedulercbsreleasejob.c index 36a31551cf..a9f8e33201 100644 --- a/cpukit/score/src/schedulercbsreleasejob.c +++ b/cpukit/score/src/schedulercbsreleasejob.c @@ -32,6 +32,7 @@ void _Scheduler_CBS_Release_job( Scheduler_CBS_Node *node = _Scheduler_CBS_Thread_get_node( the_thread ); Scheduler_CBS_Server *serv_info = node->cbs_server; Priority_Control new_priority; + Priority_Control unused; if (deadline) { /* Initializing or shifting deadline. */ @@ -51,6 +52,5 @@ void _Scheduler_CBS_Release_job( if (serv_info) the_thread->cpu_time_budget = serv_info->parameters.budget; - the_thread->real_priority = new_priority; - _Thread_Change_priority(the_thread, new_priority, true); + _Thread_Set_priority( the_thread, new_priority, &unused, true ); } diff --git a/cpukit/score/src/scheduleredfreleasejob.c b/cpukit/score/src/scheduleredfreleasejob.c index 6c1b642890..2c3db65b64 100644 --- a/cpukit/score/src/scheduleredfreleasejob.c +++ b/cpukit/score/src/scheduleredfreleasejob.c @@ -29,6 +29,7 @@ void _Scheduler_EDF_Release_job( ) { Priority_Control new_priority; + Priority_Control unused; (void) scheduler; @@ -42,6 +43,5 @@ void _Scheduler_EDF_Release_job( new_priority = the_thread->Start.initial_priority; } - the_thread->real_priority = new_priority; - _Thread_Change_priority(the_thread, new_priority, true); + _Thread_Set_priority( the_thread, new_priority, &unused, true ); } diff --git a/cpukit/score/src/thread.c b/cpukit/score/src/thread.c index 8a34ced684..ef9788ca41 100644 --- a/cpukit/score/src/thread.c +++ b/cpukit/score/src/thread.c @@ -32,6 +32,7 @@ THREAD_OFFSET_ASSERT( current_state ); THREAD_OFFSET_ASSERT( current_priority ); THREAD_OFFSET_ASSERT( real_priority ); THREAD_OFFSET_ASSERT( priority_generation ); +THREAD_OFFSET_ASSERT( priority_restore_hint ); THREAD_OFFSET_ASSERT( resource_count ); THREAD_OFFSET_ASSERT( Wait ); THREAD_OFFSET_ASSERT( Timer ); diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c index 3223544fe3..8f5d14f412 100644 --- a/cpukit/score/src/threadchangepriority.c +++ b/cpukit/score/src/threadchangepriority.c @@ -21,12 +21,13 @@ #include <rtems/score/threadimpl.h> #include <rtems/score/schedulerimpl.h> -#include <rtems/score/threadqimpl.h> void _Thread_Change_priority( - Thread_Control *the_thread, - Priority_Control new_priority, - bool prepend_it + Thread_Control *the_thread, + Priority_Control new_priority, + void *arg, + Thread_Change_priority_filter filter, + bool prepend_it ) { ISR_lock_Context lock_context; @@ -35,10 +36,20 @@ void _Thread_Change_priority( lock = _Thread_Lock_acquire( the_thread, &lock_context ); /* + * For simplicity set the priority restore hint unconditionally since this is + * an average case optimization. Otherwise complicated atomic operations + * would be necessary. Synchronize with a potential read of the resource + * count in the filter function. See also _CORE_mutex_Surrender(), + * _Thread_Set_priority_filter() and _Thread_Restore_priority_filter(). + */ + the_thread->priority_restore_hint = true; + _Atomic_Fence( ATOMIC_ORDER_ACQ_REL ); + + /* * Do not bother recomputing all the priority related information if * we are not REALLY changing priority. */ - if ( the_thread->current_priority != new_priority ) { + if ( ( *filter )( the_thread, &new_priority, arg ) ) { uint32_t my_generation; my_generation = the_thread->priority_generation + 1; @@ -72,3 +83,53 @@ void _Thread_Change_priority( _Thread_Lock_release( lock, &lock_context ); } } + +static bool _Thread_Raise_priority_filter( + Thread_Control *the_thread, + Priority_Control *new_priority, + void *arg +) +{ + return _Thread_Priority_less_than( + the_thread->current_priority, + *new_priority + ); +} + +void _Thread_Raise_priority( + Thread_Control *the_thread, + Priority_Control new_priority +) +{ + _Thread_Change_priority( + the_thread, + new_priority, + NULL, + _Thread_Raise_priority_filter, + false + ); +} + +static bool _Thread_Restore_priority_filter( + Thread_Control *the_thread, + Priority_Control *new_priority, + void *arg +) +{ + *new_priority = the_thread->real_priority; + + the_thread->priority_restore_hint = false; + + return *new_priority != the_thread->current_priority; +} + +void _Thread_Restore_priority( Thread_Control *the_thread ) +{ + _Thread_Change_priority( + the_thread, + 0, + NULL, + _Thread_Restore_priority_filter, + true + ); +} diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c index 9f5df257aa..2133d7485b 100644 --- a/cpukit/score/src/threadinitialize.c +++ b/cpukit/score/src/threadinitialize.c @@ -201,6 +201,7 @@ bool _Thread_Initialize( the_thread->Wait.queue = NULL; the_thread->Wait.operations = &_Thread_queue_Operations_default; the_thread->resource_count = 0; + the_thread->current_priority = priority; the_thread->real_priority = priority; the_thread->priority_generation = 0; the_thread->Start.initial_priority = priority; @@ -210,7 +211,7 @@ bool _Thread_Initialize( _Scheduler_Node_initialize( scheduler, the_thread ); scheduler_node_initialized = true; - _Thread_Set_priority( the_thread, priority ); + _Scheduler_Update_priority( the_thread, priority ); /* * Initialize the CPU usage statistics diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c index 8dea518e10..b98b6388f8 100644 --- a/cpukit/score/src/threadrestart.c +++ b/cpukit/score/src/threadrestart.c @@ -42,6 +42,28 @@ static Thread_Zombie_control _Thread_Zombies = { .Lock = ISR_LOCK_INITIALIZER( "thread zombies" ) }; +static bool _Thread_Raise_real_priority_filter( + Thread_Control *the_thread, + Priority_Control *new_priority_ptr, + void *arg +) +{ + Priority_Control real_priority; + Priority_Control new_priority; + Priority_Control current_priority; + + real_priority = the_thread->real_priority; + new_priority = *new_priority_ptr; + current_priority = the_thread->current_priority; + + new_priority = _Thread_Priority_highest( real_priority, new_priority ); + *new_priority_ptr = new_priority; + + the_thread->real_priority = new_priority; + + return _Thread_Priority_less_than( current_priority, new_priority ); +} + static void _Thread_Make_zombie( Thread_Control *the_thread ) { ISR_lock_Context lock_context; @@ -231,12 +253,17 @@ static void _Thread_Start_life_change( the_thread->is_preemptible = the_thread->Start.is_preemptible; the_thread->budget_algorithm = the_thread->Start.budget_algorithm; the_thread->budget_callout = the_thread->Start.budget_callout; - the_thread->real_priority = priority; _Thread_Set_state( the_thread, STATES_RESTARTING ); _Thread_queue_Extract_with_proxy( the_thread ); _Watchdog_Remove_ticks( &the_thread->Timer ); - _Scheduler_Set_priority_if_higher( scheduler, the_thread, priority ); + _Thread_Change_priority( + the_thread, + priority, + NULL, + _Thread_Raise_real_priority_filter, + false + ); _Thread_Add_post_switch_action( the_thread, &the_thread->Life.Action ); _Thread_Ready( the_thread ); } @@ -260,9 +287,9 @@ static void _Thread_Request_life_change( scheduler = _Scheduler_Get( the_thread ); if ( the_thread == executing ) { - executing->real_priority = priority; + Priority_Control unused; - _Scheduler_Set_priority_if_higher( scheduler, the_thread, priority ); + _Thread_Set_priority( the_thread, priority, &unused, true ); _Thread_Start_life_change_for_executing( executing ); } else if ( previous_life_state == THREAD_LIFE_NORMAL ) { _Thread_Start_life_change( the_thread, scheduler, priority ); @@ -270,16 +297,11 @@ static void _Thread_Request_life_change( _Thread_Clear_state( the_thread, STATES_SUSPENDED ); if ( _Thread_Is_life_terminating( additional_life_state ) ) { - the_thread->real_priority = _Scheduler_Highest_priority_of_two( - scheduler, - the_thread->real_priority, - priority - ); - - _Scheduler_Change_priority_if_higher( - scheduler, + _Thread_Change_priority( the_thread, priority, + NULL, + _Thread_Raise_real_priority_filter, false ); } diff --git a/cpukit/score/src/threadsetpriority.c b/cpukit/score/src/threadsetpriority.c index e1ff118c7e..f6a061a281 100644 --- a/cpukit/score/src/threadsetpriority.c +++ b/cpukit/score/src/threadsetpriority.c @@ -19,14 +19,41 @@ #endif #include <rtems/score/threadimpl.h> -#include <rtems/score/schedulerimpl.h> -void _Thread_Set_priority( +static bool _Thread_Set_priority_filter( Thread_Control *the_thread, - Priority_Control new_priority + Priority_Control *new_priority_ptr, + void *arg ) { - the_thread->current_priority = new_priority; + Priority_Control current_priority; + Priority_Control new_priority; + Priority_Control *old_priority_ptr; + + current_priority = the_thread->current_priority; + new_priority = *new_priority_ptr; + + old_priority_ptr = arg; + *old_priority_ptr = current_priority; + + the_thread->real_priority = new_priority; + + return _Thread_Priority_less_than( current_priority, new_priority ) + || !_Thread_Owns_resources( the_thread ); +} - _Scheduler_Update_priority( the_thread, new_priority ); +void _Thread_Set_priority( + Thread_Control *the_thread, + Priority_Control new_priority, + Priority_Control *old_priority, + bool prepend_it +) +{ + _Thread_Change_priority( + the_thread, + new_priority, + old_priority, + _Thread_Set_priority_filter, + prepend_it + ); } |