diff options
19 files changed, 741 insertions, 91 deletions
diff --git a/cpukit/include/rtems/score/mrspimpl.h b/cpukit/include/rtems/score/mrspimpl.h index 3e64ad94e6..daa309e7cb 100644 --- a/cpukit/include/rtems/score/mrspimpl.h +++ b/cpukit/include/rtems/score/mrspimpl.h @@ -268,7 +268,7 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Claim_ownership( _MRSP_Set_owner( mrsp, executing ); cpu_self = _Thread_queue_Dispatch_disable( queue_context ); _MRSP_Release( mrsp, queue_context ); - _Thread_Priority_and_sticky_update( executing, 1 ); + _Thread_Priority_update_and_make_sticky( executing ); _Thread_Dispatch_enable( cpu_self ); return STATUS_SUCCESSFUL; } @@ -384,13 +384,6 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Wait_for_ownership( _MRSP_Replace_priority( mrsp, executing, &ceiling_priority ); } else { Per_CPU_Control *cpu_self; - int sticky_level_change; - - if ( status != STATUS_DEADLOCK ) { - sticky_level_change = -1; - } else { - sticky_level_change = 0; - } _ISR_lock_ISR_disable( &queue_context->Lock_context.Lock_context ); _MRSP_Remove_priority( executing, &ceiling_priority, queue_context ); @@ -398,7 +391,13 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Wait_for_ownership( &queue_context->Lock_context.Lock_context ); _ISR_lock_ISR_enable( &queue_context->Lock_context.Lock_context ); - _Thread_Priority_and_sticky_update( executing, sticky_level_change ); + + if ( status != STATUS_DEADLOCK ) { + _Thread_Priority_update_and_clean_sticky( executing ); + } else { + _Thread_Priority_update_ignore_sticky( executing ); + } + _Thread_Dispatch_enable( cpu_self ); } @@ -493,7 +492,7 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Surrender( &queue_context->Lock_context.Lock_context ); _MRSP_Release( mrsp, queue_context ); - _Thread_Priority_and_sticky_update( executing, -1 ); + _Thread_Priority_update_and_clean_sticky( executing ); _Thread_Dispatch_enable( cpu_self ); return STATUS_SUCCESSFUL; } diff --git a/cpukit/include/rtems/score/scheduler.h b/cpukit/include/rtems/score/scheduler.h index ad9d630023..95b4414bea 100644 --- a/cpukit/include/rtems/score/scheduler.h +++ b/cpukit/include/rtems/score/scheduler.h @@ -135,6 +135,61 @@ typedef struct { ); /** + * @brief Makes the node sticky. + * + * This operation is used by _Thread_Priority_update_and_make_sticky(). It + * is only called for the scheduler node of the home scheduler. + * + * Uniprocessor schedulers schould provide + * _Scheduler_default_Sticky_do_nothing() for this operation. + * + * SMP schedulers should provide this operation using + * _Scheduler_SMP_Make_sticky(). + * + * The make and clean sticky operations are an optimization to simplify the + * control flow in the update priority operation. The update priority + * operation is used for all scheduler nodes and not just the scheduler node + * of home schedulers. The update priority operation is a commonly used + * operations together with block and unblock. The make and clean sticky + * operations are used only in specific scenarios. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ + void ( *make_sticky )( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node + ); + + /** + * @brief Cleans the sticky property from the node. + * + * This operation is used by _Thread_Priority_update_and_clean_sticky(). It + * is only called for the scheduler node of the home scheduler. + * + * Uniprocessor schedulers schould provide + * _Scheduler_default_Sticky_do_nothing() for this operation. + * + * SMP schedulers should provide this operation using + * _Scheduler_SMP_Clean_sticky(). + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ + void ( *clean_sticky )( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node + ); + + /** * @brief Pin thread operation. * * @param[in] scheduler The scheduler instance of the specified processor. @@ -400,6 +455,24 @@ Priority_Control _Scheduler_default_Unmap_priority( /** * @brief Does nothing. * + * This default implementation for the make and clean sticky operations + * should be used by uniprocessor schedulers if SMP support is enabled. + * + * @param scheduler is an unused parameter. + * + * @param the_thread is an unused parameter. + * + * @param node is an unused parameter. + */ + void _Scheduler_default_Sticky_do_nothing( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node + ); + + /** + * @brief Does nothing. + * * This default implementation for the thread pin or unpin operations should * be used by uniprocessor schedulers if SMP support is enabled. * @@ -459,6 +532,8 @@ Priority_Control _Scheduler_default_Unmap_priority( NULL, \ NULL, \ NULL, \ + _Scheduler_default_Sticky_do_nothing, \ + _Scheduler_default_Sticky_do_nothing, \ _Scheduler_default_Pin_or_unpin_do_nothing, \ _Scheduler_default_Pin_or_unpin_do_nothing, \ NULL, \ diff --git a/cpukit/include/rtems/score/scheduleredfsmp.h b/cpukit/include/rtems/score/scheduleredfsmp.h index ec975ed12f..75865e5a6e 100644 --- a/cpukit/include/rtems/score/scheduleredfsmp.h +++ b/cpukit/include/rtems/score/scheduleredfsmp.h @@ -129,6 +129,8 @@ typedef struct { _Scheduler_EDF_SMP_Ask_for_help, \ _Scheduler_EDF_SMP_Reconsider_help_request, \ _Scheduler_EDF_SMP_Withdraw_node, \ + _Scheduler_EDF_SMP_Make_sticky, \ + _Scheduler_EDF_SMP_Clean_sticky, \ _Scheduler_EDF_SMP_Pin, \ _Scheduler_EDF_SMP_Unpin, \ _Scheduler_EDF_SMP_Add_processor, \ @@ -249,6 +251,36 @@ void _Scheduler_EDF_SMP_Withdraw_node( ); /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_EDF_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_EDF_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** * @brief Pin thread operation. * * @param scheduler The scheduler instance of the specified processor. diff --git a/cpukit/include/rtems/score/schedulerimpl.h b/cpukit/include/rtems/score/schedulerimpl.h index 7319cc4e4c..eb279876c7 100644 --- a/cpukit/include/rtems/score/schedulerimpl.h +++ b/cpukit/include/rtems/score/schedulerimpl.h @@ -405,65 +405,6 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Update_priority( Thread_Control *the_thread #endif } -#if defined(RTEMS_SMP) -/** - * @brief Changes the sticky level of the home scheduler node and propagates a - * priority change of a thread to the scheduler. - * - * @param the_thread The thread changing its priority or sticky level. - * - * @see _Scheduler_Update_priority(). - */ -RTEMS_INLINE_ROUTINE void _Scheduler_Priority_and_sticky_update( - Thread_Control *the_thread, - int sticky_level_change -) -{ - Chain_Node *node; - const Chain_Node *tail; - Scheduler_Node *scheduler_node; - const Scheduler_Control *scheduler; - ISR_lock_Context lock_context; - - _Thread_Scheduler_process_requests( the_thread ); - - node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes ); - scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node ); - scheduler = _Scheduler_Node_get_scheduler( scheduler_node ); - - _Scheduler_Acquire_critical( scheduler, &lock_context ); - - scheduler_node->sticky_level += sticky_level_change; - _Assert( scheduler_node->sticky_level >= 0 ); - - ( *scheduler->Operations.update_priority )( - scheduler, - the_thread, - scheduler_node - ); - - _Scheduler_Release_critical( scheduler, &lock_context ); - - tail = _Chain_Immutable_tail( &the_thread->Scheduler.Scheduler_nodes ); - node = _Chain_Next( node ); - - while ( node != tail ) { - scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node ); - scheduler = _Scheduler_Node_get_scheduler( scheduler_node ); - - _Scheduler_Acquire_critical( scheduler, &lock_context ); - ( *scheduler->Operations.update_priority )( - scheduler, - the_thread, - scheduler_node - ); - _Scheduler_Release_critical( scheduler, &lock_context ); - - node = _Chain_Next( node ); - } -} -#endif - /** * @brief Maps a thread priority from the user domain to the scheduler domain. * diff --git a/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h b/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h index e997e81128..d77629b39d 100644 --- a/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h +++ b/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h @@ -65,6 +65,8 @@ extern "C" { _Scheduler_priority_affinity_SMP_Ask_for_help, \ _Scheduler_priority_affinity_SMP_Reconsider_help_request, \ _Scheduler_priority_affinity_SMP_Withdraw_node, \ + _Scheduler_priority_affinity_SMP_Make_sticky, \ + _Scheduler_priority_affinity_SMP_Clean_sticky, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_priority_affinity_SMP_Add_processor, \ @@ -181,6 +183,36 @@ void _Scheduler_priority_affinity_SMP_Withdraw_node( ); /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_priority_affinity_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_priority_affinity_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** * @brief Adds @a idle to @a scheduler. * * @param[in, out] scheduler The scheduler instance to add the processor to. diff --git a/cpukit/include/rtems/score/schedulerprioritysmp.h b/cpukit/include/rtems/score/schedulerprioritysmp.h index fe314fb05b..9ece9ae143 100644 --- a/cpukit/include/rtems/score/schedulerprioritysmp.h +++ b/cpukit/include/rtems/score/schedulerprioritysmp.h @@ -93,6 +93,8 @@ typedef struct { _Scheduler_priority_SMP_Ask_for_help, \ _Scheduler_priority_SMP_Reconsider_help_request, \ _Scheduler_priority_SMP_Withdraw_node, \ + _Scheduler_priority_SMP_Make_sticky, \ + _Scheduler_priority_SMP_Clean_sticky, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_priority_SMP_Add_processor, \ @@ -215,6 +217,36 @@ void _Scheduler_priority_SMP_Withdraw_node( ); /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_priority_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_priority_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** * @brief Adds @a idle to @a scheduler. * * @param[in, out] scheduler The scheduler instance to add the processor to. diff --git a/cpukit/include/rtems/score/schedulersimplesmp.h b/cpukit/include/rtems/score/schedulersimplesmp.h index c8394781c9..3b6f43869e 100644 --- a/cpukit/include/rtems/score/schedulersimplesmp.h +++ b/cpukit/include/rtems/score/schedulersimplesmp.h @@ -75,6 +75,8 @@ typedef struct { _Scheduler_simple_SMP_Ask_for_help, \ _Scheduler_simple_SMP_Reconsider_help_request, \ _Scheduler_simple_SMP_Withdraw_node, \ + _Scheduler_simple_SMP_Make_sticky, \ + _Scheduler_simple_SMP_Clean_sticky, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_simple_SMP_Add_processor, \ @@ -195,6 +197,36 @@ void _Scheduler_simple_SMP_Withdraw_node( ); /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_simple_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_simple_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** * @brief Adds @a idle to @a scheduler. * * @param[in, out] scheduler The scheduler instance to add the processor to. diff --git a/cpukit/include/rtems/score/schedulersmpimpl.h b/cpukit/include/rtems/score/schedulersmpimpl.h index 04019a1540..731b15d4bf 100644 --- a/cpukit/include/rtems/score/schedulersmpimpl.h +++ b/cpukit/include/rtems/score/schedulersmpimpl.h @@ -1688,6 +1688,90 @@ static inline void _Scheduler_SMP_Withdraw_node( } /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +static inline void _Scheduler_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node, + Scheduler_SMP_Update update, + Scheduler_SMP_Enqueue enqueue +) +{ + Scheduler_SMP_Node_state node_state; + + node_state = _Scheduler_SMP_Node_state( node ); + + if ( node_state == SCHEDULER_SMP_NODE_BLOCKED ) { + Scheduler_Context *context; + Priority_Control insert_priority; + Priority_Control priority; + + context = _Scheduler_Get_context( scheduler ); + priority = _Scheduler_Node_get_priority( node ); + priority = SCHEDULER_PRIORITY_PURIFY( priority ); + + if ( priority != _Scheduler_SMP_Node_priority( node ) ) { + ( *update )( context, node, priority ); + } + + _Scheduler_SMP_Node_change_state( node, SCHEDULER_SMP_NODE_READY ); + insert_priority = SCHEDULER_PRIORITY_APPEND( priority ); + (void) ( *enqueue )( context, node, insert_priority ); + } +} + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +static inline void _Scheduler_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node, + Scheduler_SMP_Extract extract_from_scheduled, + Scheduler_SMP_Extract extract_from_ready, + Scheduler_SMP_Get_highest_ready get_highest_ready, + Scheduler_SMP_Move move_from_ready_to_scheduled, + Scheduler_SMP_Allocate_processor allocate_processor +) +{ + Scheduler_SMP_Node_state node_state; + + node_state = _Scheduler_SMP_Node_state( node ); + + if ( node_state == SCHEDULER_SMP_NODE_SCHEDULED && node->idle != NULL ) { + Scheduler_Context *context; + + context = _Scheduler_Get_context( scheduler ); + _Scheduler_SMP_Node_change_state( node, SCHEDULER_SMP_NODE_BLOCKED ); + + ( *extract_from_scheduled )( context, node ); + + _Scheduler_SMP_Schedule_highest_ready( + context, + node, + _Thread_Get_CPU( node->idle ), + extract_from_ready, + get_highest_ready, + move_from_ready_to_scheduled, + allocate_processor + ); + } +} + +/** * @brief Starts the idle thread on the given processor. * * @param context The scheduler context instance. diff --git a/cpukit/include/rtems/score/schedulerstrongapa.h b/cpukit/include/rtems/score/schedulerstrongapa.h index bbded1b493..8db3ae8634 100644 --- a/cpukit/include/rtems/score/schedulerstrongapa.h +++ b/cpukit/include/rtems/score/schedulerstrongapa.h @@ -161,6 +161,8 @@ typedef struct { _Scheduler_strong_APA_Ask_for_help, \ _Scheduler_strong_APA_Reconsider_help_request, \ _Scheduler_strong_APA_Withdraw_node, \ + _Scheduler_strong_APA_Make_sticky, \ + _Scheduler_strong_APA_Clean_sticky, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_default_Pin_or_unpin_not_supported, \ _Scheduler_strong_APA_Add_processor, \ @@ -279,6 +281,66 @@ void _Scheduler_strong_APA_Withdraw_node( ); /** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_strong_APA_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_strong_APA_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Makes the node sticky. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to make sticky. + */ +void _Scheduler_strong_APA_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** + * @brief Cleans the sticky property from the node. + * + * @param scheduler is the scheduler of the node. + * + * @param[in, out] the_thread is the thread owning the node. + * + * @param[in, out] node is the scheduler node to clean the sticky property. + */ +void _Scheduler_strong_APA_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +); + +/** * @brief Adds the idle thread to a processor. * * @param scheduler The scheduler control instance. diff --git a/cpukit/include/rtems/score/threadimpl.h b/cpukit/include/rtems/score/threadimpl.h index 934b56468a..a55524cfd1 100644 --- a/cpukit/include/rtems/score/threadimpl.h +++ b/cpukit/include/rtems/score/threadimpl.h @@ -790,17 +790,29 @@ void _Thread_Priority_replace( */ void _Thread_Priority_update( Thread_queue_Context *queue_context ); +#if defined(RTEMS_SMP) /** - * @brief Updates the priority of the thread and changes it sticky level. + * @brief Updates the priority of the thread and makes its home scheduler node + * sticky. * - * @param the_thread The thread. - * @param sticky_level_change The new value for the sticky level. + * @param the_thread is the thread to work on. */ -#if defined(RTEMS_SMP) -void _Thread_Priority_and_sticky_update( - Thread_Control *the_thread, - int sticky_level_change -); +void _Thread_Priority_update_and_make_sticky( Thread_Control *the_thread ); + +/** + * @brief Updates the priority of the thread and cleans the sticky property of + * its home scheduler node. + * + * @param the_thread is the thread to work on. + */ +void _Thread_Priority_update_and_clean_sticky( Thread_Control *the_thread ); + +/** + * @brief Updates the priority of the thread. + * + * @param the_thread is the thread to update the priority. + */ +void _Thread_Priority_update_ignore_sticky( Thread_Control *the_thread ); #endif /** diff --git a/cpukit/score/src/schedulerdefaultmakecleansticky.c b/cpukit/score/src/schedulerdefaultmakecleansticky.c new file mode 100644 index 0000000000..e2b2d659f1 --- /dev/null +++ b/cpukit/score/src/schedulerdefaultmakecleansticky.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +/** + * @file + * + * @ingroup RTEMSScoreScheduler + * + * @brief This source file contains the implementation of + * _Scheduler_default_Sticky_do_nothing(). + */ + +/* + * Copyright (C) 2021 embedded brains GmbH (http://www.embedded-brains.de) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <rtems/score/scheduler.h> + +void _Scheduler_default_Sticky_do_nothing( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + (void) scheduler; + (void) the_thread; + (void) node; +} diff --git a/cpukit/score/src/scheduleredfsmp.c b/cpukit/score/src/scheduleredfsmp.c index 27be08ac40..0b0ee6ed21 100644 --- a/cpukit/score/src/scheduleredfsmp.c +++ b/cpukit/score/src/scheduleredfsmp.c @@ -11,7 +11,8 @@ * _Scheduler_EDF_SMP_Remove_processor(), _Scheduler_EDF_SMP_Set_affinity(), * _Scheduler_EDF_SMP_Start_idle(), _Scheduler_EDF_SMP_Unblock(), * _Scheduler_EDF_SMP_Unpin(), _Scheduler_EDF_SMP_Update_priority(), - * _Scheduler_EDF_SMP_Withdraw_node(), and _Scheduler_EDF_SMP_Yield(). + * _Scheduler_EDF_SMP_Withdraw_node(), _Scheduler_EDF_SMP_Make_sticky(), + * _Scheduler_EDF_SMP_Clean_sticky(), and _Scheduler_EDF_SMP_Yield(). */ /* @@ -619,6 +620,39 @@ void _Scheduler_EDF_SMP_Withdraw_node( ); } +void _Scheduler_EDF_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Make_sticky( + scheduler, + the_thread, + node, + _Scheduler_EDF_SMP_Do_update, + _Scheduler_EDF_SMP_Enqueue + ); +} + +void _Scheduler_EDF_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Clean_sticky( + scheduler, + the_thread, + node, + _Scheduler_EDF_SMP_Extract_from_scheduled, + _Scheduler_EDF_SMP_Extract_from_ready, + _Scheduler_EDF_SMP_Get_highest_ready, + _Scheduler_EDF_SMP_Move_from_ready_to_scheduled, + _Scheduler_EDF_SMP_Allocate_processor + ); +} + static inline void _Scheduler_EDF_SMP_Register_idle( Scheduler_Context *context, Scheduler_Node *idle_base, diff --git a/cpukit/score/src/schedulerpriorityaffinitysmp.c b/cpukit/score/src/schedulerpriorityaffinitysmp.c index 4bbf2f6e17..648c243589 100644 --- a/cpukit/score/src/schedulerpriorityaffinitysmp.c +++ b/cpukit/score/src/schedulerpriorityaffinitysmp.c @@ -12,8 +12,10 @@ * _Scheduler_priority_affinity_SMP_Remove_processor(), * _Scheduler_priority_affinity_SMP_Set_affinity(), * _Scheduler_priority_affinity_SMP_Unblock(), - * _Scheduler_priority_affinity_SMP_Update_priority(), and - * _Scheduler_priority_affinity_SMP_Withdraw_node(). + * _Scheduler_priority_affinity_SMP_Update_priority(), + * _Scheduler_priority_affinity_SMP_Withdraw_node(), + * _Scheduler_priority_affinity_SMP_Make_sticky(), and + * _Scheduler_priority_affinity_SMP_Clean_sticky(). */ /* @@ -512,6 +514,39 @@ void _Scheduler_priority_affinity_SMP_Withdraw_node( ); } +void _Scheduler_priority_affinity_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Make_sticky( + scheduler, + the_thread, + node, + _Scheduler_priority_SMP_Do_update, + _Scheduler_priority_affinity_SMP_Enqueue + ); +} + +void _Scheduler_priority_affinity_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Clean_sticky( + scheduler, + the_thread, + node, + _Scheduler_SMP_Extract_from_scheduled, + _Scheduler_priority_SMP_Extract_from_ready, + _Scheduler_priority_affinity_SMP_Get_highest_ready, + _Scheduler_priority_SMP_Move_from_ready_to_scheduled, + _Scheduler_SMP_Allocate_processor_exact + ); +} + void _Scheduler_priority_affinity_SMP_Add_processor( const Scheduler_Control *scheduler, Thread_Control *idle diff --git a/cpukit/score/src/schedulerprioritysmp.c b/cpukit/score/src/schedulerprioritysmp.c index b0b920c960..7262f48e8e 100644 --- a/cpukit/score/src/schedulerprioritysmp.c +++ b/cpukit/score/src/schedulerprioritysmp.c @@ -12,7 +12,9 @@ * _Scheduler_priority_SMP_Remove_processor(), * _Scheduler_priority_SMP_Unblock(), * _Scheduler_priority_SMP_Update_priority(), - * _Scheduler_priority_SMP_Withdraw_node(), and + * _Scheduler_priority_SMP_Withdraw_node(), + * _Scheduler_priority_SMP_Make_sticky(), + * _Scheduler_priority_SMP_Clean_sticky(), and * _Scheduler_priority_SMP_Yield(). */ @@ -265,6 +267,39 @@ void _Scheduler_priority_SMP_Withdraw_node( ); } +void _Scheduler_priority_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Make_sticky( + scheduler, + the_thread, + node, + _Scheduler_priority_SMP_Do_update, + _Scheduler_priority_SMP_Enqueue + ); +} + +void _Scheduler_priority_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Clean_sticky( + scheduler, + the_thread, + node, + _Scheduler_SMP_Extract_from_scheduled, + _Scheduler_priority_SMP_Extract_from_ready, + _Scheduler_priority_SMP_Get_highest_ready, + _Scheduler_priority_SMP_Move_from_ready_to_scheduled, + _Scheduler_SMP_Allocate_processor_lazy + ); +} + void _Scheduler_priority_SMP_Add_processor( const Scheduler_Control *scheduler, Thread_Control *idle diff --git a/cpukit/score/src/schedulersimplesmp.c b/cpukit/score/src/schedulersimplesmp.c index 02e4579aa1..b7814b0fcb 100644 --- a/cpukit/score/src/schedulersimplesmp.c +++ b/cpukit/score/src/schedulersimplesmp.c @@ -11,7 +11,9 @@ * _Scheduler_simple_SMP_Reconsider_help_request(), * _Scheduler_simple_SMP_Remove_processor(), _Scheduler_simple_SMP_Unblock(), * _Scheduler_simple_SMP_Update_priority(), - * _Scheduler_simple_SMP_Withdraw_node(), and _Scheduler_simple_SMP_Yield(). + * _Scheduler_simple_SMP_Withdraw_node(), + * _Scheduler_simple_SMP_Make_sticky(), _Scheduler_simple_SMP_Clean_sticky(), + * and _Scheduler_simple_SMP_Yield(). */ /* @@ -335,6 +337,39 @@ void _Scheduler_simple_SMP_Withdraw_node( ); } +void _Scheduler_simple_SMP_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Make_sticky( + scheduler, + the_thread, + node, + _Scheduler_simple_SMP_Do_update, + _Scheduler_simple_SMP_Enqueue + ); +} + +void _Scheduler_simple_SMP_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Clean_sticky( + scheduler, + the_thread, + node, + _Scheduler_SMP_Extract_from_scheduled, + _Scheduler_simple_SMP_Extract_from_ready, + _Scheduler_simple_SMP_Get_highest_ready, + _Scheduler_simple_SMP_Move_from_ready_to_scheduled, + _Scheduler_SMP_Allocate_processor_lazy + ); +} + void _Scheduler_simple_SMP_Add_processor( const Scheduler_Control *scheduler, Thread_Control *idle diff --git a/cpukit/score/src/schedulerstrongapa.c b/cpukit/score/src/schedulerstrongapa.c index afd9fcc709..6c7ab942fa 100644 --- a/cpukit/score/src/schedulerstrongapa.c +++ b/cpukit/score/src/schedulerstrongapa.c @@ -31,7 +31,9 @@ * _Scheduler_strong_APA_Set_affinity(), * _Scheduler_strong_APA_Set_scheduled(), _Scheduler_strong_APA_Start_idle(), * _Scheduler_strong_APA_Unblock(), _Scheduler_strong_APA_Update_priority(), - * _Scheduler_strong_APA_Withdraw_node(), and _Scheduler_strong_APA_Yield(). + * _Scheduler_strong_APA_Withdraw_node(), + * _Scheduler_strong_APA_Make_sticky(), _Scheduler_strong_APA_Clean_sticky(), + * and _Scheduler_strong_APA_Yield(). */ /* @@ -935,6 +937,39 @@ void _Scheduler_strong_APA_Withdraw_node( ); } +void _Scheduler_strong_APA_Make_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Make_sticky( + scheduler, + the_thread, + node, + _Scheduler_strong_APA_Do_update, + _Scheduler_strong_APA_Enqueue + ); +} + +void _Scheduler_strong_APA_Clean_sticky( + const Scheduler_Control *scheduler, + Thread_Control *the_thread, + Scheduler_Node *node +) +{ + _Scheduler_SMP_Clean_sticky( + scheduler, + the_thread, + node, + _Scheduler_SMP_Extract_from_scheduled, + _Scheduler_strong_APA_Extract_from_ready, + _Scheduler_strong_APA_Get_highest_ready, + _Scheduler_strong_APA_Move_from_ready_to_scheduled, + _Scheduler_strong_APA_Allocate_processor + ); +} + static inline void _Scheduler_strong_APA_Register_idle( Scheduler_Context *context, Scheduler_Node *idle_base, diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c index bd4fef279b..637d5e810d 100644 --- a/cpukit/score/src/threadchangepriority.c +++ b/cpukit/score/src/threadchangepriority.c @@ -372,18 +372,140 @@ void _Thread_Priority_update( Thread_queue_Context *queue_context ) } #if defined(RTEMS_SMP) -void _Thread_Priority_and_sticky_update( +static void _Thread_Priority_update_helping( Thread_Control *the_thread, - int sticky_level_change + Chain_Node *first_node ) { - ISR_lock_Context lock_context; + const Chain_Node *tail; + Chain_Node *node; + + tail = _Chain_Immutable_tail( &the_thread->Scheduler.Scheduler_nodes ); + node = _Chain_Next( first_node ); + + while ( node != tail ) { + Scheduler_Node *scheduler_node; + const Scheduler_Control *scheduler; + ISR_lock_Context lock_context; + + scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node ); + scheduler = _Scheduler_Node_get_scheduler( scheduler_node ); + + _Scheduler_Acquire_critical( scheduler, &lock_context ); + ( *scheduler->Operations.update_priority )( + scheduler, + the_thread, + scheduler_node + ); + _Scheduler_Release_critical( scheduler, &lock_context ); + + node = _Chain_Next( node ); + } +} + +void _Thread_Priority_update_and_make_sticky( Thread_Control *the_thread ) +{ + ISR_lock_Context lock_context; + ISR_lock_Context lock_context_2; + Chain_Node *node; + Scheduler_Node *scheduler_node; + const Scheduler_Control *scheduler; + int new_sticky_level; + int make_sticky_level; _Thread_State_acquire( the_thread, &lock_context ); - _Scheduler_Priority_and_sticky_update( + _Thread_Scheduler_process_requests( the_thread ); + + node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes ); + scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node ); + scheduler = _Scheduler_Node_get_scheduler( scheduler_node ); + + _Scheduler_Acquire_critical( scheduler, &lock_context_2 ); + + new_sticky_level = scheduler_node->sticky_level + 1; + scheduler_node->sticky_level = new_sticky_level; + _Assert( new_sticky_level >= 1 ); + + /* + * The sticky level is incremented by the scheduler block operation, so for a + * ready thread, the change to sticky happens at a level of two. + */ + make_sticky_level = 1 + (int) _Thread_Is_ready( the_thread ); + + if ( new_sticky_level == make_sticky_level ) { + ( *scheduler->Operations.make_sticky )( + scheduler, + the_thread, + scheduler_node + ); + } + + ( *scheduler->Operations.update_priority )( + scheduler, the_thread, - sticky_level_change + scheduler_node ); + + _Scheduler_Release_critical( scheduler, &lock_context_2 ); + _Thread_Priority_update_helping( the_thread, node ); + _Thread_State_release( the_thread, &lock_context ); +} + +void _Thread_Priority_update_and_clean_sticky( Thread_Control *the_thread ) +{ + ISR_lock_Context lock_context; + ISR_lock_Context lock_context_2; + Chain_Node *node; + Scheduler_Node *scheduler_node; + const Scheduler_Control *scheduler; + int new_sticky_level; + int clean_sticky_level; + + _Thread_State_acquire( the_thread, &lock_context ); + _Thread_Scheduler_process_requests( the_thread ); + + node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes ); + scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node ); + scheduler = _Scheduler_Node_get_scheduler( scheduler_node ); + + _Scheduler_Acquire_critical( scheduler, &lock_context_2 ); + + new_sticky_level = scheduler_node->sticky_level - 1; + scheduler_node->sticky_level = new_sticky_level; + _Assert( new_sticky_level >= 0 ); + + /* + * The sticky level is incremented by the scheduler block operation, so for a + * ready thread, the change to sticky happens at a level of one. + */ + clean_sticky_level = (int) _Thread_Is_ready( the_thread ); + + if ( new_sticky_level == clean_sticky_level ) { + ( *scheduler->Operations.clean_sticky )( + scheduler, + the_thread, + scheduler_node + ); + } + + ( *scheduler->Operations.update_priority )( + scheduler, + the_thread, + scheduler_node + ); + + _Scheduler_Release_critical( scheduler, &lock_context_2 ); + _Thread_Priority_update_helping( the_thread, node ); + _Thread_State_release( the_thread, &lock_context ); +} + +void _Thread_Priority_update_ignore_sticky( Thread_Control *the_thread ) +{ + ISR_lock_Context lock_context; + + _Thread_State_acquire( the_thread, &lock_context ); + _Thread_Scheduler_process_requests( the_thread ); + _Scheduler_Update_priority( the_thread ); _Thread_State_release( the_thread, &lock_context ); } #endif diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c index 5fc357ec82..ed6c64543c 100644 --- a/cpukit/score/src/threadqenqueue.c +++ b/cpukit/score/src/threadqenqueue.c @@ -518,7 +518,7 @@ Status_Control _Thread_queue_Enqueue_sticky( ); _Thread_Priority_update( queue_context ); - _Thread_Priority_and_sticky_update( the_thread, 1 ); + _Thread_Priority_update_and_make_sticky( the_thread ); _Thread_Dispatch_enable( cpu_self ); while ( @@ -899,8 +899,8 @@ void _Thread_queue_Surrender_sticky( queue, &queue_context->Lock_context.Lock_context ); - _Thread_Priority_and_sticky_update( previous_owner, -1 ); - _Thread_Priority_and_sticky_update( new_owner, 0 ); + _Thread_Priority_update_and_clean_sticky( previous_owner ); + _Thread_Priority_update_ignore_sticky( new_owner ); _Thread_Dispatch_enable( cpu_self ); } #endif diff --git a/spec/build/cpukit/objsmp.yml b/spec/build/cpukit/objsmp.yml index 1a55708638..ac57bb27b1 100644 --- a/spec/build/cpukit/objsmp.yml +++ b/spec/build/cpukit/objsmp.yml @@ -13,6 +13,7 @@ source: - cpukit/score/src/percpujobs.c - cpukit/score/src/percpustatewait.c - cpukit/score/src/profilingsmplock.c +- cpukit/score/src/schedulerdefaultmakecleansticky.c - cpukit/score/src/schedulerdefaultpinunpin.c - cpukit/score/src/schedulerdefaultpinunpindonothing.c - cpukit/score/src/schedulerdefaultsetaffinity.c |