From cbb1103a3ca9d2f42501ba373b5d559b981fb1c7 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 27 Jul 2021 13:40:51 +0200 Subject: score: Simplify SMP processor state handling The per-CPU states which control the SMP system initialization were added quite early during the SMP support development. Replace this initial implementation with a simplified one. There is no longer a global SMP lock required which serialized the state changes of all processors. The new implementation better integrates with the per-CPU jobs. --- cpukit/include/rtems/score/percpu.h | 42 +++----- cpukit/include/rtems/score/smpimpl.h | 61 +++++++++--- cpukit/score/src/percpu.c | 151 +---------------------------- cpukit/score/src/smp.c | 142 ++++++++++++++++++++++++--- cpukit/score/src/smpmulticastaction.c | 36 ++----- cpukit/score/src/threadstartmultitasking.c | 4 +- 6 files changed, 209 insertions(+), 227 deletions(-) (limited to 'cpukit') diff --git a/cpukit/include/rtems/score/percpu.h b/cpukit/include/rtems/score/percpu.h index e79596c244..f72339620d 100644 --- a/cpukit/include/rtems/score/percpu.h +++ b/cpukit/include/rtems/score/percpu.h @@ -102,15 +102,18 @@ struct Scheduler_Context; * The processor state controls the life cycle of processors at the lowest * level. No multi-threading or other high-level concepts matter here. * - * State changes must be initiated via _Per_CPU_State_change(). This function - * may not return in case someone requested a shutdown. The - * _SMP_Send_message() function will be used to notify other processors about - * state changes if the other processor is in the up state. + * The state of a processor is indicated by the Per_CPU_Control::state membe. + * The current state of a processor can be get by _Per_CPU_Get_state(). Only + * the processor associated with the control may change its state using + * _Per_CPU_Set_state(). * * Due to the sequential nature of the basic system initialization one * processor has a special role. It is the processor executing the boot_card() * function. This processor is called the boot processor. All other - * processors are called secondary. + * processors are called secondary. The boot processor uses + * _SMP_Request_start_multitasking() to indicate that processors should start + * multiprocessing. Secondary processors will wait for this request in + * _SMP_Start_multitasking_on_secondary_processor(). * * @dot * digraph states { @@ -150,22 +153,12 @@ typedef enum { * the first time. The boot processor will wait for all secondary processors * to change into this state. In case a secondary processor does not reach * this state the system will not start. The secondary processors wait now - * for a change into the PER_CPU_STATE_REQUEST_START_MULTITASKING state set - * by the boot processor once all secondary processors reached the - * PER_CPU_STATE_READY_TO_START_MULTITASKING state. + * for a change into the PER_CPU_STATE_UP state set requested by the boot + * processor through ::_SMP_Ready_to_start_multitasking once all secondary + * processors reached the PER_CPU_STATE_READY_TO_START_MULTITASKING state. */ PER_CPU_STATE_READY_TO_START_MULTITASKING, - /** - * @brief Multitasking start of processor is requested. - * - * The boot processor completed system initialization and is about to perform - * a context switch to its heir thread. Secondary processors should now - * issue a context switch to the heir thread. This normally enables - * interrupts on the processor for the first time. - */ - PER_CPU_STATE_REQUEST_START_MULTITASKING, - /** * @brief Normal multitasking state. */ @@ -547,11 +540,12 @@ typedef struct Per_CPU_Control { char *data; /** - * @brief Indicates the current state of the CPU. + * @brief Indicates the current state of the processor. * - * This member is protected by the _Per_CPU_State_lock lock. + * Only the processor associated with this control is allowed to change + * this member. * - * @see _Per_CPU_State_change(). + * @see _Per_CPU_Get_state() and _Per_CPU_Set_state(). */ Atomic_Uint state; @@ -801,6 +795,7 @@ static inline void _Per_CPU_Set_state( Per_CPU_State state ) { + _Assert( cpu_self == _Per_CPU_Get() ); _Atomic_Store_uint( &cpu_self->state, (unsigned int) state, @@ -808,11 +803,6 @@ static inline void _Per_CPU_Set_state( ); } -void _Per_CPU_State_change( - Per_CPU_Control *cpu, - Per_CPU_State new_state -); - /** * @brief Waits for a processor to change into a non-initial state. * diff --git a/cpukit/include/rtems/score/smpimpl.h b/cpukit/include/rtems/score/smpimpl.h index 32704d7288..eea48f5060 100644 --- a/cpukit/include/rtems/score/smpimpl.h +++ b/cpukit/include/rtems/score/smpimpl.h @@ -50,6 +50,16 @@ extern "C" { */ #define SMP_MESSAGE_PERFORM_JOBS 0x2UL +/** + * @brief SMP message to force the message processing in + * _SMP_Try_to_process_message(). + * + * This message bit is never sent to a processor. It is only used to force the + * message processing in _SMP_Try_to_process_message(). Any non-zero value + * would do it. + */ +#define SMP_MESSAGE_FORCE_PROCESSING 0x4UL + /** * @brief SMP fatal codes. */ @@ -130,11 +140,47 @@ RTEMS_NO_RETURN void _SMP_Start_multitasking_on_secondary_processor( ); /** - * @brief Interrupts handler for inter-processor interrupts. + * @brief Processes the SMP message. + * + * @param[in, out] cpu_self is the processor control of the processor executing + * this function. + * + * @return Returns the processed message. + */ +long unsigned _SMP_Process_message( + Per_CPU_Control *cpu_self, + long unsigned message +); + +/** + * @brief Tries to process the current SMP message. + * + * This function may be used in busy wait loops. + * + * @param cpu_self is the processor control of the processor executing this + * function. + * + * @param message is used to check if the SMP message processing should be + * carried out. If it is not equal to zero, then _SMP_Process_message() is + * called with a newly fetched message. This parameter is not used to process + * the message. It is only used to check if the processing is necessary. + * Use #SMP_MESSAGE_FORCE_PROCESSING to force the message processing. + */ +void _SMP_Try_to_process_message( + Per_CPU_Control *cpu_self, + unsigned long message +); + +/** + * @brief Processes an inter-processor interrupt. + * + * Use this function for the inter-processor interrupt handler. Never call + * this function in a tight loop. * - * @param[in, out] cpu_self The cpu control for the operation. + * @param[in, out] cpu_self is the processor control of the processor executing + * this function. * - * @return The received message. + * @return Returns the processed message. */ static inline long unsigned _SMP_Inter_processor_interrupt_handler( Per_CPU_Control *cpu_self @@ -155,14 +201,7 @@ static inline long unsigned _SMP_Inter_processor_interrupt_handler( ); if ( RTEMS_PREDICT_FALSE( message != 0 ) ) { - if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) { - _SMP_Fatal( SMP_FATAL_SHUTDOWN_RESPONSE ); - /* does not continue past here */ - } - - if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) { - _Per_CPU_Perform_jobs( cpu_self ); - } + return _SMP_Process_message( cpu_self, message ); } return message; diff --git a/cpukit/score/src/percpu.c b/cpukit/score/src/percpu.c index 7fbc1c8637..3bacbd7f55 100644 --- a/cpukit/score/src/percpu.c +++ b/cpukit/score/src/percpu.c @@ -3,8 +3,8 @@ * * @ingroup RTEMSScorePerCPU * - * @brief This source file contains a definition of ::_Per_CPU_Information and - * the implementation of _Per_CPU_State_change(). + * @brief This source file contains the uniprocessor definition of + * ::_Per_CPU_Information and some static assertions. */ /* @@ -21,10 +21,6 @@ #endif #include -#include -#include -#include -#include RTEMS_STATIC_ASSERT( sizeof( CPU_Uint32ptr ) >= sizeof( uintptr_t ), @@ -36,148 +32,7 @@ RTEMS_STATIC_ASSERT( CPU_Uint32ptr_greater_equal_uint32_t ); -#if defined(RTEMS_SMP) - -ISR_LOCK_DEFINE( static, _Per_CPU_State_lock, "Per-CPU State" ) - -static void _Per_CPU_State_acquire( ISR_lock_Context *lock_context ) -{ - _ISR_lock_ISR_disable_and_acquire( &_Per_CPU_State_lock, lock_context ); -} - -static void _Per_CPU_State_release( ISR_lock_Context *lock_context ) -{ - _ISR_lock_Release_and_ISR_enable( &_Per_CPU_State_lock, lock_context ); -} - -static void _Per_CPU_State_busy_wait( - Per_CPU_Control *cpu, - Per_CPU_State new_state -) -{ - Per_CPU_State state; - - state = _Per_CPU_Get_state( cpu ); - - switch ( new_state ) { - case PER_CPU_STATE_REQUEST_START_MULTITASKING: - while ( - state != PER_CPU_STATE_READY_TO_START_MULTITASKING - && state != PER_CPU_STATE_SHUTDOWN - ) { - _Per_CPU_Perform_jobs( cpu ); - state = _Per_CPU_Get_state( cpu ); - } - break; - case PER_CPU_STATE_UP: - while ( - state != PER_CPU_STATE_REQUEST_START_MULTITASKING - && state != PER_CPU_STATE_SHUTDOWN - ) { - _Per_CPU_Perform_jobs( cpu ); - state = _Per_CPU_Get_state( cpu ); - } - break; - default: - /* No need to wait */ - break; - } -} - -static Per_CPU_State _Per_CPU_State_get_next( - Per_CPU_State current_state, - Per_CPU_State new_state -) -{ - switch ( current_state ) { - case PER_CPU_STATE_INITIAL: - switch ( new_state ) { - case PER_CPU_STATE_READY_TO_START_MULTITASKING: - case PER_CPU_STATE_SHUTDOWN: - /* Change is acceptable */ - break; - default: - new_state = PER_CPU_STATE_SHUTDOWN; - break; - } - break; - case PER_CPU_STATE_READY_TO_START_MULTITASKING: - switch ( new_state ) { - case PER_CPU_STATE_REQUEST_START_MULTITASKING: - case PER_CPU_STATE_SHUTDOWN: - /* Change is acceptable */ - break; - default: - new_state = PER_CPU_STATE_SHUTDOWN; - break; - } - break; - case PER_CPU_STATE_REQUEST_START_MULTITASKING: - switch ( new_state ) { - case PER_CPU_STATE_UP: - case PER_CPU_STATE_SHUTDOWN: - /* Change is acceptable */ - break; - default: - new_state = PER_CPU_STATE_SHUTDOWN; - break; - } - break; - default: - new_state = PER_CPU_STATE_SHUTDOWN; - break; - } - - return new_state; -} - -void _Per_CPU_State_change( - Per_CPU_Control *cpu, - Per_CPU_State new_state -) -{ - ISR_lock_Context lock_context; - Per_CPU_State next_state; - - _Per_CPU_State_busy_wait( cpu, new_state ); - - _Per_CPU_State_acquire( &lock_context ); - - next_state = _Per_CPU_State_get_next( _Per_CPU_Get_state( cpu ), new_state ); - _Per_CPU_Set_state( cpu, next_state ); - - if ( next_state == PER_CPU_STATE_SHUTDOWN ) { - uint32_t cpu_max = rtems_configuration_get_maximum_processors(); - uint32_t cpu_index; - - for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { - Per_CPU_Control *cpu_other = _Per_CPU_Get_by_index( cpu_index ); - - if ( cpu_other != cpu ) { - switch ( _Per_CPU_Get_state( cpu_other ) ) { - case PER_CPU_STATE_UP: - _SMP_Send_message( cpu_index, SMP_MESSAGE_SHUTDOWN ); - break; - default: - /* Nothing to do */ - break; - } - - _Per_CPU_Set_state( cpu_other, PER_CPU_STATE_SHUTDOWN ); - } - } - } - - _Per_CPU_State_release( &lock_context ); - - if ( - next_state == PER_CPU_STATE_SHUTDOWN - && new_state != PER_CPU_STATE_SHUTDOWN - ) { - _SMP_Fatal( SMP_FATAL_SHUTDOWN ); - } -} -#else +#if !defined(RTEMS_SMP) /* * On single core systems, we can efficiently directly access a single * statically allocated per cpu structure. And the fields are initialized diff --git a/cpukit/score/src/smp.c b/cpukit/score/src/smp.c index 0488464da0..2c37cd6884 100644 --- a/cpukit/score/src/smp.c +++ b/cpukit/score/src/smp.c @@ -5,11 +5,12 @@ * * @brief This source file contains the definition of ::_SMP_Online_processors * and ::_SMP_Processor_maximum and the implementation of - * _SMP_Handler_initialize(), _SMP_Request_shutdown(), - * _SMP_Request_start_multitasking(), _SMP_Send_message(), - * _SMP_Send_message_broadcast(), _SMP_Send_message_multicast(), - * _SMP_Should_start_processor(), and - * _SMP_Start_multitasking_on_secondary_processor(). + * _SMP_Handler_initialize(), _SMP_Process_message(), + * _SMP_Request_shutdown(), _SMP_Request_start_multitasking(), + * _SMP_Send_message(), _SMP_Send_message_broadcast(), + * _SMP_Send_message_multicast(), _SMP_Should_start_processor(), + * _SMP_Start_multitasking_on_secondary_processor(), and + * _SMP_Try_to_process_message(). */ /* @@ -35,6 +36,15 @@ #error "deferred FP switch not implemented for SMP" #endif +/** + * @brief Indicates if the system is ready to start multitasking. + * + * Only the boot processor is allowed to change this object. If the object has + * a non-zero value and no fatal error occurred, then secondary processors + * should call _Thread_Start_multitasking() to start multiprocessing. + */ +static Atomic_Uint _SMP_Ready_to_start_multitasking; + Processor_mask _SMP_Online_processors; uint32_t _SMP_Processor_maximum; @@ -159,20 +169,38 @@ void _SMP_Request_start_multitasking( void ) uint32_t cpu_max; uint32_t cpu_index; - cpu_self = _Per_CPU_Get(); - _Per_CPU_State_change( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); - cpu_max = _SMP_Get_processor_maximum(); + cpu_self = _Per_CPU_Get(); + /* + * Wait until all other online processors reached the + * PER_CPU_STATE_READY_TO_START_MULTITASKING state. The waiting is done + * without a timeout. If secondary processors cannot reach this state, then + * it is expected that they indicate this failure with an + * ::SMP_MESSAGE_SHUTDOWN message or reset the system. + */ for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { Per_CPU_Control *cpu; cpu = _Per_CPU_Get_by_index( cpu_index ); - if ( _Per_CPU_Is_processor_online( cpu ) ) { - _Per_CPU_State_change( cpu, PER_CPU_STATE_REQUEST_START_MULTITASKING ); + if ( cpu != cpu_self && _Per_CPU_Is_processor_online( cpu ) ) { + while ( + _Per_CPU_Get_state( cpu ) != PER_CPU_STATE_READY_TO_START_MULTITASKING + ) { + _SMP_Try_to_process_message( + cpu_self, + _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) + ); + } } } + + _Atomic_Store_uint( + &_SMP_Ready_to_start_multitasking, + 1U, + ATOMIC_ORDER_RELEASE + ); } bool _SMP_Should_start_processor( uint32_t cpu_index ) @@ -183,6 +211,22 @@ bool _SMP_Should_start_processor( uint32_t cpu_index ) return _Scheduler_Should_start_processor( assignment ); } +static void _SMP_Wait_for_start_multitasking( Per_CPU_Control *cpu_self ) +{ + unsigned int ready; + + do { + _SMP_Try_to_process_message( + cpu_self, + _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) + ); + ready = _Atomic_Load_uint( + &_SMP_Ready_to_start_multitasking, + ATOMIC_ORDER_ACQUIRE + ); + } while ( ready == 0U ); +} + void _SMP_Start_multitasking_on_secondary_processor( Per_CPU_Control *cpu_self ) @@ -199,28 +243,96 @@ void _SMP_Start_multitasking_on_secondary_processor( _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_UNASSIGNED_PROCESSOR ); } - _Per_CPU_State_change( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); - + _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); + _SMP_Wait_for_start_multitasking( cpu_self ); _Thread_Start_multitasking(); } void _SMP_Request_shutdown( void ) { ISR_Level level; + uint32_t cpu_max; + uint32_t cpu_index_self; + uint32_t cpu_index; _ISR_Local_disable( level ); (void) level; - _Per_CPU_State_change( _Per_CPU_Get(), PER_CPU_STATE_SHUTDOWN ); + cpu_max = _SMP_Processor_configured_maximum; + cpu_index_self = _SMP_Get_current_processor(); + + for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { + Per_CPU_Control *cpu; + + cpu = _Per_CPU_Get_by_index( cpu_index ); + + if ( cpu_index == cpu_index_self ) { + _Per_CPU_Set_state( cpu, PER_CPU_STATE_SHUTDOWN ); + } else { + _Atomic_Fetch_or_ulong( + &cpu->message, + SMP_MESSAGE_SHUTDOWN, + ATOMIC_ORDER_RELEASE + ); + + if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) { + _CPU_SMP_Send_interrupt( cpu_index ); + } + } + } +} + +long unsigned _SMP_Process_message( + Per_CPU_Control *cpu_self, + long unsigned message +) +{ + if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) { + /* Check the state to prevent recursive shutdowns */ + if ( _Per_CPU_Get_state( cpu_self ) != PER_CPU_STATE_SHUTDOWN ) { + _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_SHUTDOWN ); + _SMP_Fatal( SMP_FATAL_SHUTDOWN_RESPONSE ); + } + } + + if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) { + _Per_CPU_Perform_jobs( cpu_self ); + } +} + +void _SMP_Try_to_process_message( + Per_CPU_Control *cpu_self, + unsigned long message +) +{ + if ( message != 0 ) { + /* + * Fetch the current message. Only a read-modify-write operation + * guarantees that we get an up to date message. This is especially + * important if the function was called using SMP_MESSAGE_FORCE_PROCESSING. + */ + message = _Atomic_Exchange_ulong( + &cpu_self->message, + 0, + ATOMIC_ORDER_ACQUIRE + ); + + _SMP_Process_message( cpu_self, message ); + } } void _SMP_Send_message( uint32_t cpu_index, unsigned long message ) { Per_CPU_Control *cpu = _Per_CPU_Get_by_index( cpu_index ); - _Atomic_Fetch_or_ulong( &cpu->message, message, ATOMIC_ORDER_RELEASE ); + (void) _Atomic_Fetch_or_ulong( + &cpu->message, message, + ATOMIC_ORDER_RELEASE + ); - _CPU_SMP_Send_interrupt( cpu_index ); + if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) { + _CPU_SMP_Send_interrupt( cpu_index ); + } } void _SMP_Send_message_broadcast( unsigned long message ) diff --git a/cpukit/score/src/smpmulticastaction.c b/cpukit/score/src/smpmulticastaction.c index 5d65ef14ca..8dbdef80c7 100644 --- a/cpukit/score/src/smpmulticastaction.c +++ b/cpukit/score/src/smpmulticastaction.c @@ -92,27 +92,6 @@ void _Per_CPU_Add_job( Per_CPU_Control *cpu, Per_CPU_Job *job ) _Per_CPU_Jobs_release_and_ISR_enable( cpu, &lock_context ); } -static void _Per_CPU_Try_perform_jobs( Per_CPU_Control *cpu_self ) -{ - unsigned long message; - - message = _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ); - - if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) { - bool success; - - success = _Atomic_Compare_exchange_ulong( - &cpu_self->message, &message, - message & ~SMP_MESSAGE_PERFORM_JOBS, ATOMIC_ORDER_RELAXED, - ATOMIC_ORDER_RELAXED - ); - - if ( success ) { - _Per_CPU_Perform_jobs( cpu_self ); - } - } -} - void _Per_CPU_Wait_for_job( const Per_CPU_Control *cpu, const Per_CPU_Job *job @@ -122,17 +101,22 @@ void _Per_CPU_Wait_for_job( _Atomic_Load_ulong( &job->done, ATOMIC_ORDER_ACQUIRE ) != PER_CPU_JOB_DONE ) { + Per_CPU_Control *cpu_self; + switch ( _Per_CPU_Get_state( cpu ) ) { case PER_CPU_STATE_INITIAL: case PER_CPU_STATE_READY_TO_START_MULTITASKING: - case PER_CPU_STATE_REQUEST_START_MULTITASKING: case PER_CPU_STATE_UP: /* - * Calling this function with the current processor is intentional. - * We have to perform our own jobs here in case inter-processor - * interrupts are not working. + * Calling this function with the current processor is intentional. We + * have to perform our own jobs here in case inter-processor interrupts + * are not working. */ - _Per_CPU_Try_perform_jobs( _Per_CPU_Get() ); + cpu_self = _Per_CPU_Get(); + _SMP_Try_to_process_message( + cpu_self, + _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) + ); break; default: _SMP_Fatal( SMP_FATAL_WRONG_CPU_STATE_TO_PERFORM_JOBS ); diff --git a/cpukit/score/src/threadstartmultitasking.c b/cpukit/score/src/threadstartmultitasking.c index 094a535394..9fa52a58ac 100644 --- a/cpukit/score/src/threadstartmultitasking.c +++ b/cpukit/score/src/threadstartmultitasking.c @@ -22,6 +22,7 @@ #include #include +#include void _Thread_Start_multitasking( void ) { @@ -29,7 +30,8 @@ void _Thread_Start_multitasking( void ) Thread_Control *heir; #if defined(RTEMS_SMP) - _Per_CPU_State_change( cpu_self, PER_CPU_STATE_UP ); + _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_UP ); + _SMP_Try_to_process_message( cpu_self, SMP_MESSAGE_FORCE_PROCESSING ); /* * Threads begin execution in the _Thread_Handler() function. This -- cgit v1.2.3