diff options
Diffstat (limited to 'cpukit/score/src/smp.c')
| -rw-r--r-- | cpukit/score/src/smp.c | 228 |
1 files changed, 145 insertions, 83 deletions
diff --git a/cpukit/score/src/smp.c b/cpukit/score/src/smp.c index 2a3405c6ef..75520829e1 100644 --- a/cpukit/score/src/smp.c +++ b/cpukit/score/src/smp.c @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + /** * @file * @@ -5,20 +7,37 @@ * * @brief This source file contains the definition of ::_SMP_Online_processors * and ::_SMP_Processor_maximum and the implementation of - * _SMP_Handler_initialize(), _SMP_Request_shutdown(), - * _SMP_Request_start_multitasking(), _SMP_Send_message(), - * _SMP_Send_message_broadcast(), _SMP_Send_message_multicast(), - * _SMP_Should_start_processor(), and - * _SMP_Start_multitasking_on_secondary_processor(). + * _SMP_Handler_initialize(), _SMP_Process_message(), + * _SMP_Request_shutdown(), _SMP_Request_start_multitasking(), + * _SMP_Send_message(), _SMP_Should_start_processor(), + * _SMP_Start_multitasking_on_secondary_processor(), and + * _SMP_Try_to_process_message(). */ /* * COPYRIGHT (c) 1989-2011. * On-Line Applications Research Corporation (OAR). * - * The license and distribution terms for this file may be - * found in the file LICENSE in this distribution or at - * http://www.rtems.org/license/LICENSE. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ #ifdef HAVE_CONFIG_H @@ -27,19 +46,23 @@ #include <rtems/score/smpimpl.h> #include <rtems/score/assert.h> -#include <rtems/score/memory.h> -#include <rtems/score/percpudata.h> #include <rtems/score/schedulerimpl.h> #include <rtems/score/threadimpl.h> #include <rtems/config.h> -#include <rtems/sysinit.h> - -#include <string.h> #if CPU_USE_DEFERRED_FP_SWITCH == TRUE #error "deferred FP switch not implemented for SMP" #endif +/** + * @brief Indicates if the system is ready to start multitasking. + * + * Only the boot processor is allowed to change this object. If the object has + * a non-zero value and no fatal error occurred, then secondary processors + * should call _Thread_Start_multitasking() to start multiprocessing. + */ +static Atomic_Uint _SMP_Ready_to_start_multitasking; + Processor_mask _SMP_Online_processors; uint32_t _SMP_Processor_maximum; @@ -164,20 +187,38 @@ void _SMP_Request_start_multitasking( void ) uint32_t cpu_max; uint32_t cpu_index; - cpu_self = _Per_CPU_Get(); - _Per_CPU_State_change( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); - cpu_max = _SMP_Get_processor_maximum(); + cpu_self = _Per_CPU_Get(); + /* + * Wait until all other online processors reached the + * PER_CPU_STATE_READY_TO_START_MULTITASKING state. The waiting is done + * without a timeout. If secondary processors cannot reach this state, then + * it is expected that they indicate this failure with an + * ::SMP_MESSAGE_SHUTDOWN message or reset the system. + */ for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { Per_CPU_Control *cpu; cpu = _Per_CPU_Get_by_index( cpu_index ); - if ( _Per_CPU_Is_processor_online( cpu ) ) { - _Per_CPU_State_change( cpu, PER_CPU_STATE_REQUEST_START_MULTITASKING ); + if ( cpu != cpu_self && _Per_CPU_Is_processor_online( cpu ) ) { + while ( + _Per_CPU_Get_state( cpu ) != PER_CPU_STATE_READY_TO_START_MULTITASKING + ) { + _SMP_Try_to_process_message( + cpu_self, + _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) + ); + } } } + + _Atomic_Store_uint( + &_SMP_Ready_to_start_multitasking, + 1U, + ATOMIC_ORDER_RELEASE + ); } bool _SMP_Should_start_processor( uint32_t cpu_index ) @@ -188,6 +229,22 @@ bool _SMP_Should_start_processor( uint32_t cpu_index ) return _Scheduler_Should_start_processor( assignment ); } +static void _SMP_Wait_for_start_multitasking( Per_CPU_Control *cpu_self ) +{ + unsigned int ready; + + do { + _SMP_Try_to_process_message( + cpu_self, + _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) + ); + ready = _Atomic_Load_uint( + &_SMP_Ready_to_start_multitasking, + ATOMIC_ORDER_ACQUIRE + ); + } while ( ready == 0U ); +} + void _SMP_Start_multitasking_on_secondary_processor( Per_CPU_Control *cpu_self ) @@ -196,6 +253,12 @@ void _SMP_Start_multitasking_on_secondary_processor( cpu_index_self = _Per_CPU_Get_index( cpu_self ); + /* + * Call fatal error and per-CPU job handlers with thread dispatching + * disabled. + */ + cpu_self->thread_dispatch_disable_level = 1; + if ( cpu_index_self >= rtems_configuration_get_maximum_processors() ) { _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_INVALID_PROCESSOR ); } @@ -204,7 +267,12 @@ void _SMP_Start_multitasking_on_secondary_processor( _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_UNASSIGNED_PROCESSOR ); } - _Per_CPU_State_change( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); + _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING ); + _SMP_Wait_for_start_multitasking( cpu_self ); + + if ( !_Per_CPU_Is_processor_online( cpu_self ) ) { + _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_NOT_ONLINE_PROCESSOR ); + } _Thread_Start_multitasking(); } @@ -212,92 +280,86 @@ void _SMP_Start_multitasking_on_secondary_processor( void _SMP_Request_shutdown( void ) { ISR_Level level; + uint32_t cpu_max; + uint32_t cpu_index_self; + uint32_t cpu_index; _ISR_Local_disable( level ); (void) level; - _Per_CPU_State_change( _Per_CPU_Get(), PER_CPU_STATE_SHUTDOWN ); -} + cpu_max = _SMP_Processor_configured_maximum; + cpu_index_self = _SMP_Get_current_processor(); -void _SMP_Send_message( uint32_t cpu_index, unsigned long message ) -{ - Per_CPU_Control *cpu = _Per_CPU_Get_by_index( cpu_index ); + for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { + Per_CPU_Control *cpu; - _Atomic_Fetch_or_ulong( &cpu->message, message, ATOMIC_ORDER_RELEASE ); + cpu = _Per_CPU_Get_by_index( cpu_index ); - _CPU_SMP_Send_interrupt( cpu_index ); + if ( cpu_index == cpu_index_self ) { + _Per_CPU_Set_state( cpu, PER_CPU_STATE_SHUTDOWN ); + } else { + _Atomic_Fetch_or_ulong( + &cpu->message, + SMP_MESSAGE_SHUTDOWN, + ATOMIC_ORDER_RELEASE + ); + + if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) { + _CPU_SMP_Send_interrupt( cpu_index ); + } + } + } } -void _SMP_Send_message_broadcast( unsigned long message ) +void _SMP_Process_message( Per_CPU_Control *cpu_self, long unsigned message ) { - uint32_t cpu_max; - uint32_t cpu_index_self; - uint32_t cpu_index; + if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) { + ISR_Level level; - _Assert( _Debug_Is_thread_dispatching_allowed() ); - cpu_max = _SMP_Get_processor_maximum(); - cpu_index_self = _SMP_Get_current_processor(); + _CPU_ISR_Disable( level ); + (void) level; - for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { - if ( - cpu_index != cpu_index_self - && _Processor_mask_Is_set( &_SMP_Online_processors, cpu_index ) - ) { - _SMP_Send_message( cpu_index, message ); + /* Check the state to prevent recursive shutdowns */ + if ( _Per_CPU_Get_state( cpu_self ) != PER_CPU_STATE_SHUTDOWN ) { + _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_SHUTDOWN ); + _SMP_Fatal( SMP_FATAL_SHUTDOWN_RESPONSE ); } } + + if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) { + _Per_CPU_Perform_jobs( cpu_self ); + } } -void _SMP_Send_message_multicast( - const Processor_mask *targets, - unsigned long message +void _SMP_Try_to_process_message( + Per_CPU_Control *cpu_self, + unsigned long message ) { - uint32_t cpu_max; - uint32_t cpu_index; - - cpu_max = _SMP_Get_processor_maximum(); - - for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) { - if ( _Processor_mask_Is_set( targets, cpu_index ) ) { - _SMP_Send_message( cpu_index, message ); - } + if ( message != 0 ) { + /* + * Fetch the current message. Only a read-modify-write operation + * guarantees that we get an up to date message. This is especially + * important if the function was called using SMP_MESSAGE_FORCE_PROCESSING. + */ + message = _Atomic_Exchange_ulong( + &cpu_self->message, + 0, + ATOMIC_ORDER_ACQUIRE + ); + + _SMP_Process_message( cpu_self, message ); } } -static void _Per_CPU_Data_initialize( void ) +void _SMP_Send_message( Per_CPU_Control *cpu, unsigned long message ) { - uintptr_t size; - - size = RTEMS_LINKER_SET_SIZE( _Per_CPU_Data ); - - if ( size > 0 ) { - const Memory_Information *mem; - Per_CPU_Control *cpu; - uint32_t cpu_index; - uint32_t cpu_max; + (void) _Atomic_Fetch_or_ulong( + &cpu->message, message, + ATOMIC_ORDER_RELEASE + ); - mem = _Memory_Get(); - cpu = _Per_CPU_Get_by_index( 0 ); - cpu->data = RTEMS_LINKER_SET_BEGIN( _Per_CPU_Data ); - - cpu_max = rtems_configuration_get_maximum_processors(); - - for ( cpu_index = 1 ; cpu_index < cpu_max ; ++cpu_index ) { - cpu = _Per_CPU_Get_by_index( cpu_index ); - cpu->data = _Memory_Allocate( mem, size, CPU_CACHE_LINE_BYTES ); - - if( cpu->data == NULL ) { - _Internal_error( INTERNAL_ERROR_NO_MEMORY_FOR_PER_CPU_DATA ); - } - - memcpy( cpu->data, RTEMS_LINKER_SET_BEGIN( _Per_CPU_Data ), size); - } + if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) { + _CPU_SMP_Send_interrupt( _Per_CPU_Get_index( cpu ) ); } } - -RTEMS_SYSINIT_ITEM( - _Per_CPU_Data_initialize, - RTEMS_SYSINIT_PER_CPU_DATA, - RTEMS_SYSINIT_ORDER_MIDDLE -); |