smp: Simplify SMP initialization sequence

Delete bsp_smp_wait_for().  Other parts of the system work without
timeout, e.g. the spinlocks.  Using a timeout here does not make the
system more robust.

Delete bsp_smp_cpu_state and replace it with Per_CPU_State.  The
Per_CPU_State follows the Score naming conventions.  Add
_Per_CPU_Change_state() and _Per_CPU_Wait_for_state() functions to
change and observe states.

Use Per_CPU_State in Per_CPU_Control instead of the anonymous integer.

Add _CPU_Processor_event_broadcast() and _CPU_Processor_event_receive()
functions provided by the CPU port.  Use these functions in
_Per_CPU_Change_state() and _Per_CPU_Wait_for_state().

Add prototype for _SMP_Send_message().

Delete RTEMS_BSP_SMP_FIRST_TASK message.  The first context switch is
now performed in rtems_smp_secondary_cpu_initialize().  Issuing the
first context switch in the context of the inter-processor interrupt is
not possible on systems with a modern interrupt controller.  Such an
interrupt controler usually requires a handshake protocol with interrupt
acknowledge and end of interrupt signals.  A direct context switch in an
interrupt handler circumvents the interrupt processing epilogue and may
leave the system in an inconsistent state.

Release lock in rtems_smp_process_interrupt() even if no message was
delivered.  This prevents deadlock of the system.

Simplify and format _SMP_Send_message(),
_SMP_Request_other_cores_to_perform_first_context_switch(),
_SMP_Request_other_cores_to_dispatch() and
_SMP_Request_other_cores_to_shutdown().

1 files changed, 72 insertions, 17 deletions

diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 5469d25c5d..735b422c54 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -60,32 +60,71 @@ extern "C" {
 typedef struct Thread_Control_struct Thread_Control;
 #endif
 
+#ifdef RTEMS_SMP
+
 typedef enum {
+  /**
+   * @brief The per CPU controls are initialized to zero.
+   *
+   * In this state the only valid field of the per CPU controls for secondary
+   * processors is the per CPU state.  The secondary processors should perform
+   * their basic initialization now and change into the
+   * PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING state once this is complete.
+   *
+   * The owner of the per CPU state field is the secondary processor in this
+   * state.
+   */
+  PER_CPU_STATE_BEFORE_INITIALIZATION,
 
   /**
-   *  This defines the constant used to indicate that the cpu code is in
-   *  its initial powered up start.
+   * @brief Secondary processor is ready to begin multitasking.
+   *
+   * The secondary processor performed its basic initialization and is ready to
+   * receive inter-processor interrupts.  Interrupt delivery must be disabled
+   * in this state, but requested inter-processor interrupts must be recorded
+   * and must be delivered once the secondary processor enables interrupts for
+   * the first time.  The main processor will wait for all secondary processors
+   * to change into this state.  In case a secondary processor does not reach
+   * this state the system will not start.  The secondary processors wait now
+   * for a change into the PER_CPU_STATE_BEGIN_MULTITASKING state set by the
+   * main processor once all secondary processors reached the
+   * PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING state.
+   *
+   * The owner of the per CPU state field is the main processor in this state.
    */
-   RTEMS_BSP_SMP_CPU_INITIAL_STATE = 1,
+  PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING,
 
   /**
-   *  This defines the constant used to indicate that the cpu code has
-   *  completed basic initialization and awaits further commands.
+   * @brief Multitasking begin of secondary processor is requested.
+   *
+   * The main processor completed system initialization and is about to perform
+   * a context switch to its heir thread.  Secondary processors should now
+   * issue a context switch to the heir thread.  This normally enables
+   * interrupts on the processor for the first time.
+   *
+   * The owner of the per CPU state field is the secondary processor in this
+   * state.
    */
-   RTEMS_BSP_SMP_CPU_INITIALIZED = 2,
+  PER_CPU_STATE_BEGIN_MULTITASKING,
 
   /**
-   *  This defines the constant used to indicate that the cpu code has
-   *  completed basic initialization and awaits further commands.
+   * @brief Normal multitasking state.
+   *
+   * The owner of the per CPU state field is the secondary processor in this
+   * state.
    */
-  RTEMS_BSP_SMP_CPU_UP = 3,
+  PER_CPU_STATE_UP,
 
   /**
-   *  This defines the constant used to indicate that the cpu code has
-   *  shut itself down.
+   * @brief This is the terminal state.
+   *
+   * The owner of the per CPU state field is the secondary processor in this
+   * state.
    */
-  RTEMS_BSP_SMP_CPU_SHUTDOWN = 4
-} bsp_smp_cpu_state;
+  PER_CPU_STATE_SHUTDOWN
+} Per_CPU_State;
+
+#endif /* RTEMS_SMP */
 
 /**
  *  @brief Per CPU Core Structure
@@ -133,15 +172,21 @@ typedef struct {
     /** This element is used to lock this structure */
     SMP_lock_spinlock_simple_Control  lock;
 
-    /** This indicates that the CPU is online. */
-    uint32_t                          state;
-
     /**
      *  This is the request for the interrupt.
      *
      *  @note This may become a chain protected by atomic instructions.
      */
-    uint32_t                          message;
+    uint32_t message;
+
+    /**
+     * @brief Indicates the current state of the CPU.
+     *
+     * This field is not protected by a lock.
+     *
+     * @see _Per_CPU_Change_state() and _Per_CPU_Wait_for_state().
+     */
+    Per_CPU_State state;
   #endif
 } Per_CPU_Control;
 #endif
@@ -218,6 +263,16 @@ void _SMP_Handler_initialize(void);
  */
 void _Per_CPU_Initialize(void);
 
+void _Per_CPU_Change_state(
+  Per_CPU_Control *per_cpu,
+  Per_CPU_State new_state
+);
+
+void _Per_CPU_Wait_for_state(
+  const Per_CPU_Control *per_cpu,
+  Per_CPU_State desired_state
+);
+
 #endif
 
 /*