smp: Simplify SMP initialization sequence

Delete bsp_smp_wait_for().  Other parts of the system work without
timeout, e.g. the spinlocks.  Using a timeout here does not make the
system more robust.

Delete bsp_smp_cpu_state and replace it with Per_CPU_State.  The
Per_CPU_State follows the Score naming conventions.  Add
_Per_CPU_Change_state() and _Per_CPU_Wait_for_state() functions to
change and observe states.

Use Per_CPU_State in Per_CPU_Control instead of the anonymous integer.

Add _CPU_Processor_event_broadcast() and _CPU_Processor_event_receive()
functions provided by the CPU port.  Use these functions in
_Per_CPU_Change_state() and _Per_CPU_Wait_for_state().

Add prototype for _SMP_Send_message().

Delete RTEMS_BSP_SMP_FIRST_TASK message.  The first context switch is
now performed in rtems_smp_secondary_cpu_initialize().  Issuing the
first context switch in the context of the inter-processor interrupt is
not possible on systems with a modern interrupt controller.  Such an
interrupt controler usually requires a handshake protocol with interrupt
acknowledge and end of interrupt signals.  A direct context switch in an
interrupt handler circumvents the interrupt processing epilogue and may
leave the system in an inconsistent state.

Release lock in rtems_smp_process_interrupt() even if no message was
delivered.  This prevents deadlock of the system.

Simplify and format _SMP_Send_message(),
_SMP_Request_other_cores_to_perform_first_context_switch(),
_SMP_Request_other_cores_to_dispatch() and
_SMP_Request_other_cores_to_shutdown().

5 files changed, 8 insertions, 104 deletions

diff --git a/c/src/lib/libbsp/i386/shared/smp/smp-imps.c b/c/src/lib/libbsp/i386/shared/smp/smp-imps.c
index 62ce8aca27..916379ed02 100644
--- a/c/src/lib/libbsp/i386/shared/smp/smp-imps.c
+++ b/c/src/lib/libbsp/i386/shared/smp/smp-imps.c
@@ -71,7 +71,6 @@
 #define PHYS_TO_VIRTUAL(x)    /* convert physical address "x" to virtual */
 #define VIRTUAL_TO_PHYS(x)    /* convert virtual address "x" to physical */
 #define UDELAY(x)             /* delay roughly at least "x" microsecs */
-#define TEST_BOOTED(x)        /* test bootaddr x to see if CPU started */
 #define READ_MSR_LO(x)        /* Read MSR low function */
 #else
 #include <string.h>
@@ -125,9 +124,6 @@ static void UDELAY(int x)
 #define READ_MSR_LO(_x) \
   (unsigned int)(read_msr(_x) & 0xffffffff)
 
-#define TEST_BOOTED(_cpu) \
-  (_Per_CPU_Information[_cpu].state == RTEMS_BSP_SMP_CPU_INITIALIZED)
-
 static inline unsigned long long read_msr(unsigned int msr)
 {
   unsigned long long value;
@@ -311,22 +307,6 @@ boot_cpu(imps_processor *proc)
   }
 
   /*
-   *  Check to see if other processor has started.
-   */
-  bsp_smp_wait_for(
-    (volatile unsigned int *)&_Per_CPU_Information[imps_num_cpus].state,
-    RTEMS_BSP_SMP_CPU_INITIALIZED,
-    1600
-  );
-  if ( _Per_CPU_Information[imps_num_cpus].state ==
-        RTEMS_BSP_SMP_CPU_INITIALIZED )
-    printk("#%d  Application Processor (AP)", imps_num_cpus);
-  else {
-    printk("CPU Not Responding, DISABLED");
-    success = 0;
-  }
-
-  /*
    *  Generic CPU startup sequence ends here, the rest is cleanup.
    */
 
@@ -835,32 +815,3 @@ void bsp_smp_broadcast_interrupt(void)
   /* Single broadcast interrupt */
   send_ipi( 0, LAPIC_ICR_DS_ALLEX | 0x30 );
 }
-
-void bsp_smp_wait_for(
-  volatile unsigned int *address,
-  unsigned int           desired,
-  int                    maximum_usecs
-)
-{
-  int iterations;
-  volatile int i;
-  volatile unsigned int *p = (volatile unsigned int *)address;
-
-  for (iterations=0 ;  iterations < maximum_usecs ; iterations++ ) {
-    if ( *p == desired )
-      break;
-    #ifdef __SSE3__
-      __builtin_ia32_monitor( (const void *)address, 0, 0 );
-      if ( *p == desired )
-        break;
-      __builtin_ia32_mwait( 0, 0 );
-    #endif
-
-    /*
-     *  Until i386 ms delay does not depend upon the clock we
-     *  will use this less sophisticated delay. 
-     */
-    for(i=5000; i>0; i--)
-      ;
-  }
-}
diff --git a/c/src/lib/libbsp/shared/smp/bspsmp_wait_for.c b/c/src/lib/libbsp/shared/smp/bspsmp_wait_for.c
deleted file mode 100644
index 8c4dcf75c1..0000000000
--- a/c/src/lib/libbsp/shared/smp/bspsmp_wait_for.c
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- *
- *  COPYRIGHT (c) 1989-2011.
- *  On-Line Applications Research Corporation (OAR).
- *
- *  The license and distribution terms for this file may be
- *  found in the file LICENSE in this distribution or at
- *  http://www.rtems.com/license/LICENSE.
- */
-
-void rtems_bsp_delay( int usec );
-
-void bsp_smp_wait_for(
-  volatile unsigned int *address,
-  unsigned int           desired,
-  int                    maximum_usecs
-)
-{
-  int iterations;
-  volatile unsigned int *p = address;
-
-  for (iterations=0 ;  iterations < maximum_usecs ; iterations++ ) {
-    if ( *p == desired )
-      break;
-
-    rtems_bsp_delay( 1 );
-  }
-}
diff --git a/c/src/lib/libbsp/sparc/erc32/Makefile.am b/c/src/lib/libbsp/sparc/erc32/Makefile.am
index ddb8d94c3b..988f3ec88a 100644
--- a/c/src/lib/libbsp/sparc/erc32/Makefile.am
+++ b/c/src/lib/libbsp/sparc/erc32/Makefile.am
@@ -66,8 +66,7 @@ libbsp_a_SOURCES += \
     ../../shared/src/irq-shell.c
 
 if HAS_SMP
-libbsp_a_SOURCES += ../../shared/smp/getcpuid.c ../../shared/smp/smp_stub.c \
-    ../../shared/smp/bspsmp_wait_for.c
+libbsp_a_SOURCES += ../../shared/smp/getcpuid.c ../../shared/smp/smp_stub.c
 endif
 
 if HAS_NETWORKING
diff --git a/c/src/lib/libbsp/sparc/leon2/Makefile.am b/c/src/lib/libbsp/sparc/leon2/Makefile.am
index 9699bf1a32..ae6dfb0155 100644
--- a/c/src/lib/libbsp/sparc/leon2/Makefile.am
+++ b/c/src/lib/libbsp/sparc/leon2/Makefile.am
@@ -119,8 +119,7 @@ libbsp_a_SOURCES += ../../sparc/shared/i2c/i2cmst.c
 libbsp_a_SOURCES += timer/timer.c
 
 if HAS_SMP
-libbsp_a_SOURCES += ../../shared/smp/getcpuid.c ../../shared/smp/smp_stub.c \
-    ../../shared/smp/bspsmp_wait_for.c
+libbsp_a_SOURCES += ../../shared/smp/getcpuid.c ../../shared/smp/smp_stub.c
 endif
 
 if HAS_NETWORKING
diff --git a/c/src/lib/libbsp/sparc/leon3/smp/smp_leon3.c b/c/src/lib/libbsp/sparc/leon3/smp/smp_leon3.c
index 6a9c189ec2..90818194cd 100644
--- a/c/src/lib/libbsp/sparc/leon3/smp/smp_leon3.c
+++ b/c/src/lib/libbsp/sparc/leon3/smp/smp_leon3.c
@@ -33,7 +33,7 @@ static inline unsigned int sparc_leon3_get_cctrl( void )
   return v;
 }
 
-rtems_isr bsp_ap_ipi_isr(
+static rtems_isr bsp_ap_ipi_isr(
   rtems_vector_number vector
 )
 {
@@ -99,10 +99,11 @@ uint32_t bsp_smp_initialize( uint32_t configured_cpu_count )
     bsp_smp_delay( 1000000 );
     #if defined(RTEMS_DEBUG)
       printk(
-	"CPU %d is %s\n",
-	cpu,
-	((_Per_CPU_Information[cpu].state == RTEMS_BSP_SMP_CPU_INITIALIZED) ?
-	   "online" : "offline")
+        "CPU %d is %s\n",
+        cpu,
+        _Per_CPU_Information[cpu].state
+          == PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING ?
+            "online" : "offline"
       );
     #endif
   }
@@ -160,21 +161,3 @@ void bsp_smp_delay( int max )
 {
    __delay( max );
 }
-
-void bsp_smp_wait_for(
-  volatile unsigned int *address,
-  unsigned int           desired,
-  int                    maximum_usecs
-)
-{
-  int iterations;
-  volatile unsigned int *p = address;
-
-  for (iterations=0 ;  iterations < maximum_usecs ; iterations++ ) {
-    if ( *p == desired )
-      break;
-    bsp_smp_delay( 5000 );
-  }
-}
-
-