2011-05-26 Jennifer Averett <Jennifer.Averett@OARcorp.com>

	PR 1796/cpukit
	* sapi/src/exshutdown.c, score/include/rtems/score/percpu.h,
	score/include/rtems/score/smp.h, score/src/smp.c,
	score/src/threaddispatch.c, score/src/threadhandler.c: Added SMP
	interprocess communications.

7 files changed, 227 insertions, 40 deletions

diff --git a/cpukit/ChangeLog b/cpukit/ChangeLog
index 5a4e3bfcc0..a53fa7e46d 100644
--- a/cpukit/ChangeLog
+++ b/cpukit/ChangeLog
@@ -1,3 +1,11 @@
+2011-05-26	Jennifer Averett <Jennifer.Averett@OARcorp.com>
+
+	PR 1796/cpukit
+	* sapi/src/exshutdown.c, score/include/rtems/score/percpu.h,
+	score/include/rtems/score/smp.h, score/src/smp.c,
+	score/src/threaddispatch.c, score/src/threadhandler.c: Added SMP
+	interprocess communications.
+
 2011-05-24	Ralf Corsépius <ralf.corsepius@rtems.org>
 
 	* score/include/rtems/score/percpu.h,
diff --git a/cpukit/sapi/src/exshutdown.c b/cpukit/sapi/src/exshutdown.c
index 5b0a5a1de1..1de3b843e2 100644
--- a/cpukit/sapi/src/exshutdown.c
+++ b/cpukit/sapi/src/exshutdown.c
@@ -1,7 +1,7 @@
 /*
  *  Initialization Manager
  *
- *  COPYRIGHT (c) 1989-1999.
+ *  COPYRIGHT (c) 1989-2011.
  *  On-Line Applications Research Corporation (OAR).
  *
  *  The license and distribution terms for this file may be
@@ -20,6 +20,10 @@
 #include <rtems/score/thread.h>
 #include <rtems/score/interr.h>
 
+#if defined(RTEMS_SMP)
+  #include <rtems/score/smp.h>
+#endif
+
 /*
  *  rtems_shutdown_executive
  *
@@ -37,6 +41,9 @@ void rtems_shutdown_executive(
 )
 {
   if ( _System_state_Is_up( _System_state_Get() ) ) {
+    #if defined(RTEMS_SMP)
+      _SMP_Request_other_cores_to_shutdown();
+    #endif
     _System_state_Set( SYSTEM_STATE_SHUTDOWN );
     _Thread_Stop_multitasking();
   }
diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 2631235f06..0e116800a8 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -79,9 +79,15 @@ typedef enum {
 
   /**
    *  This defines the constant used to indicate that the cpu code has
+   *  completed basic initialization and awaits further commands.
+   */
+  RTEMS_BSP_SMP_CPU_UP = 3,
+
+  /**
+   *  This defines the constant used to indicate that the cpu code has
    *  shut itself down.
    */
-  RTEMS_BSP_SMP_CPU_SHUTDOWN = 3
+  RTEMS_BSP_SMP_CPU_SHUTDOWN = 4
 } bsp_smp_cpu_state;
 
 /**
diff --git a/cpukit/score/include/rtems/score/smp.h b/cpukit/score/include/rtems/score/smp.h
index 55545b66ac..727a139e72 100644
--- a/cpukit/score/include/rtems/score/smp.h
+++ b/cpukit/score/include/rtems/score/smp.h
@@ -87,6 +87,14 @@ void _SMP_Broadcast_message(
 );
 
 /**
+ *  @brief Request Other Cores to Perform First Context Switch
+ *
+ *  Send message to other cores requesting them to perform
+ *  their first context switch operation.
+ */
+void _SMP_Request_other_cores_to_perform_first_context_switch(void);
+
+/**
  *  @brief Request Dispatch on Other Cores
  *
  *  Send message to other cores requesting them to perform
diff --git a/cpukit/score/src/smp.c b/cpukit/score/src/smp.c
index 704a22fd36..4fbe4176e2 100644
--- a/cpukit/score/src/smp.c
+++ b/cpukit/score/src/smp.c
@@ -19,30 +19,35 @@
 #include <rtems/score/thread.h>
 
 #if defined(RTEMS_SMP)
-#define SMP_DEBUG
+  #define RTEMS_DEBUG
+#endif
 
-#if defined(SMP_DEBUG)
+#if defined(RTEMS_DEBUG)
   #include <rtems/bspIo.h>
 #endif
 
+/*
+ *  Process request to switch to the first task on a secondary core.
+ */
 void rtems_smp_run_first_task(int cpu)
 {
   Thread_Control *heir;
 
   /*
-   *  This CPU has an heir thread so we need to dispatch it.
-   */
-  heir = _Thread_Heir;
-
-  /*
-   *  This is definitely a hack until we have SMP scheduling.  Since there
-   *  is only one executing and heir right now, we have to fake this out.
+   *  The Scheduler will have selected the heir thread for each CPU core.
+   *  Now we have been requested to perform the first context switch.  So
+   *  force a switch to the designated heir and make it executing on 
+   *  THIS core.
    */
-  _Thread_Dispatch_set_disable_level(1);
+  heir              = _Thread_Heir;
   _Thread_Executing = heir;
+
   _CPU_Context_switch_to_first_task_smp( &heir->Registers );
 }
 
+/*
+ *  Process request to initialize this secondary core.
+ */
 void rtems_smp_secondary_cpu_initialize(void)
 {
   int cpu;
@@ -51,7 +56,7 @@ void rtems_smp_secondary_cpu_initialize(void)
 
   bsp_smp_secondary_cpu_initialize(cpu);
 
-  #if defined(SMP_DEBUG)
+  #if defined(RTEMS_DEBUG)
     printk( "Made it to %d -- ", cpu );
   #endif
 
@@ -63,15 +68,26 @@ void rtems_smp_secondary_cpu_initialize(void)
   _Per_CPU_Information[cpu].state = RTEMS_BSP_SMP_CPU_INITIALIZED;
 
   /*
-   *  HACK: Should not have to enable interrupts in real system here.
-   *        It should happen as part of switching to the first task.
+   *  With this secondary core out of reset, we can wait for the
+   *  request to switch to the first task.
+   *
+   *  XXX When SMP ISR code is complete, do we want interrupts on
+   *  XXX or off at this point?
    */
-
-  _Per_CPU_Information[cpu].isr_nest_level = 1;
   _ISR_Set_level( 0 );
-  while(1) ;
+  while(1) {
+    bsp_smp_wait_for(
+      (volatile unsigned int *)&_Per_CPU_Information[cpu].message,
+      RTEMS_BSP_SMP_FIRST_TASK,
+      10000
+    );
+  }
 }
 
+/*
+ *  Process an interrupt processor interrupt which indicates a request
+ *  from another core.
+ */
 void rtems_smp_process_interrupt(void)
 {
   int        cpu;
@@ -80,31 +96,53 @@ void rtems_smp_process_interrupt(void)
 
   cpu = bsp_smp_processor_id();
 
-  level = _SMP_lock_spinlock_simple_Obtain( &_Per_CPU_Information[cpu].lock );
-    message = _Per_CPU_Information[cpu].message;
-    _Per_CPU_Information[cpu].message &= ~message;
-  _SMP_lock_spinlock_simple_Release( &_Per_CPU_Information[cpu].lock, level );
+  level = _SMP_lock_Simple_Spinlock_Obtain( &_Per_CPU_Information[cpu].lock );
+  message = _Per_CPU_Information[cpu].message;
 
-  #if defined(SMP_DEBUG)
+  #if defined(RTEMS_DEBUG)
     {
       void *sp = __builtin_frame_address(0);
-      if ( !(message & RTEMS_BSP_SMP_SHUTDOWN) )
-	printk( "ISR on CPU %d -- (0x%02x) (0x%p)\n", cpu, message, sp );
-      printk( "Dispatch level %d\n", _Thread_Dispatch_disable_level );
+      if ( !(message & RTEMS_BSP_SMP_SHUTDOWN) ) {
+        printk( "ISR on CPU %d -- (0x%02x) (0x%p)\n", cpu, message, sp );
+	if ( message & RTEMS_BSP_SMP_CONTEXT_SWITCH_NECESSARY )
+	  printk( "context switch necessary\n" );
+	if ( message & RTEMS_BSP_SMP_SIGNAL_TO_SELF )
+	  printk( "signal to self\n" );
+	if ( message & RTEMS_BSP_SMP_SHUTDOWN )
+	  printk( "shutdown\n" );
+	if ( message & RTEMS_BSP_SMP_FIRST_TASK )
+	  printk( "switch to first task\n" );
+      }
+ 
+      printk( "Dispatch level %d\n", _Thread_Dispatch_get_disable_level() );
     }
   #endif
 
   if ( message & RTEMS_BSP_SMP_FIRST_TASK ) {
+    /*
+     * XXX Thread dispatch disable level at this point will have to be
+     * XXX revisited when Interrupts on SMP is addressed.
+     */
+    _Thread_Dispatch_disable_level--; /* undo ISR code */
     _Per_CPU_Information[cpu].isr_nest_level = 0;
-    _Per_CPU_Information[cpu].message = 0;
-    _Per_CPU_Information[cpu].state = RTEMS_BSP_SMP_CPU_INITIALIZED;
+    _Per_CPU_Information[cpu].message &= ~message;
+    _Per_CPU_Information[cpu].state = RTEMS_BSP_SMP_CPU_UP;
+
+    _SMP_lock_Simple_Spinlock_Release( &_Per_CPU_Information[cpu].lock, level );
+    _Thread_Disable_dispatch();
     rtems_smp_run_first_task(cpu);
     /* does not return */
   }
 
   if ( message & RTEMS_BSP_SMP_SHUTDOWN ) {
-    ISR_Level level;
-    _Thread_Dispatch_set_disable_level(0);
+    /*
+     * XXX Thread dispatch disable level at this point will have to be
+     * XXX revisited when Interrupts on SMP is addressed.
+     */
+    _Per_CPU_Information[cpu].message &= ~message;
+    _SMP_lock_Simple_Spinlock_Release( &_Per_CPU_Information[cpu].lock, level );
+
+    _Thread_Dispatch_disable_level--; /* undo ISR code */
     _Per_CPU_Information[cpu].isr_nest_level = 0;
     _Per_CPU_Information[cpu].state = RTEMS_BSP_SMP_CPU_SHUTDOWN;
     _ISR_Disable( level );
@@ -114,12 +152,22 @@ void rtems_smp_process_interrupt(void)
   }
 
   if ( message & RTEMS_BSP_SMP_CONTEXT_SWITCH_NECESSARY ) {
-    printk( "switch needed\n" );
-    _Per_CPU_Information[cpu].dispatch_necessary = true;
+    #if defined(RTEMS_DEBUG)
+      printk( "switch needed\n" );
+    #endif
+    /*
+     * XXX Thread dispatch disable level at this point will have to be
+     * XXX revisited when Interrupts on SMP is addressed.
+     */
+    _Per_CPU_Information[cpu].message &= ~message;
+    _SMP_lock_Simple_Spinlock_Release( &_Per_CPU_Information[cpu].lock, level );
   }
 }
 
-void rtems_smp_send_message(
+/*
+ *  Send an interrupt processor request to another cpu.
+ */
+void _SMP_Send_message(
   int       cpu,
   uint32_t  message
 )
@@ -132,7 +180,10 @@ void rtems_smp_send_message(
   bsp_smp_interrupt_cpu( cpu );
 }
 
-void rtems_smp_broadcast_message(
+/*
+ *  Send interrupt processor request to all other nodes
+ */
+void _SMP_Broadcast_message(
   uint32_t  message
 )
 {
@@ -151,4 +202,82 @@ void rtems_smp_broadcast_message(
   }
   bsp_smp_broadcast_interrupt();
 }
-#endif
+
+/*
+ *  Send interrupt processor requests to perform first context switch 
+ */
+void _SMP_Request_other_cores_to_perform_first_context_switch(void)
+{
+  int    cpu;
+
+  for (cpu=1 ; cpu < _SMP_Processor_count ; cpu++ ) {
+    _SMP_Send_message( cpu, RTEMS_BSP_SMP_FIRST_TASK );
+    while (_Per_CPU_Information[cpu].state != RTEMS_BSP_SMP_CPU_UP ) {
+      bsp_smp_wait_for(
+        (volatile unsigned int *)&_Per_CPU_Information[cpu].state,
+        RTEMS_BSP_SMP_CPU_UP,
+        10000
+      );
+    }
+  }
+}
+
+/*
+ *  Send message to other cores requesting them to perform
+ *  a thread dispatch operation.
+ */
+void _SMP_Request_other_cores_to_dispatch(void)
+{
+  int i;
+  int cpu;
+
+  cpu = bsp_smp_processor_id();
+
+  if ( !_System_state_Is_up (_System_state_Current) )
+    return;
+  for (i=1 ; i < _SMP_Processor_count ; i++ ) {
+    if ( cpu == i )
+      continue;
+    if ( _Per_CPU_Information[i].state != RTEMS_BSP_SMP_CPU_UP )
+      continue;
+    if ( !_Per_CPU_Information[i].dispatch_necessary )
+      continue;
+    _SMP_Send_message( i, RTEMS_BSP_SMP_CONTEXT_SWITCH_NECESSARY );
+    bsp_smp_wait_for(
+      (volatile unsigned int *)&_Per_CPU_Information[i].message,
+      0,
+      10000
+    );
+  }
+}
+
+/*
+ *  Send message to other cores requesting them to shutdown.
+ */
+void _SMP_Request_other_cores_to_shutdown(void)
+{
+  bool allDown;
+  int  ncpus;
+  int  cpu;
+
+  ncpus = _SMP_Processor_count;
+
+  _SMP_Broadcast_message( RTEMS_BSP_SMP_SHUTDOWN );
+
+  allDown = true;
+  for (cpu=1 ; cpu<ncpus ; cpu++ ) {
+     bsp_smp_wait_for(
+       (unsigned int *)&_Per_CPU_Information[cpu].state,
+       RTEMS_BSP_SMP_CPU_SHUTDOWN,
+       10000
+    );
+    if ( _Per_CPU_Information[cpu].state != RTEMS_BSP_SMP_CPU_SHUTDOWN )
+      allDown = false;
+  }
+  if ( !allDown )
+    printk( "All CPUs not successfully shutdown -- timed out\n" );
+  #if (RTEMS_DEBUG)
+    else 
+      printk( "All CPUs shutdown successfully\n" );
+  #endif
+}
diff --git a/cpukit/score/src/threaddispatch.c b/cpukit/score/src/threaddispatch.c
index a5b5cf94c5..38d24a5017 100644
--- a/cpukit/score/src/threaddispatch.c
+++ b/cpukit/score/src/threaddispatch.c
@@ -33,6 +33,10 @@
   #include <rtems/score/timestamp.h>
 #endif
 
+#if defined(RTEMS_SMP)
+  #include <rtems/score/smp.h>
+#endif
+
 /**
  *  _Thread_Dispatch
  *
@@ -47,18 +51,28 @@
  *    dispatch thread
  *    no dispatch thread
  */
-
 void _Thread_Dispatch( void )
 {
   Thread_Control   *executing;
   Thread_Control   *heir;
   ISR_Level         level;
 
+  _Thread_Disable_dispatch();
+  #if defined(RTEMS_SMP)
+    /*
+     *  If necessary, send dispatch request to other cores.
+     */
+    _SMP_Request_other_cores_to_dispatch();
+  #endif
+
+  /*
+   *  Now determine if we need to perform a dispatch on the current CPU.
+   */
   executing   = _Thread_Executing;
   _ISR_Disable( level );
   while ( _Thread_Dispatch_necessary == true ) {
+
     heir = _Thread_Heir;
-   _Thread_Dispatch_set_disable_level( 1 );
     _Thread_Dispatch_necessary = false;
     _Thread_Executing = heir;
 
@@ -96,7 +110,10 @@ void _Thread_Dispatch( void )
         _Thread_Time_of_last_context_switch = uptime;
       }
     #else
-      heir->cpu_time_used++;
+      {
+        TOD_Get_uptime( &_Thread_Time_of_last_context_switch );
+        heir->cpu_time_used++;
+      }
     #endif
 
     /*
diff --git a/cpukit/score/src/threadhandler.c b/cpukit/score/src/threadhandler.c
index 37185ad745..84fe9c10d3 100644
--- a/cpukit/score/src/threadhandler.c
+++ b/cpukit/score/src/threadhandler.c
@@ -2,7 +2,7 @@
  *  Thread Handler
  *
  *
- *  COPYRIGHT (c) 1989-2009.
+ *  COPYRIGHT (c) 1989-2011.
  *  On-Line Applications Research Corporation (OAR).
  *
  *  The license and distribution terms for this file may be
@@ -51,6 +51,11 @@
   #define EXECUTE_GLOBAL_CONSTRUCTORS
 #endif
 
+#if defined(RTEMS_SMP)
+  #include <rtems/score/smp.h>
+#endif
+
+
 /*PAGE
  *
  *  _Thread_Handler
@@ -138,8 +143,15 @@ void _Thread_Handler( void )
      */
     if (!doneCons) /* && (volatile void *)_init) */ {
       INIT_NAME ();
+   
+      #if defined(RTEMS_SMP)
+        _Thread_Disable_dispatch();
+          _SMP_Request_other_cores_to_perform_first_context_switch();
+        _Thread_Enable_dispatch();
+      #endif
+
     }
-  #endif
+ #endif
 
   if ( executing->Start.prototype == THREAD_START_NUMERIC ) {
     executing->Wait.return_argument =