score: SMP initialization and shutdown changes

Rename _SMP_Request_other_cores_to_perform_first_context_switch() into
_SMP_Request_start_multitasking() since this requests now a multitasking
start on all configured and available processors.  The name corresponds
_Thread_Start_multitasking() and
_SMP_Start_multitasking_on_secondary_processor() actions issued in
response to this request.  Move in source file to right place.

Rename PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING into
PER_CPU_STATE_READY_TO_START_MULTITASKING.

Rename PER_CPU_STATE_BEGIN_MULTITASKING into
PER_CPU_STATE_REQUEST_START_MULTITASKING.

Rename _SMP_Request_other_cores_to_shutdown() into
_SMP_Request_shutdown().

Add a per-CPU state lock to protect all changes.  This was necessary to
offer a controlled shutdown of the system (atomic read/writes alone are
not sufficient for this kind of synchronization).

Add documentation for Per_CPU_State.

Delete debug output.

New tests smptests/smpfatal01 and smptests/smpfatal02.

18 files changed, 565 insertions, 119 deletions

diff --git a/cpukit/sapi/src/exinit.c b/cpukit/sapi/src/exinit.c
index d265455afd..ee1c7fdecc 100644
--- a/cpukit/sapi/src/exinit.c
+++ b/cpukit/sapi/src/exinit.c
@@ -210,7 +210,7 @@ void rtems_initialize_start_multitasking(void)
 {
   _System_state_Set( SYSTEM_STATE_UP );
 
-  _SMP_Request_other_cores_to_perform_first_context_switch();
+  _SMP_Request_start_multitasking();
 
   _Thread_Start_multitasking();
 
diff --git a/cpukit/score/cpu/no_cpu/rtems/score/cpu.h b/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
index 5c34e17d03..4155883f79 100644
--- a/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
+++ b/cpukit/score/cpu/no_cpu/rtems/score/cpu.h
@@ -1462,10 +1462,10 @@ CPU_Counter_ticks _CPU_Counter_difference(
 
 #ifdef RTEMS_SMP
   /**
-   * @brief Performs CPU specific SMP initialization in the context of the main
+   * @brief Performs CPU specific SMP initialization in the context of the boot
    * processor.
    *
-   * This function is invoked on the main processor by RTEMS during
+   * This function is invoked on the boot processor by RTEMS during
    * initialization.  All interrupt stacks are allocated at this point in case
    * the CPU port allocates the interrupt stacks.
    *
diff --git a/cpukit/score/include/rtems/score/percpu.h b/cpukit/score/include/rtems/score/percpu.h
index 4c46b50425..ca9185ea7a 100644
--- a/cpukit/score/include/rtems/score/percpu.h
+++ b/cpukit/score/include/rtems/score/percpu.h
@@ -70,64 +70,83 @@ typedef struct Thread_Control_struct Thread_Control;
   #error "deferred FP switch not implemented for SMP"
 #endif
 
+/**
+ * @brief State of a processor.
+ *
+ * The processor state controls the life cycle of processors at the lowest
+ * level.  No multi-threading or other high-level concepts matter here.
+ *
+ * State changes must be initiated via _Per_CPU_Change_state().  This function
+ * may not return in case someone requested a shutdown.  The
+ * _SMP_Send_message() function will be used to notify other processors about
+ * state changes if the other processor is in the up state.
+ *
+ * Due to the sequential nature of the basic system initialization one
+ * processor has a special role.  It is the processor executing the boot_card()
+ * function.  This processor is called the boot processor.  All other
+ * processors are called secondary.
+ *
+ * @dot
+ * digraph states {
+ *   i [label="PER_CPU_STATE_INITIAL"];
+ *   rdy [label="PER_CPU_STATE_READY_TO_START_MULTITASKING"];
+ *   reqsm [label="PER_CPU_STATE_REQUEST_START_MULTITASKING"];
+ *   u [label="PER_CPU_STATE_UP"];
+ *   s [label="PER_CPU_STATE_SHUTDOWN"];
+ *   i -> rdy [label="processor\ncompleted initialization"];
+ *   rdy -> reqsm [label="boot processor\ncompleted initialization"];
+ *   reqsm -> u [label="processor\nstarts multitasking"];
+ *   i -> s;
+ *   rdy -> s;
+ *   reqsm -> s;
+ *   u -> s;
+ * }
+ * @enddot
+ */
 typedef enum {
   /**
    * @brief The per CPU controls are initialized to zero.
    *
-   * In this state the only valid field of the per CPU controls for secondary
-   * processors is the per CPU state.  The secondary processors should perform
-   * their basic initialization now and change into the
-   * PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING state once this is complete.
-   *
-   * The owner of the per CPU state field is the secondary processor in this
-   * state.
+   * The boot processor executes the sequential boot code in this state.  The
+   * secondary processors should perform their basic initialization now and
+   * change into the PER_CPU_STATE_READY_TO_START_MULTITASKING state once this
+   * is complete.
    */
-  PER_CPU_STATE_BEFORE_INITIALIZATION,
+  PER_CPU_STATE_INITIAL,
 
   /**
-   * @brief Secondary processor is ready to begin multitasking.
+   * @brief Processor is ready to start multitasking.
    *
    * The secondary processor performed its basic initialization and is ready to
    * receive inter-processor interrupts.  Interrupt delivery must be disabled
    * in this state, but requested inter-processor interrupts must be recorded
    * and must be delivered once the secondary processor enables interrupts for
-   * the first time.  The main processor will wait for all secondary processors
+   * the first time.  The boot processor will wait for all secondary processors
    * to change into this state.  In case a secondary processor does not reach
    * this state the system will not start.  The secondary processors wait now
-   * for a change into the PER_CPU_STATE_BEGIN_MULTITASKING state set by the
-   * main processor once all secondary processors reached the
-   * PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING state.
-   *
-   * The owner of the per CPU state field is the main processor in this state.
+   * for a change into the PER_CPU_STATE_REQUEST_START_MULTITASKING state set
+   * by the boot processor once all secondary processors reached the
+   * PER_CPU_STATE_READY_TO_START_MULTITASKING state.
    */
-  PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING,
+  PER_CPU_STATE_READY_TO_START_MULTITASKING,
 
   /**
-   * @brief Multitasking begin of secondary processor is requested.
+   * @brief Multitasking start of processor is requested.
    *
-   * The main processor completed system initialization and is about to perform
+   * The boot processor completed system initialization and is about to perform
    * a context switch to its heir thread.  Secondary processors should now
    * issue a context switch to the heir thread.  This normally enables
    * interrupts on the processor for the first time.
-   *
-   * The owner of the per CPU state field is the secondary processor in this
-   * state.
    */
-  PER_CPU_STATE_BEGIN_MULTITASKING,
+  PER_CPU_STATE_REQUEST_START_MULTITASKING,
 
   /**
    * @brief Normal multitasking state.
-   *
-   * The owner of the per CPU state field is the secondary processor in this
-   * state.
    */
   PER_CPU_STATE_UP,
 
   /**
    * @brief This is the terminal state.
-   *
-   * The owner of the per CPU state field is the secondary processor in this
-   * state.
    */
   PER_CPU_STATE_SHUTDOWN
 } Per_CPU_State;
@@ -313,16 +332,11 @@ static inline void _Per_CPU_Send_interrupt( const Per_CPU_Control *per_cpu )
  */
 void _Per_CPU_Initialize(void);
 
-void _Per_CPU_Change_state(
+void _Per_CPU_State_change(
   Per_CPU_Control *per_cpu,
   Per_CPU_State new_state
 );
 
-void _Per_CPU_Wait_for_state(
-  const Per_CPU_Control *per_cpu,
-  Per_CPU_State desired_state
-);
-
 #endif /* defined( RTEMS_SMP ) */
 
 /*
diff --git a/cpukit/score/include/rtems/score/smpimpl.h b/cpukit/score/include/rtems/score/smpimpl.h
index d68af43916..da08cf54e8 100644
--- a/cpukit/score/include/rtems/score/smpimpl.h
+++ b/cpukit/score/include/rtems/score/smpimpl.h
@@ -108,8 +108,6 @@ static inline void _SMP_Inter_processor_interrupt_handler( void )
     _Per_CPU_Release_and_ISR_enable( self_cpu, level );
 
     if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) {
-      _Per_CPU_Change_state( self_cpu, PER_CPU_STATE_SHUTDOWN );
-
       rtems_fatal( RTEMS_FATAL_SOURCE_SMP, SMP_FATAL_SHUTDOWN );
       /* does not continue past here */
     }
@@ -143,27 +141,27 @@ void _SMP_Broadcast_message(
 #endif /* defined( RTEMS_SMP ) */
 
 /**
- *  @brief Request other cores to perform first context switch.
- *
- *  Send message to other cores requesting them to perform
- *  their first context switch operation.
+ * @brief Requests a multitasking start on all configured and available
+ * processors.
  */
 #if defined( RTEMS_SMP )
-  void _SMP_Request_other_cores_to_perform_first_context_switch( void );
+  void _SMP_Request_start_multitasking( void );
 #else
-  #define _SMP_Request_other_cores_to_perform_first_context_switch() \
+  #define _SMP_Request_start_multitasking() \
     do { } while ( 0 )
 #endif
 
 /**
- *  @brief Request other cores to shutdown.
+ * @brief Requests a shutdown of all processors.
+ *
+ * This function is a part of the system termination procedure.
  *
- *  Send message to other cores requesting them to shutdown.
+ * @see _Terminate().
  */
 #if defined( RTEMS_SMP )
-  void _SMP_Request_other_cores_to_shutdown( void );
+  void _SMP_Request_shutdown( void );
 #else
-  #define _SMP_Request_other_cores_to_shutdown() \
+  #define _SMP_Request_shutdown() \
     do { } while ( 0 )
 #endif
 
diff --git a/cpukit/score/src/interr.c b/cpukit/score/src/interr.c
index c2a9fbe00f..11578a6101 100644
--- a/cpukit/score/src/interr.c
+++ b/cpukit/score/src/interr.c
@@ -39,7 +39,7 @@ void _Terminate(
   _ISR_Disable_without_giant( level );
   (void) level;
 
-  _SMP_Request_other_cores_to_shutdown();
+  _SMP_Request_shutdown();
 
   _User_extensions_Fatal( the_source, is_internal, the_error );
 
diff --git a/cpukit/score/src/percpu.c b/cpukit/score/src/percpu.c
index c68f378d1a..3a7a84518e 100644
--- a/cpukit/score/src/percpu.c
+++ b/cpukit/score/src/percpu.c
@@ -19,26 +19,156 @@
 #endif
 
 #include <rtems/score/percpu.h>
+#include <rtems/score/assert.h>
+#include <rtems/score/smpimpl.h>
+#include <rtems/config.h>
+#include <rtems/fatal.h>
 
 #if defined(RTEMS_SMP)
-  void _Per_CPU_Change_state(
-    Per_CPU_Control *per_cpu,
-    Per_CPU_State new_state
-  )
-  {
-    per_cpu->state = new_state;
-    _CPU_SMP_Processor_event_broadcast();
+
+static SMP_lock_Control _Per_CPU_State_lock = SMP_LOCK_INITIALIZER;
+
+static ISR_Level _Per_CPU_State_acquire( void )
+{
+  ISR_Level level;
+
+  _SMP_lock_ISR_disable_and_acquire( &_Per_CPU_State_lock, level );
+
+  return level;
+}
+
+static void _Per_CPU_State_release( ISR_Level level )
+{
+  _SMP_lock_Release_and_ISR_enable( &_Per_CPU_State_lock, level );
+}
+
+static void _Per_CPU_State_busy_wait(
+  const Per_CPU_Control *per_cpu,
+  Per_CPU_State new_state
+)
+{
+  Per_CPU_State state = per_cpu->state;
+
+  switch ( new_state ) {
+    case PER_CPU_STATE_REQUEST_START_MULTITASKING:
+      while (
+        state != PER_CPU_STATE_READY_TO_START_MULTITASKING
+          && state != PER_CPU_STATE_SHUTDOWN
+      ) {
+        _CPU_SMP_Processor_event_receive();
+        state = per_cpu->state;
+      }
+      break;
+    case PER_CPU_STATE_UP:
+      while (
+        state != PER_CPU_STATE_REQUEST_START_MULTITASKING
+          && state != PER_CPU_STATE_SHUTDOWN
+      ) {
+        _CPU_SMP_Processor_event_receive();
+        state = per_cpu->state;
+      }
+      break;
+    default:
+      /* No need to wait */
+      break;
+  }
+}
+
+static Per_CPU_State _Per_CPU_State_get_next(
+  Per_CPU_State current_state,
+  Per_CPU_State new_state
+)
+{
+  switch ( current_state ) {
+    case PER_CPU_STATE_INITIAL:
+      switch ( new_state ) {
+        case PER_CPU_STATE_READY_TO_START_MULTITASKING:
+        case PER_CPU_STATE_SHUTDOWN:
+          /* Change is acceptable */
+          break;
+        default:
+          new_state = PER_CPU_STATE_SHUTDOWN;
+          break;
+      }
+      break;
+    case PER_CPU_STATE_READY_TO_START_MULTITASKING:
+      switch ( new_state ) {
+        case PER_CPU_STATE_REQUEST_START_MULTITASKING:
+        case PER_CPU_STATE_SHUTDOWN:
+          /* Change is acceptable */
+          break;
+        default:
+          new_state = PER_CPU_STATE_SHUTDOWN;
+          break;
+      }
+      break;
+    case PER_CPU_STATE_REQUEST_START_MULTITASKING:
+      switch ( new_state ) {
+        case PER_CPU_STATE_UP:
+        case PER_CPU_STATE_SHUTDOWN:
+          /* Change is acceptable */
+          break;
+        default:
+          new_state = PER_CPU_STATE_SHUTDOWN;
+          break;
+      }
+      break;
+    default:
+      new_state = PER_CPU_STATE_SHUTDOWN;
+      break;
   }
 
-  void _Per_CPU_Wait_for_state(
-    const Per_CPU_Control *per_cpu,
-    Per_CPU_State desired_state
-  )
-  {
-    while ( per_cpu->state != desired_state ) {
-      _CPU_SMP_Processor_event_receive();
+  return new_state;
+}
+
+void _Per_CPU_State_change(
+  Per_CPU_Control *per_cpu,
+  Per_CPU_State new_state
+)
+{
+  ISR_Level level;
+  Per_CPU_State next_state;
+
+  _Per_CPU_State_busy_wait( per_cpu, new_state );
+
+  level = _Per_CPU_State_acquire();
+  next_state = _Per_CPU_State_get_next( per_cpu->state, new_state );
+  per_cpu->state = next_state;
+
+  if ( next_state == PER_CPU_STATE_SHUTDOWN ) {
+    uint32_t ncpus = rtems_configuration_get_maximum_processors();
+    uint32_t cpu;
+
+    for ( cpu = 0 ; cpu < ncpus ; ++cpu ) {
+      Per_CPU_Control *other_cpu = _Per_CPU_Get_by_index( cpu );
+
+      if ( per_cpu != other_cpu ) {
+        switch ( other_cpu->state ) {
+          case PER_CPU_STATE_UP:
+            _SMP_Send_message( cpu, SMP_MESSAGE_SHUTDOWN );
+            break;
+          default:
+            /* Nothing to do */
+            break;
+        }
+
+        other_cpu->state = PER_CPU_STATE_SHUTDOWN;
+      }
     }
   }
+
+  _CPU_SMP_Processor_event_broadcast();
+
+  _Per_CPU_State_release( level );
+
+  if (
+    next_state == PER_CPU_STATE_SHUTDOWN
+      && new_state != PER_CPU_STATE_SHUTDOWN
+  ) {
+    rtems_fatal( RTEMS_FATAL_SOURCE_SMP, SMP_FATAL_SHUTDOWN );
+  }
+}
+
 #else
   /*
    * On single core systems, we can efficiently directly access a single
diff --git a/cpukit/score/src/smp.c b/cpukit/score/src/smp.c
index 59036eb466..0f632236b7 100644
--- a/cpukit/score/src/smp.c
+++ b/cpukit/score/src/smp.c
@@ -24,10 +24,6 @@
 #include <rtems/score/threadimpl.h>
 #include <rtems/config.h>
 
-#if defined(RTEMS_DEBUG)
-  #include <rtems/bspIo.h>
-#endif
-
 void _SMP_Handler_initialize( void )
 {
   uint32_t max_cpus = rtems_configuration_get_maximum_processors();
@@ -47,21 +43,38 @@ void _SMP_Handler_initialize( void )
   _SMP_Processor_count = max_cpus;
 }
 
-void _SMP_Start_multitasking_on_secondary_processor( void )
+void _SMP_Request_start_multitasking( void )
 {
   Per_CPU_Control *self_cpu = _Per_CPU_Get();
+  uint32_t ncpus = _SMP_Get_processor_count();
+  uint32_t cpu;
+
+  _Per_CPU_State_change( self_cpu, PER_CPU_STATE_READY_TO_START_MULTITASKING );
 
-  #if defined(RTEMS_DEBUG)
-    printk( "Made it to %d -- ", _Per_CPU_Get_index( self_cpu ) );
-  #endif
+  for ( cpu = 0 ; cpu < ncpus ; ++cpu ) {
+    Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
+
+    _Per_CPU_State_change( per_cpu, PER_CPU_STATE_REQUEST_START_MULTITASKING );
+  }
+}
 
-  _Per_CPU_Change_state( self_cpu, PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING );
+void _SMP_Start_multitasking_on_secondary_processor( void )
+{
+  Per_CPU_Control *self_cpu = _Per_CPU_Get();
 
-  _Per_CPU_Wait_for_state( self_cpu, PER_CPU_STATE_BEGIN_MULTITASKING );
+  _Per_CPU_State_change( self_cpu, PER_CPU_STATE_READY_TO_START_MULTITASKING );
 
   _Thread_Start_multitasking();
 }
 
+void _SMP_Request_shutdown( void )
+{
+  uint32_t self = _SMP_Get_current_processor();
+  Per_CPU_Control *self_cpu = _Per_CPU_Get_by_index( self );
+
+  _Per_CPU_State_change( self_cpu, PER_CPU_STATE_SHUTDOWN );
+}
+
 void _SMP_Send_message( uint32_t cpu, uint32_t message )
 {
   Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
@@ -88,47 +101,3 @@ void _SMP_Broadcast_message( uint32_t message )
     }
   }
 }
-
-void _SMP_Request_other_cores_to_perform_first_context_switch( void )
-{
-  uint32_t self = _SMP_Get_current_processor();
-  uint32_t ncpus = _SMP_Get_processor_count();
-  uint32_t cpu;
-
-  for ( cpu = 0 ; cpu < ncpus ; ++cpu ) {
-    Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
-
-    if ( cpu != self ) {
-      _Per_CPU_Wait_for_state(
-        per_cpu,
-        PER_CPU_STATE_READY_TO_BEGIN_MULTITASKING
-      );
-
-      _Per_CPU_Change_state( per_cpu, PER_CPU_STATE_BEGIN_MULTITASKING );
-    }
-  }
-}
-
-void _SMP_Request_other_cores_to_shutdown( void )
-{
-  uint32_t self = _SMP_Get_current_processor();
-
-  /*
-   * Do not use _SMP_Get_processor_count() since this value might be not
-   * initialized yet.  For example due to a fatal error in the middle of
-   * _CPU_SMP_Initialize().
-   */
-  uint32_t ncpus = rtems_configuration_get_maximum_processors();
-
-  uint32_t cpu;
-
-  for ( cpu = 0 ; cpu < ncpus ; ++cpu ) {
-    if ( cpu != self ) {
-      const Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
-
-      if ( per_cpu->state != PER_CPU_STATE_BEFORE_INITIALIZATION ) {
-        _SMP_Send_message( cpu, SMP_MESSAGE_SHUTDOWN );
-      }
-    }
-  }
-}
diff --git a/cpukit/score/src/threadstartmultitasking.c b/cpukit/score/src/threadstartmultitasking.c
index d73e5b4194..2c40170f36 100644
--- a/cpukit/score/src/threadstartmultitasking.c
+++ b/cpukit/score/src/threadstartmultitasking.c
@@ -26,7 +26,7 @@ void _Thread_Start_multitasking( void )
   Thread_Control  *heir;
 
 #if defined(RTEMS_SMP)
-  _Per_CPU_Change_state( self_cpu, PER_CPU_STATE_UP );
+  _Per_CPU_State_change( self_cpu, PER_CPU_STATE_UP );
 
   /*
    * Threads begin execution in the _Thread_Handler() function.   This
diff --git a/testsuites/smptests/Makefile.am b/testsuites/smptests/Makefile.am
index 023b7e9ed6..8422f3a7a4 100644
--- a/testsuites/smptests/Makefile.am
+++ b/testsuites/smptests/Makefile.am
@@ -11,6 +11,8 @@ SUBDIRS += smp07
 SUBDIRS += smp08
 SUBDIRS += smp09
 SUBDIRS += smpatomic01
+SUBDIRS += smpfatal01
+SUBDIRS += smpfatal02
 SUBDIRS += smplock01
 SUBDIRS += smpmigration01
 SUBDIRS += smpschedule01
diff --git a/testsuites/smptests/configure.ac b/testsuites/smptests/configure.ac
index 5c6877217a..19e32f359f 100644
--- a/testsuites/smptests/configure.ac
+++ b/testsuites/smptests/configure.ac
@@ -65,6 +65,8 @@ smp07/Makefile
 smp08/Makefile
 smp09/Makefile
 smpatomic01/Makefile
+smpfatal01/Makefile
+smpfatal02/Makefile
 smplock01/Makefile
 smpmigration01/Makefile
 smppsxsignal01/Makefile
diff --git a/testsuites/smptests/smpfatal01/Makefile.am b/testsuites/smptests/smpfatal01/Makefile.am
new file mode 100644
index 0000000000..2aaee2b7f3
--- /dev/null
+++ b/testsuites/smptests/smpfatal01/Makefile.am
@@ -0,0 +1,19 @@
+rtems_tests_PROGRAMS = smpfatal01
+smpfatal01_SOURCES = init.c
+
+dist_rtems_tests_DATA = smpfatal01.scn smpfatal01.doc
+
+include $(RTEMS_ROOT)/make/custom/@RTEMS_BSP@.cfg
+include $(top_srcdir)/../automake/compile.am
+include $(top_srcdir)/../automake/leaf.am
+
+AM_CPPFLAGS += -I$(top_srcdir)/../support/include
+
+LINK_OBJS = $(smpfatal01_OBJECTS)
+LINK_LIBS = $(smpfatal01_LDLIBS)
+
+smpfatal01$(EXEEXT): $(smpfatal01_OBJECTS) $(smpfatal01_DEPENDENCIES)
+	@rm -f smpfatal01$(EXEEXT)
+	$(make-exe)
+
+include $(top_srcdir)/../automake/local.am
diff --git a/testsuites/smptests/smpfatal01/init.c b/testsuites/smptests/smpfatal01/init.c
new file mode 100644
index 0000000000..c78b29fbe8
--- /dev/null
+++ b/testsuites/smptests/smpfatal01/init.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2014 embedded brains GmbH.  All rights reserved.
+ *
+ *  embedded brains GmbH
+ *  Dornierstr. 4
+ *  82178 Puchheim
+ *  Germany
+ *  <rtems@embedded-brains.de>
+ *
+ * The license and distribution terms for this file may be
+ * found in the file LICENSE in this distribution or at
+ * http://www.rtems.com/license/LICENSE.
+ */
+
+#ifdef HAVE_CONFIG_H
+  #include "config.h"
+#endif
+
+#include <rtems.h>
+#include <rtems/score/percpu.h>
+#include <rtems/score/smpimpl.h>
+
+#include <assert.h>
+#include <stdlib.h>
+
+#define MAX_CPUS 32
+
+static uint32_t main_cpu;
+
+static void Init(rtems_task_argument arg)
+{
+  assert(0);
+}
+
+static void end_of_test(void)
+{
+  printk( "*** END OF TEST SMPFATAL 1 ***\n" );
+}
+
+static void fatal_extension(
+  rtems_fatal_source source,
+  bool is_internal,
+  rtems_fatal_code code
+)
+{
+  if (source == RTEMS_FATAL_SOURCE_SMP) {
+    uint32_t self = rtems_smp_get_current_processor();
+
+    assert(!is_internal);
+    assert(code == SMP_FATAL_SHUTDOWN);
+
+    if (self == main_cpu) {
+      uint32_t cpu;
+
+      for (cpu = 0; cpu < MAX_CPUS; ++cpu) {
+        const Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
+        Per_CPU_State state = per_cpu->state;
+
+        assert(state == PER_CPU_STATE_SHUTDOWN);
+      }
+
+      end_of_test();
+    }
+  }
+}
+
+static rtems_status_code test_driver_init(
+  rtems_device_major_number major,
+  rtems_device_minor_number minor,
+  void *arg
+)
+{
+  uint32_t self = rtems_smp_get_current_processor();
+  uint32_t cpu_count = rtems_smp_get_processor_count();
+  uint32_t cpu;
+
+  printk("\n\n*** TEST SMPFATAL 1 ***\n");
+
+  assert(rtems_configuration_get_maximum_processors() == MAX_CPUS);
+
+  main_cpu = self;
+
+  for (cpu = 0; cpu < MAX_CPUS; ++cpu) {
+    const Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
+    Per_CPU_State state = per_cpu->state;
+
+    if (cpu == self) {
+      assert(state == PER_CPU_STATE_INITIAL);
+    } else if (cpu < cpu_count) {
+      assert(
+        state == PER_CPU_STATE_INITIAL
+          || state == PER_CPU_STATE_READY_TO_START_MULTITASKING
+      );
+    } else {
+      assert(state == PER_CPU_STATE_INITIAL);
+    }
+  }
+
+  if (cpu_count > 1) {
+    uint32_t other = (self + 1) % cpu_count;
+    Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( other );
+
+    per_cpu->state = PER_CPU_STATE_SHUTDOWN;
+  } else {
+    end_of_test();
+    exit(0);
+  }
+
+  return RTEMS_SUCCESSFUL;
+}
+
+#define CONFIGURE_APPLICATION_DOES_NOT_NEED_CLOCK_DRIVER
+#define CONFIGURE_APPLICATION_NEEDS_CONSOLE_DRIVER
+
+#define CONFIGURE_APPLICATION_EXTRA_DRIVERS \
+  { .initialization_entry = test_driver_init }
+
+#define CONFIGURE_INITIAL_EXTENSIONS { .fatal = fatal_extension }
+
+#define CONFIGURE_SMP_APPLICATION
+
+#define CONFIGURE_SMP_MAXIMUM_PROCESSORS MAX_CPUS
+
+#define CONFIGURE_MAXIMUM_TASKS 1
+
+#define CONFIGURE_RTEMS_INIT_TASKS_TABLE
+
+#define CONFIGURE_INIT
+
+#include <rtems/confdefs.h>
diff --git a/testsuites/smptests/smpfatal01/smpfatal01.doc b/testsuites/smptests/smpfatal01/smpfatal01.doc
new file mode 100644
index 0000000000..c037cfe78b
--- /dev/null
+++ b/testsuites/smptests/smpfatal01/smpfatal01.doc
@@ -0,0 +1,12 @@
+This file describes the directives and concepts tested by this test set.
+
+test set name: smpfatal01
+
+directives:
+
+  - _Per_CPU_State_change()
+
+concepts:
+
+  - Ensure that the system termination in case of shutdown detection at a
+    secondary processors works during driver initialization.
diff --git a/testsuites/smptests/smpfatal01/smpfatal01.scn b/testsuites/smptests/smpfatal01/smpfatal01.scn
new file mode 100644
index 0000000000..0b67121949
--- /dev/null
+++ b/testsuites/smptests/smpfatal01/smpfatal01.scn
@@ -0,0 +1,2 @@
+*** TEST SMPFATAL 1 ***
+*** END OF TEST SMPFATAL 1 ***
diff --git a/testsuites/smptests/smpfatal02/Makefile.am b/testsuites/smptests/smpfatal02/Makefile.am
new file mode 100644
index 0000000000..bbce3b7b6d
--- /dev/null
+++ b/testsuites/smptests/smpfatal02/Makefile.am
@@ -0,0 +1,19 @@
+rtems_tests_PROGRAMS = smpfatal02
+smpfatal02_SOURCES = init.c
+
+dist_rtems_tests_DATA = smpfatal02.scn smpfatal02.doc
+
+include $(RTEMS_ROOT)/make/custom/@RTEMS_BSP@.cfg
+include $(top_srcdir)/../automake/compile.am
+include $(top_srcdir)/../automake/leaf.am
+
+AM_CPPFLAGS += -I$(top_srcdir)/../support/include
+
+LINK_OBJS = $(smpfatal02_OBJECTS)
+LINK_LIBS = $(smpfatal02_LDLIBS)
+
+smpfatal02$(EXEEXT): $(smpfatal02_OBJECTS) $(smpfatal02_DEPENDENCIES)
+	@rm -f smpfatal02$(EXEEXT)
+	$(make-exe)
+
+include $(top_srcdir)/../automake/local.am
diff --git a/testsuites/smptests/smpfatal02/init.c b/testsuites/smptests/smpfatal02/init.c
new file mode 100644
index 0000000000..1e8da268a9
--- /dev/null
+++ b/testsuites/smptests/smpfatal02/init.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2014 embedded brains GmbH.  All rights reserved.
+ *
+ *  embedded brains GmbH
+ *  Dornierstr. 4
+ *  82178 Puchheim
+ *  Germany
+ *  <rtems@embedded-brains.de>
+ *
+ * The license and distribution terms for this file may be
+ * found in the file LICENSE in this distribution or at
+ * http://www.rtems.com/license/LICENSE.
+ */
+
+#ifdef HAVE_CONFIG_H
+  #include "config.h"
+#endif
+
+#include <rtems.h>
+#include <rtems/score/percpu.h>
+#include <rtems/score/smpimpl.h>
+
+#include <assert.h>
+#include <stdlib.h>
+
+#define MAX_CPUS 32
+
+static uint32_t main_cpu;
+
+static void Init(rtems_task_argument arg)
+{
+  assert(0);
+}
+
+static void end_of_test(void)
+{
+  printk( "*** END OF TEST SMPFATAL 2 ***\n" );
+}
+
+static void fatal_extension(
+  rtems_fatal_source source,
+  bool is_internal,
+  rtems_fatal_code code
+)
+{
+  if (
+    source == RTEMS_FATAL_SOURCE_APPLICATION
+      || source == RTEMS_FATAL_SOURCE_SMP
+  ) {
+    uint32_t self = rtems_smp_get_current_processor();
+
+    assert(!is_internal);
+
+    if (self == main_cpu) {
+      uint32_t cpu;
+
+      assert(source == RTEMS_FATAL_SOURCE_APPLICATION);
+      assert(code == 0xdeadbeef);
+
+      for (cpu = 0; cpu < MAX_CPUS; ++cpu) {
+        const Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
+        Per_CPU_State state = per_cpu->state;
+
+        assert(state == PER_CPU_STATE_SHUTDOWN);
+      }
+
+      end_of_test();
+    } else {
+      assert(source == RTEMS_FATAL_SOURCE_SMP);
+      assert(code == SMP_FATAL_SHUTDOWN);
+    }
+  }
+}
+
+static rtems_status_code test_driver_init(
+  rtems_device_major_number major,
+  rtems_device_minor_number minor,
+  void *arg
+)
+{
+  uint32_t self = rtems_smp_get_current_processor();
+  uint32_t cpu_count = rtems_smp_get_processor_count();
+  uint32_t cpu;
+
+  printk("\n\n*** TEST SMPFATAL 2 ***\n");
+
+  assert(rtems_configuration_get_maximum_processors() == MAX_CPUS);
+
+  main_cpu = self;
+
+  for (cpu = 0; cpu < MAX_CPUS; ++cpu) {
+    const Per_CPU_Control *per_cpu = _Per_CPU_Get_by_index( cpu );
+    Per_CPU_State state = per_cpu->state;
+
+    if (cpu == self) {
+      assert(state == PER_CPU_STATE_INITIAL);
+    } else if (cpu < cpu_count) {
+      assert(
+        state == PER_CPU_STATE_INITIAL
+          || state == PER_CPU_STATE_READY_TO_START_MULTITASKING
+      );
+    } else {
+      assert(state == PER_CPU_STATE_INITIAL);
+    }
+  }
+
+  if (cpu_count > 1) {
+    rtems_fatal(RTEMS_FATAL_SOURCE_APPLICATION, 0xdeadbeef);
+  } else {
+    end_of_test();
+    exit(0);
+  }
+
+  return RTEMS_SUCCESSFUL;
+}
+
+#define CONFIGURE_APPLICATION_DOES_NOT_NEED_CLOCK_DRIVER
+#define CONFIGURE_APPLICATION_NEEDS_CONSOLE_DRIVER
+
+#define CONFIGURE_APPLICATION_EXTRA_DRIVERS \
+  { .initialization_entry = test_driver_init }
+
+#define CONFIGURE_INITIAL_EXTENSIONS { .fatal = fatal_extension }
+
+#define CONFIGURE_SMP_APPLICATION
+
+#define CONFIGURE_SMP_MAXIMUM_PROCESSORS MAX_CPUS
+
+#define CONFIGURE_MAXIMUM_TASKS 1
+
+#define CONFIGURE_RTEMS_INIT_TASKS_TABLE
+
+#define CONFIGURE_INIT
+
+#include <rtems/confdefs.h>
diff --git a/testsuites/smptests/smpfatal02/smpfatal02.doc b/testsuites/smptests/smpfatal02/smpfatal02.doc
new file mode 100644
index 0000000000..9e2e002b37
--- /dev/null
+++ b/testsuites/smptests/smpfatal02/smpfatal02.doc
@@ -0,0 +1,12 @@
+This file describes the directives and concepts tested by this test set.
+
+test set name: smpfatal02
+
+directives:
+
+  - _Per_CPU_State_change()
+
+concepts:
+
+  - Ensure that the system termination in case of fatal errors during driver
+    initialization works.
diff --git a/testsuites/smptests/smpfatal02/smpfatal02.scn b/testsuites/smptests/smpfatal02/smpfatal02.scn
new file mode 100644
index 0000000000..cde11a0e73
--- /dev/null
+++ b/testsuites/smptests/smpfatal02/smpfatal02.scn
@@ -0,0 +1,2 @@
+*** TEST SMPFATAL 2 ***
+*** END OF TEST SMPFATAL 2 ***