score: Implement forced thread migration

The current implementation of task migration in RTEMS has some
implications with respect to the interrupt latency. It is crucial to
preserve the system invariant that a task can execute on at most one
processor in the system at a time. This is accomplished with a boolean
indicator in the task context. The processor architecture specific
low-level task context switch code will mark that a task context is no
longer executing and waits that the heir context stopped execution
before it restores the heir context and resumes execution of the heir
task. So there is one point in time in which a processor is without a
task. This is essential to avoid cyclic dependencies in case multiple
tasks migrate at once. Otherwise some supervising entity is necessary to
prevent life-locks. Such a global supervisor would lead to scalability
problems so this approach is not used. Currently the thread dispatch is
performed with interrupts disabled. So in case the heir task is
currently executing on another processor then this prolongs the time of
disabled interrupts since one processor has to wait for another
processor to make progress.

It is difficult to avoid this issue with the interrupt latency since
interrupts normally store the context of the interrupted task on its
stack. In case a task is marked as not executing we must not use its
task stack to store such an interrupt context. We cannot use the heir
stack before it stopped execution on another processor. So if we enable
interrupts during this transition we have to provide an alternative task
independent stack for this time frame. This issue needs further
investigation.

4 files changed, 291 insertions, 0 deletions

diff --git a/testsuites/smptests/smpmigration02/Makefile.am b/testsuites/smptests/smpmigration02/Makefile.am
new file mode 100644
index 0000000000..8dcd8ad68f
--- /dev/null
+++ b/testsuites/smptests/smpmigration02/Makefile.am
@@ -0,0 +1,19 @@
+rtems_tests_PROGRAMS = smpmigration02
+smpmigration02_SOURCES = init.c
+
+dist_rtems_tests_DATA = smpmigration02.scn smpmigration02.doc
+
+include $(RTEMS_ROOT)/make/custom/@RTEMS_BSP@.cfg
+include $(top_srcdir)/../automake/compile.am
+include $(top_srcdir)/../automake/leaf.am
+
+AM_CPPFLAGS += -I$(top_srcdir)/../support/include
+
+LINK_OBJS = $(smpmigration02_OBJECTS)
+LINK_LIBS = $(smpmigration02_LDLIBS)
+
+smpmigration02$(EXEEXT): $(smpmigration02_OBJECTS) $(smpmigration02_DEPENDENCIES)
+	@rm -f smpmigration02$(EXEEXT)
+	$(make-exe)
+
+include $(top_srcdir)/../automake/local.am
diff --git a/testsuites/smptests/smpmigration02/init.c b/testsuites/smptests/smpmigration02/init.c
new file mode 100644
index 0000000000..7cf4651663
--- /dev/null
+++ b/testsuites/smptests/smpmigration02/init.c
@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2014 embedded brains GmbH.  All rights reserved.
+ *
+ *  embedded brains GmbH
+ *  Dornierstr. 4
+ *  82178 Puchheim
+ *  Germany
+ *  <rtems@embedded-brains.de>
+ *
+ * The license and distribution terms for this file may be
+ * found in the file LICENSE in this distribution or at
+ * http://www.rtems.org/license/LICENSE.
+ */
+
+#ifdef HAVE_CONFIG_H
+  #include "config.h"
+#endif
+
+#include <rtems.h>
+#include <rtems/libcsupport.h>
+
+#include "tmacros.h"
+
+const char rtems_test_name[] = "SMPMIGRATION 2";
+
+#define CPU_COUNT 32
+
+#define TASK_COUNT (CPU_COUNT + 1)
+
+#define PRIO_LOW 3
+
+#define PRIO_HIGH 2
+
+typedef struct {
+  uint32_t value;
+  uint32_t cache_line_separation[31];
+} test_counter;
+
+typedef struct {
+  test_counter counters[TASK_COUNT];
+  rtems_id scheduler_ids[CPU_COUNT];
+  rtems_id task_ids[TASK_COUNT];
+} test_context;
+
+static test_context test_instance;
+
+static void task(rtems_task_argument arg)
+{
+  test_context *ctx = &test_instance;
+  rtems_status_code sc;
+  uint32_t cpu_count = rtems_get_processor_count();
+  uint32_t cpu_index = rtems_get_current_processor();
+
+  while (true) {
+    cpu_index = (cpu_index + 1) % cpu_count;
+
+    sc = rtems_task_set_scheduler(RTEMS_SELF, ctx->scheduler_ids[cpu_index]);
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+
+    ++ctx->counters[arg].value;
+
+    rtems_test_assert(cpu_index == rtems_get_current_processor());
+  }
+}
+
+static void test(void)
+{
+  test_context *ctx = &test_instance;
+  rtems_status_code sc;
+  uint32_t cpu_count = rtems_get_processor_count();
+  uint32_t cpu_index;
+  uint32_t task_count = cpu_count + 1;
+  uint32_t task_index;
+
+  for (cpu_index = 0; cpu_index < cpu_count; ++cpu_index) {
+    sc = rtems_scheduler_ident(cpu_index, &ctx->scheduler_ids[cpu_index]);
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+  }
+
+  for (task_index = 0; task_index < task_count; ++task_index) {
+    rtems_id task_id;
+
+    sc = rtems_task_create(
+      rtems_build_name('T', 'A', 'S', 'K'),
+      task_index > 0 ? PRIO_LOW : PRIO_HIGH,
+      RTEMS_MINIMUM_STACK_SIZE,
+      RTEMS_DEFAULT_MODES,
+      RTEMS_DEFAULT_ATTRIBUTES,
+      &task_id
+    );
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+
+    sc = rtems_task_set_scheduler(task_id, ctx->scheduler_ids[task_index % cpu_count]);
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+
+    sc = rtems_task_start(task_id, task, task_index);
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+
+    ctx->task_ids[task_index] = task_id;
+  }
+
+  sc = rtems_task_wake_after(30 * rtems_clock_get_ticks_per_second());
+  rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+
+  for (task_index = 0; task_index < task_count; ++task_index) {
+    printf(
+      "task %" PRIu32 " counter: %" PRIu32 "\n",
+      task_index,
+      ctx->counters[task_index].value
+    );
+
+    sc = rtems_task_delete(ctx->task_ids[task_index]);
+    rtems_test_assert(sc == RTEMS_SUCCESSFUL);
+  }
+}
+
+static void Init(rtems_task_argument arg)
+{
+  rtems_resource_snapshot snapshot;
+
+  TEST_BEGIN();
+
+  rtems_resource_snapshot_take(&snapshot);
+
+  test();
+
+  rtems_test_assert(rtems_resource_snapshot_check(&snapshot));
+
+  TEST_END();
+  rtems_test_exit(0);
+}
+
+#define CONFIGURE_APPLICATION_NEEDS_CLOCK_DRIVER
+#define CONFIGURE_APPLICATION_NEEDS_CONSOLE_DRIVER
+
+#define CONFIGURE_SMP_APPLICATION
+
+#define CONFIGURE_SMP_MAXIMUM_PROCESSORS CPU_COUNT
+
+#define CONFIGURE_SCHEDULER_SIMPLE_SMP
+
+#include <rtems/scheduler.h>
+
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(0);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(1);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(2);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(3);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(4);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(5);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(6);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(7);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(8);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(9);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(10);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(11);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(12);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(13);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(14);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(15);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(16);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(17);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(18);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(19);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(20);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(21);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(22);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(23);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(24);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(25);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(26);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(27);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(28);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(29);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(30);
+RTEMS_SCHEDULER_CONTEXT_SIMPLE_SMP(31);
+
+#define CONFIGURE_SCHEDULER_CONTROLS \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(0, 0), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(1, 1), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(2, 2), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(3, 3), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(4, 4), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(5, 5), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(6, 6), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(7, 7), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(8, 8), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(9, 9), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(10, 10), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(11, 11), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(12, 12), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(13, 13), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(14, 14), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(15, 15), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(16, 16), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(17, 17), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(18, 18), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(19, 19), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(20, 20), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(21, 21), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(22, 22), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(23, 23), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(24, 24), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(25, 25), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(26, 26), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(27, 27), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(28, 28), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(29, 29), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(30, 30), \
+  RTEMS_SCHEDULER_CONTROL_SIMPLE_SMP(31, 31)
+
+#define CONFIGURE_SMP_SCHEDULER_ASSIGNMENTS \
+  RTEMS_SCHEDULER_ASSIGN(0, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(1, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(2, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(3, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(4, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(5, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(6, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(7, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(8, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(9, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(10, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(11, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(12, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(13, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(14, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(15, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(16, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(17, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(18, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(19, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(20, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(21, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(22, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(23, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(24, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(25, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(26, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(27, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(28, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(29, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(30, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL), \
+  RTEMS_SCHEDULER_ASSIGN(31, RTEMS_SCHEDULER_ASSIGN_PROCESSOR_OPTIONAL)
+
+#define CONFIGURE_MAXIMUM_TASKS (1 + TASK_COUNT)
+
+#define CONFIGURE_INITIAL_EXTENSIONS RTEMS_TEST_INITIAL_EXTENSION
+
+#define CONFIGURE_RTEMS_INIT_TASKS_TABLE
+
+#define CONFIGURE_INIT
+
+#include <rtems/confdefs.h>
diff --git a/testsuites/smptests/smpmigration02/smpmigration02.doc b/testsuites/smptests/smpmigration02/smpmigration02.doc
new file mode 100644
index 0000000000..bfae205f5d
--- /dev/null
+++ b/testsuites/smptests/smpmigration02/smpmigration02.doc
@@ -0,0 +1,12 @@
+This file describes the directives and concepts tested by this test set.
+
+test set name: smpmigration02
+
+directives:
+
+  - _Scheduler_SMP_Allocate_processor()
+  - _CPU_Context_switch()
+
+concepts:
+
+  - Ensure that forced thread migration works.
diff --git a/testsuites/smptests/smpmigration02/smpmigration02.scn b/testsuites/smptests/smpmigration02/smpmigration02.scn
new file mode 100644
index 0000000000..c436e19a67
--- /dev/null
+++ b/testsuites/smptests/smpmigration02/smpmigration02.scn
@@ -0,0 +1,7 @@
+*** BEGIN OF TEST SMPMIGRATION 2 ***
+task 0 counter: 1137459
+task 1 counter: 1136714
+task 2 counter: 1136713
+task 3 counter: 1136712
+task 4 counter: 1136711
+*** END OF TEST SMPMIGRATION 2 ***