score: Use priority inheritance for thread join

Threads may join the thread termination of another thread using the
pthread_join() or rtems_task_delete() directives.  The thread cancel operation
used a special case priority boosting mechanism implemented by
_Thread_Raise_real_priority().  The problem was that this approach

* is not transitive,

* does not account for priority adjustments of the calling task
  while waiting for the join,

* does not support clustered scheduling, and

* does not detect deadlocks.

All these problems are fixed by using a priority inheritance thread queue for
the join operation.

Close #4679.

2 files changed, 78 insertions, 31 deletions

diff --git a/cpukit/include/rtems/rtems/tasks.h b/cpukit/include/rtems/rtems/tasks.h
index 81757db8c7..ba05d92531 100644
--- a/cpukit/include/rtems/rtems/tasks.h
+++ b/cpukit/include/rtems/rtems/tasks.h
@@ -413,8 +413,8 @@ rtems_task_priority _RTEMS_Maximum_priority( void );
 /**
  * @ingroup RTEMSAPIClassicTasks
  *
- * @brief This constant variable provides the lowest (least important) task
- *   priority of the first configured scheduler.
+ * @brief This runtime constant represents the lowest (least important) task
+ *   priority of the scheduler with index zero.
  */
 #define RTEMS_MAXIMUM_PRIORITY _RTEMS_Maximum_priority()
 
@@ -1031,13 +1031,25 @@ rtems_status_code rtems_task_restart(
  * @retval ::RTEMS_CALLED_FROM_ISR The directive was called from within
  *   interrupt context.
  *
+ * @retval ::RTEMS_INCORRECT_STATE The task termination procedure was started,
+ *   however, waiting for the terminating task would have resulted in a
+ *   deadlock.
+ *
  * @retval ::RTEMS_ILLEGAL_ON_REMOTE_OBJECT The task resided on a remote node.
  *
  * @par Notes
  * @parblock
- * RTEMS stops the execution of the task and reclaims the stack memory, any
- * allocated delay or timeout timers, the TCB, and, if the task is
- * #RTEMS_FLOATING_POINT, its floating point context area. RTEMS explicitly
+ * The task deletion is done in several steps.  Firstly, the task is marked as
+ * terminating.  While the task life of the terminating task is protected, it
+ * executes normally until it disables the task life protection or it deletes
+ * itself.  A terminating task will eventually stop its normal execution and
+ * start its termination procedure.  The procedure executes in the context of
+ * the terminating task.  The task termination procedure involves the
+ * destruction of POSIX key values and running the task termination user
+ * extensions.  Once complete the execution of the task is stopped and
+ * task-specific resources are reclaimed by the system, such as the stack
+ * memory, any allocated delay or timeout timers, the TCB, and, if the task is
+ * #RTEMS_FLOATING_POINT, its floating point context area.  RTEMS explicitly
  * does not reclaim the following resources: region segments, partition
  * buffers, semaphores, timers, or rate monotonic periods.
  *
@@ -1049,10 +1061,12 @@ rtems_status_code rtems_task_restart(
  * resources and delete itself by restarting it with a special argument or by
  * sending it a message, an event, or a signal.
  *
- * Deletion of the current task (#RTEMS_SELF) will force RTEMS to select
+ * Deletion of the calling task (#RTEMS_SELF) will force RTEMS to select
  * another task to execute.
  *
- * The TCB for the deleted task is reclaimed by RTEMS.
+ * When a task deletes another task, the calling task waits until the task
+ * termination procedure of the task being deleted has completed.  The
+ * terminating task inherits the eligible priorities of the calling task.
  *
  * When a global task is deleted, the task identifier must be transmitted to
  * every node in the system for deletion from the local copy of the global
diff --git a/cpukit/include/rtems/score/threadimpl.h b/cpukit/include/rtems/score/threadimpl.h
index 37932b367d..e6e77b195c 100644
--- a/cpukit/include/rtems/score/threadimpl.h
+++ b/cpukit/include/rtems/score/threadimpl.h
@@ -380,15 +380,23 @@ RTEMS_NO_RETURN void _Thread_Exit(
 );
 
 /**
- * @brief Joins the currently executing thread with the given thread to wait
- *      for.
+ * @brief Joins the currently executing thread with the thread.
  *
- * @param[in, out] the_thread The thread to wait for.
- * @param waiting_for_join The states control for the join.
- * @param[in, out] executing The currently executing thread.
- * @param queue_context The thread queue context.
+ * @param[in, out] the_thread is the thread to join.
+ *
+ * @param waiting_for_join is the thread state for the currently executing
+ *   thread.
+ *
+ * @param[in, out] executing is the currently executing thread.
+ *
+ * @param queue_context[in, out] is the thread queue context.  The caller shall
+ *   have acquired the thread state lock using the thread queue context.
+ *
+ * @retval STATUS_SUCCESSFUL The operation was successful.
+ *
+ * @retval STATUS_DEADLOCK A deadlock occurred.
  */
-void _Thread_Join(
+Status_Control _Thread_Join(
   Thread_Control       *the_thread,
   States_Control        waiting_for_join,
   Thread_Control       *executing,
@@ -396,23 +404,41 @@ void _Thread_Join(
 );
 
 /**
+ * @brief Indicates the resulting state of _Thread_Cancel().
+ */
+typedef enum {
+  /**
+   * @brief Indicates that the thread cancel operation is done.
+   */
+  THREAD_CANCEL_DONE,
+
+  /**
+   * @brief Indicates that the thread cancel operation is in progress.
+   *
+   * The cancelled thread is terminating.  It may be joined.
+   */
+  THREAD_CANCEL_IN_PROGRESS
+} Thread_Cancel_state;
+
+/**
  * @brief Cancels the thread.
  *
- * @param[in, out] the_thread The thread to cancel.
- * @param executing The currently executing thread.
- * @param exit_value The exit value for the thread.
+ * @param[in, out] the_thread is the thread to cancel.
+
+ * @param[in, out] executing is the currently executing thread.
+
+ * @param[in, out] life_states_to_clear is the set of thread life states to
+ *   clear for the thread to cancel.
+
+ * @param exit_value is the exit value for the thread to cancel.
  */
-void _Thread_Cancel(
-  Thread_Control *the_thread,
-  Thread_Control *executing,
-  void           *exit_value
+Thread_Cancel_state _Thread_Cancel(
+  Thread_Control   *the_thread,
+  Thread_Control   *executing,
+  Thread_Life_state life_states_to_clear,
+  void             *exit_value
 );
 
-typedef struct {
-  Thread_queue_Context  Base;
-  Thread_Control       *cancel;
-} Thread_Close_context;
-
 /**
  * @brief Closes the thread.
  *
@@ -420,14 +446,21 @@ typedef struct {
  * case the executing thread is not terminated, then this function waits until
  * the terminating thread reached the zombie state.
  *
- * @param the_thread The thread to close.
- * @param executing The currently executing thread.
- * @param[in, out] context The thread close context.
+ * @param the_thread is the thread to close.
+ *
+ * @param[in, out] executing is the currently executing thread.
+ *
+ * @param[in, out] queue_context is the thread queue context.  The caller shall
+ *   have disabled interrupts using the thread queue context.
+ *
+ * @retval STATUS_SUCCESSFUL The operation was successful.
+ *
+ * @retval STATUS_DEADLOCK A deadlock occurred.
  */
-void _Thread_Close(
+Status_Control _Thread_Close(
   Thread_Control       *the_thread,
   Thread_Control       *executing,
-  Thread_Close_context *context
+  Thread_queue_Context *queue_context
 );
 
 /**