score: Rework CORE priority ceiling mutex

Rework seize and surrender methods to use CORE_ceiling_mutex_Control.
This eliminates CORE_mutex_Disciplines.

6 files changed, 199 insertions, 160 deletions

diff --git a/cpukit/score/include/rtems/score/coremutex.h b/cpukit/score/include/rtems/score/coremutex.h
index 173bb2efa6..704ea0da24 100644
--- a/cpukit/score/include/rtems/score/coremutex.h
+++ b/cpukit/score/include/rtems/score/coremutex.h
@@ -42,22 +42,6 @@ extern "C" {
 /**@{*/
 
 /**
- *  @brief The blocking disciplines for a mutex.
- *
- *  This enumerated type defines the blocking disciplines for a mutex.
- */
-typedef enum {
-  /** This specifies that threads will wait for the mutex in priority order.
-   *  Additionally, the Priority Inheritance Protocol will be in effect.
-   */
-  CORE_MUTEX_DISCIPLINES_PRIORITY_INHERIT,
-  /** This specifies that threads will wait for the mutex in priority order.
-   *  Additionally, the Priority Ceiling Protocol will be in effect.
-   */
-  CORE_MUTEX_DISCIPLINES_PRIORITY_CEILING
-}   CORE_mutex_Disciplines;
-
-/**
  *  @brief The possible behaviors for lock nesting.
  *
  *  This enumerated type defines the possible behaviors for
@@ -96,14 +80,6 @@ typedef struct {
    *  be when attempting to acquire the mutex when it is already locked.
    */
   CORE_mutex_Nesting_behaviors lock_nesting_behavior;
-  /** This field indicates whether threads waiting on the mutex block in
-   *  FIFO or priority order.
-   */
-  CORE_mutex_Disciplines       discipline;
-  /** This field contains the ceiling priority to be used if that protocol
-   *  is selected.
-   */
-  Priority_Control             priority_ceiling;
 }   CORE_mutex_Attributes;
 
 /**
diff --git a/cpukit/score/include/rtems/score/coremuteximpl.h b/cpukit/score/include/rtems/score/coremuteximpl.h
index fc2ffd9020..f459743346 100644
--- a/cpukit/score/include/rtems/score/coremuteximpl.h
+++ b/cpukit/score/include/rtems/score/coremuteximpl.h
@@ -128,42 +128,6 @@ RTEMS_INLINE_ROUTINE bool _CORE_mutex_Is_locked(
 }
 
 /**
- * @brief Does mutex use priority inheritance.
- *
- * This routine returns true if the mutex's wait discipline is
- * INHERIT_PRIORITY and false otherwise.
- *
- * @param[in] the_attribute is the attribute set of the mutex.
- *
- * @retval true The mutex is using priority inheritance.
- * @retval false The mutex is not using priority inheritance.
- */
-RTEMS_INLINE_ROUTINE bool _CORE_mutex_Is_inherit_priority(
-  const CORE_mutex_Attributes *the_attribute
-)
-{
-  return the_attribute->discipline == CORE_MUTEX_DISCIPLINES_PRIORITY_INHERIT;
-}
-
-/**
- * @brief Does mutex use priority ceiling.
- *
- * This routine returns true if the mutex's wait discipline is
- * PRIORITY_CEILING and false otherwise.
- *
- * @param[in] the_attribute is the attribute set of the mutex.
- *
- * @retval true The mutex is using priority ceiling.
- * @retval false The mutex is not using priority ceiling.
- */
-RTEMS_INLINE_ROUTINE bool _CORE_mutex_Is_priority_ceiling(
-  const CORE_mutex_Attributes *the_attribute
-)
-{
-  return the_attribute->discipline == CORE_MUTEX_DISCIPLINES_PRIORITY_CEILING;
-}
-
-/**
  *  @brief Attempt to receive a unit from the_mutex.
  *
  *  This routine attempts to receive a unit from the_mutex.
@@ -191,40 +155,7 @@ RTEMS_INLINE_ROUTINE Status_Control _CORE_mutex_Seize_interrupt_trylock(
     the_mutex->nest_count = 1;
     ++executing->resource_count;
 
-    if ( !_CORE_mutex_Is_priority_ceiling( &the_mutex->Attributes ) ) {
-      _CORE_mutex_Release( the_mutex, queue_context );
-    } else {
-      /*
-       * must be CORE_MUTEX_DISCIPLINES_PRIORITY_CEILING
-       *
-       * we possibly bump the priority of the current holder -- which
-       * happens to be _Thread_Executing.
-       */
-      Priority_Control  ceiling;
-      Priority_Control  current;
-
-      ceiling = the_mutex->Attributes.priority_ceiling;
-      current = executing->current_priority;
-      if ( current == ceiling ) {
-        _CORE_mutex_Release( the_mutex, queue_context );
-      } else if ( current > ceiling ) {
-        Per_CPU_Control *cpu_self;
-
-        cpu_self = _Thread_Dispatch_disable_critical(
-          &queue_context->Lock_context
-        );
-        _CORE_mutex_Release( the_mutex, queue_context );
-        _Thread_Raise_priority( executing, ceiling );
-        _Thread_Dispatch_enable( cpu_self );
-      } else /* if ( current < ceiling ) */ {
-        the_mutex->holder = NULL;
-        the_mutex->nest_count = 0;     /* undo locking above */
-        executing->resource_count--;   /* undo locking above */
-        _CORE_mutex_Release( the_mutex, queue_context );
-        return STATUS_MUTEX_CEILING_VIOLATED;
-      }
-    }
-
+    _CORE_mutex_Release( the_mutex, queue_context );
     return STATUS_SUCCESSFUL;
   }
 
@@ -345,6 +276,32 @@ RTEMS_INLINE_ROUTINE bool _CORE_mutex_Is_owner(
   return _CORE_mutex_Get_owner( the_mutex ) == the_thread;
 }
 
+RTEMS_INLINE_ROUTINE void _CORE_mutex_Restore_priority(
+  Thread_Control *executing
+)
+{
+  /*
+   *  Whether or not someone is waiting for the mutex, an
+   *  inherited priority must be lowered if this is the last
+   *  mutex (i.e. resource) this task has.
+   */
+  if ( !_Thread_Owns_resources( executing ) ) {
+    /*
+     * Ensure that the executing resource count is visible to all other
+     * processors and that we read the latest priority restore hint.
+     */
+    _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
+
+    if ( executing->priority_restore_hint ) {
+      Per_CPU_Control *cpu_self;
+
+      cpu_self = _Thread_Dispatch_disable();
+      _Thread_Restore_priority( executing );
+      _Thread_Dispatch_enable( cpu_self );
+    }
+  }
+}
+
 RTEMS_INLINE_ROUTINE void _CORE_recursive_mutex_Initialize(
   CORE_recursive_mutex_Control *the_mutex
 )
@@ -479,6 +436,159 @@ RTEMS_INLINE_ROUTINE void _CORE_recursive_mutex_Surrender_no_protocol_classic(
   );
 }
 
+RTEMS_INLINE_ROUTINE void _CORE_ceiling_mutex_Initialize(
+  CORE_ceiling_mutex_Control *the_mutex,
+  Priority_Control            priority_ceiling
+)
+{
+  _CORE_recursive_mutex_Initialize( &the_mutex->Recursive );
+  the_mutex->priority_ceiling = priority_ceiling;
+}
+
+RTEMS_INLINE_ROUTINE Status_Control _CORE_ceiling_mutex_Set_owner(
+  CORE_ceiling_mutex_Control *the_mutex,
+  Thread_Control             *owner,
+  Thread_queue_Context       *queue_context
+)
+{
+  Priority_Control  priority_ceiling;
+  Priority_Control  current_priority;
+  Per_CPU_Control  *cpu_self;
+
+  priority_ceiling = the_mutex->priority_ceiling;
+  current_priority = owner->current_priority;
+
+  if ( current_priority < priority_ceiling ) {
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+    return STATUS_MUTEX_CEILING_VIOLATED;
+  }
+
+  _CORE_mutex_Set_owner( &the_mutex->Recursive.Mutex, owner );
+  ++owner->resource_count;
+
+  if ( current_priority == priority_ceiling ) {
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+    return STATUS_SUCCESSFUL;
+  }
+
+  cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context );
+  _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+  _Thread_Raise_priority( owner, priority_ceiling );
+  _Thread_Dispatch_enable( cpu_self );
+  return STATUS_SUCCESSFUL;
+}
+
+RTEMS_INLINE_ROUTINE Status_Control _CORE_ceiling_mutex_Seize(
+  CORE_ceiling_mutex_Control    *the_mutex,
+  Thread_Control                *executing,
+  bool                           wait,
+  Watchdog_Interval              timeout,
+  Status_Control              ( *nested )( CORE_recursive_mutex_Control * ),
+  Thread_queue_Context          *queue_context
+)
+{
+  Thread_Control *owner;
+
+  _CORE_mutex_Acquire_critical( &the_mutex->Recursive.Mutex, queue_context );
+
+  owner = _CORE_mutex_Get_owner( &the_mutex->Recursive.Mutex );
+
+  if ( owner == NULL ) {
+    return _CORE_ceiling_mutex_Set_owner(
+      the_mutex,
+      executing,
+      queue_context
+    );
+  }
+
+  if ( owner == executing ) {
+    Status_Control status;
+
+    status = ( *nested )( &the_mutex->Recursive );
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+    return status;
+  }
+
+  return _CORE_mutex_Seize_no_protocol_slow(
+    &the_mutex->Recursive.Mutex,
+    CORE_MUTEX_TQ_OPERATIONS,
+    executing,
+    wait,
+    timeout,
+    queue_context
+  );
+}
+
+RTEMS_INLINE_ROUTINE Status_Control _CORE_ceiling_mutex_Surrender(
+  CORE_ceiling_mutex_Control *the_mutex,
+  Thread_Control             *executing,
+  Thread_queue_Context       *queue_context
+)
+{
+  unsigned int    nest_level;
+  Thread_Control *new_owner;
+
+  _CORE_mutex_Acquire_critical( &the_mutex->Recursive.Mutex, queue_context );
+
+  if ( !_CORE_mutex_Is_owner( &the_mutex->Recursive.Mutex, executing ) ) {
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+    return STATUS_NOT_OWNER;
+  }
+
+  nest_level = the_mutex->Recursive.nest_level;
+
+  if ( nest_level > 0 ) {
+    the_mutex->Recursive.nest_level = nest_level - 1;
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+    return STATUS_SUCCESSFUL;
+  }
+
+  --executing->resource_count;
+
+  new_owner = _Thread_queue_First_locked(
+    &the_mutex->Recursive.Mutex.Wait_queue,
+    CORE_MUTEX_TQ_OPERATIONS
+  );
+  _CORE_mutex_Set_owner( &the_mutex->Recursive.Mutex, new_owner );
+
+  if ( new_owner != NULL ) {
+    bool unblock;
+
+    /*
+     * We must extract the thread now since this will restore its default
+     * thread lock.  This is necessary to avoid a deadlock in the
+     * _Thread_Change_priority() below due to a recursive thread queue lock
+     * acquire.
+     */
+    unblock = _Thread_queue_Extract_locked(
+      &the_mutex->Recursive.Mutex.Wait_queue.Queue,
+      CORE_MUTEX_TQ_OPERATIONS,
+      new_owner,
+      queue_context
+    );
+
+#if defined(RTEMS_MULTIPROCESSING)
+    if ( _Objects_Is_local_id( new_owner->Object.id ) )
+#endif
+    {
+      ++new_owner->resource_count;
+      _Thread_Raise_priority( new_owner, the_mutex->priority_ceiling );
+    }
+
+    _Thread_queue_Unblock_critical(
+      unblock,
+      &the_mutex->Recursive.Mutex.Wait_queue.Queue,
+      new_owner,
+      &queue_context->Lock_context
+    );
+  } else {
+    _CORE_mutex_Release( &the_mutex->Recursive.Mutex, queue_context );
+  }
+
+  _CORE_mutex_Restore_priority( executing );
+  return STATUS_SUCCESSFUL;
+}
+
 /** @} */
 
 #ifdef __cplusplus
diff --git a/cpukit/score/src/apimutex.c b/cpukit/score/src/apimutex.c
index 8af374aaef..e3c6e5a705 100644
--- a/cpukit/score/src/apimutex.c
+++ b/cpukit/score/src/apimutex.c
@@ -48,9 +48,7 @@ void _API_Mutex_Allocate(
   API_Mutex_Control *mutex;
 
   CORE_mutex_Attributes attr =  {
-    CORE_MUTEX_NESTING_ACQUIRES,
-    CORE_MUTEX_DISCIPLINES_PRIORITY_INHERIT,
-    0
+    CORE_MUTEX_NESTING_ACQUIRES
   };
 
   mutex = (API_Mutex_Control *)
diff --git a/cpukit/score/src/coremutex.c b/cpukit/score/src/coremutex.c
index 6f73c1bd16..9c6b7a8b93 100644
--- a/cpukit/score/src/coremutex.c
+++ b/cpukit/score/src/coremutex.c
@@ -39,41 +39,9 @@ Status_Control _CORE_mutex_Initialize(
   the_mutex->Attributes    = *the_mutex_attributes;
 
   if ( initially_locked ) {
-    bool              is_priority_ceiling;
-    Priority_Control  ceiling;
-    Per_CPU_Control  *cpu_self;
-
     the_mutex->nest_count = 1;
     the_mutex->holder     = executing;
-
-    /* The mutex initialization is only protected by the allocator lock */
-    cpu_self = _Thread_Dispatch_disable();
-
-    is_priority_ceiling =
-      _CORE_mutex_Is_priority_ceiling( &the_mutex->Attributes );
-    ceiling = the_mutex->Attributes.priority_ceiling;
-
-    /*
-     * The test to check for a ceiling violation is a bit arbitrary.  In case
-     * this thread is the owner of a priority inheritance mutex, then it may
-     * get a higher priority later or anytime on SMP configurations.
-     */
-    if ( is_priority_ceiling && executing->current_priority < ceiling ) {
-      /*
-       * There is no need to undo the previous work since this error aborts
-       * the object creation.
-       */
-      _Thread_Dispatch_enable( cpu_self );
-      return STATUS_MUTEX_CEILING_VIOLATED;
-    }
-
     executing->resource_count++;
-
-    if ( is_priority_ceiling ) {
-      _Thread_Raise_priority( executing, ceiling );
-    }
-
-    _Thread_Dispatch_enable( cpu_self );
   } else {
     the_mutex->nest_count = 0;
     the_mutex->holder     = NULL;
diff --git a/cpukit/score/src/coremutexseize.c b/cpukit/score/src/coremutexseize.c
index 596378fda6..ed5eb0aeec 100644
--- a/cpukit/score/src/coremutexseize.c
+++ b/cpukit/score/src/coremutexseize.c
@@ -29,6 +29,8 @@ Status_Control _CORE_mutex_Seize_interrupt_blocking(
   Thread_queue_Context *queue_context
 )
 {
+  Thread_Control *holder;
+
 #if !defined(RTEMS_SMP)
   /*
    * We must disable thread dispatching here since we enable the interrupts for
@@ -37,32 +39,27 @@ Status_Control _CORE_mutex_Seize_interrupt_blocking(
   _Thread_Dispatch_disable();
 #endif
 
-  if ( _CORE_mutex_Is_inherit_priority( &the_mutex->Attributes ) ) {
-    Thread_Control *holder = the_mutex->holder;
+  holder = the_mutex->holder;
 
 #if !defined(RTEMS_SMP)
-    /*
-     * To enable interrupts here works only since exactly one executing thread
-     * exists and only threads are allowed to seize and surrender mutexes with
-     * the priority inheritance protocol.  On SMP configurations more than one
-     * executing thread may exist, so here we must not release the lock, since
-     * otherwise the current holder may be no longer the holder of the mutex
-     * once we released the lock.
-     */
-    _CORE_mutex_Release( the_mutex, queue_context );
+  /*
+   * To enable interrupts here works only since exactly one executing thread
+   * exists and only threads are allowed to seize and surrender mutexes with
+   * the priority inheritance protocol.  On SMP configurations more than one
+   * executing thread may exist, so here we must not release the lock, since
+   * otherwise the current holder may be no longer the holder of the mutex
+   * once we released the lock.
+   */
+  _CORE_mutex_Release( the_mutex, queue_context );
 #endif
 
-    _Thread_Inherit_priority( holder, executing );
-
-#if !defined(RTEMS_SMP)
-    _ISR_lock_ISR_disable( &queue_context->Lock_context );
-    _CORE_mutex_Acquire_critical( the_mutex, queue_context );
-#endif
-  }
+  _Thread_Inherit_priority( holder, executing );
 
 #if defined(RTEMS_SMP)
   _Thread_queue_Context_set_expected_level( queue_context, 1 );
 #else
+  _ISR_lock_ISR_disable( &queue_context->Lock_context );
+  _CORE_mutex_Acquire_critical( the_mutex, queue_context );
   _Thread_queue_Context_set_expected_level( queue_context, 2 );
 #endif
 
diff --git a/cpukit/score/src/coremutexsurrender.c b/cpukit/score/src/coremutexsurrender.c
index 2d976e0c22..5075e204c9 100644
--- a/cpukit/score/src/coremutexsurrender.c
+++ b/cpukit/score/src/coremutexsurrender.c
@@ -118,19 +118,8 @@ Status_Control _CORE_mutex_Surrender(
     if ( _Objects_Is_local_id( the_thread->Object.id ) )
 #endif
     {
-      switch ( the_mutex->Attributes.discipline ) {
-        case CORE_MUTEX_DISCIPLINES_PRIORITY_INHERIT:
-          the_thread->resource_count++;
-          _Thread_queue_Boost_priority( &the_mutex->Wait_queue.Queue, the_thread );
-          break;
-        case CORE_MUTEX_DISCIPLINES_PRIORITY_CEILING:
-          the_thread->resource_count++;
-          _Thread_Raise_priority(
-            the_thread,
-            the_mutex->Attributes.priority_ceiling
-          );
-          break;
-      }
+      the_thread->resource_count++;
+      _Thread_queue_Boost_priority( &the_mutex->Wait_queue.Queue, the_thread );
     }
 
     _Thread_queue_Unblock_critical(
@@ -164,5 +153,6 @@ Status_Control _CORE_mutex_Surrender(
     }
   }
 
+  _CORE_mutex_Restore_priority( holder );
   return STATUS_SUCCESSFUL;
 }