Fix an assertion noted by Jussi Kivilinna.

This was a consequence of the recent robust mutex changes.  If robust mutexes are selected, then each mutex that a thread takes is retained in a list in threads TCB.  If the thread exits and that list is not empty, then we know that the thread exitted while holding mutexes.  And, in that case, each will be marked as inconsistent and the any waiter for the thread is awakened.

For the case of pthread_mutex_trywait(), the mutex was not being added to the list!  while not usually a fatal error, this was caught by an assertion when pthread_mutex_unlock() was called:  It tried to remove the mutex from the TCB list and it was not there when, of course, it shoule be.

The fix was to add pthread_mutex_trytake() which does sem_trywait() and if successful, does correctly add the mutext to the TCB list.  This should eliminated the assertion.

sched/pthread/pthread.h

@@ -112,10 +112,12 @@ int pthread_givesemaphore(sem_t *sem);
 
 #ifndef CONFIG_PTHREAD_MUTEX_UNSAFE
 int pthread_mutex_take(FAR struct pthread_mutex_s *mutex, bool intr);
+int pthread_mutex_trytake(FAR struct pthread_mutex_s *mutex);
 int pthread_mutex_give(FAR struct pthread_mutex_s *mutex);
 void pthread_mutex_inconsistent(FAR struct pthread_tcb_s *tcb);
 #else
 #  define pthread_mutex_take(m,i)  pthread_takesemaphore(&(m)->sem,(i))
+#  define pthread_mutex_trytake(m) sem_trywait(&(m)->sem)
 #  define pthread_mutex_give(m)    pthread_givesemaphore(&(m)->sem)
 #endif
 

sched/pthread/pthread_mutex.c

@@ -50,6 +50,39 @@
 #include "sched/sched.h"
 #include "pthread/pthread.h"
 
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: pthread_mutex_add
+ *
+ * Description:
+ *   Add the mutex to the list of mutexes held by this thread.
+ *
+ * Parameters:
+ *  mutex - The mux to be locked
+ *
+ * Return Value:
+ *   None
+ *
+ ****************************************************************************/
+
+static void pthread_mutex_add(FAR struct pthread_mutex_s *mutex)
+{
+  FAR struct pthread_tcb_s *rtcb = (FAR struct pthread_tcb_s *)this_task();
+  irqstate_t flags;
+
+  DEBUGASSERT(mutex->flink == NULL);
+
+  /* Add the mutex to the list of mutexes held by this task */
+
+  flags        = enter_critical_section();
+  mutex->flink = rtcb->mhead;
+  rtcb->mhead  = mutex;
+  leave_critical_section(flags);
+}
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -58,8 +91,8 @@
  * Name: pthread_mutex_take
  *
  * Description:
- *   Take the pthread_mutex and, if successful, add the mutex to the ist of
- *   mutexes held by this thread.
+ *   Take the pthread_mutex, waiting if necessary.  If successful, add the
+ *   mutex to the list of mutexes held by this thread.
  *
  * Parameters:
  *  mutex - The mux to be locked
@@ -84,15 +117,26 @@ int pthread_mutex_take(FAR struct pthread_mutex_s *mutex, bool intr)
 
       sched_lock();
 
+      /* Error out if the mutex is already in an inconsistent state. */
+
+      if ((mutex->flags & _PTHREAD_MFLAGS_INCONSISTENT) != 0)
+        {
+          ret = EOWNERDEAD;
+        }
+      else
+        {
           /* Take semaphore underlying the mutex */
 
           ret = pthread_takesemaphore(&mutex->sem, intr);
-      if (ret == OK)
+          if (ret < OK)
             {
-          DEBUGASSERT(mutex->flink == NULL);
-
-          /* Check if the holder of the mutex has terminated.  In that case,
-           * the state of the mutex is inconsistent and we return EOWNERDEAD.
+              ret = get_errno();
+            }
+          else
+            {
+              /* Check if the holder of the mutex has terminated without
+               * releasing.  In that case, the state of the mutex is
+               * inconsistent and we return EOWNERDEAD.
                */
 
               if ((mutex->flags & _PTHREAD_MFLAGS_INCONSISTENT) != 0)
@@ -104,18 +148,73 @@ int pthread_mutex_take(FAR struct pthread_mutex_s *mutex, bool intr)
 
               else
                 {
-              FAR struct pthread_tcb_s *rtcb = (FAR struct pthread_tcb_s *)this_task();
-              irqstate_t flags;
-
-              flags = enter_critical_section();
-              mutex->flink = rtcb->mhead;
-              rtcb->mhead = mutex;
-              leave_critical_section(flags);
+                  pthread_mutex_add(mutex);
                 }
             }
         }
 
       sched_unlock();
+    }
+
+  return ret;
+}
+
+/****************************************************************************
+ * Name: pthread_mutex_trytake
+ *
+ * Description:
+ *   Try to take the pthread_mutex without waiting.  If successful, add the
+ *   mutex to the list of mutexes held by this thread.
+ *
+ * Parameters:
+ *  mutex - The mux to be locked
+ *  intr  - false: ignore EINTR errors when locking; true tread EINTR as
+ *          other errors by returning the errno value
+ *
+ * Return Value:
+ *   0 on success or an errno value on failure.
+ *
+ ****************************************************************************/
+
+int pthread_mutex_trytake(FAR struct pthread_mutex_s *mutex)
+{
+  int ret = EINVAL;
+
+  /* Verify input parameters */
+
+  DEBUGASSERT(mutex != NULL);
+  if (mutex != NULL)
+    {
+      /* Make sure that no unexpected context switches occur */
+
+      sched_lock();
+
+      /* Error out if the mutex is already in an inconsistent state. */
+
+      if ((mutex->flags & _PTHREAD_MFLAGS_INCONSISTENT) != 0)
+        {
+          ret = EOWNERDEAD;
+        }
+      else
+        {
+          /* Try to take the semaphore underlying the mutex */
+
+          ret = sem_trywait(&mutex->sem);
+          if (ret < OK)
+            {
+              ret = get_errno();
+            }
+          else
+            {
+              /* Add the mutex to the list of mutexes held by this task */
+
+              pthread_mutex_add(mutex);
+            }
+        }
+
+      sched_unlock();
+    }
+
   return ret;
 }
 

sched/pthread/pthread_mutextrylock.c

@@ -102,7 +102,7 @@ int pthread_mutex_trylock(FAR pthread_mutex_t *mutex)
 
       /* Try to get the semaphore. */
 
-      status = sem_trywait((FAR sem_t *)&mutex->sem);
+      status = pthread_mutex_trytake(mutex);
       if (status == OK)
         {
           /* If we successfully obtained the semaphore, then indicate
@@ -120,14 +120,11 @@ int pthread_mutex_trylock(FAR pthread_mutex_t *mutex)
           ret = OK;
         }
 
-      /* sem_trywait failed */
+       /* pthread_mutex_trytake failed.  Did it fail because the semaphore
+        * was not avaialable?
+        */
 
-      else
-        {
-          /* Did it fail because the semaphore was not avaialable? */
-
-          int errcode = get_errno();
-          if (errcode == EAGAIN)
+      else if (status == EAGAIN)
         {
 #ifdef CONFIG_PTHREAD_MUTEX_TYPES
           /* Check if recursive mutex was locked by the calling thread. */
@@ -196,7 +193,6 @@ int pthread_mutex_trylock(FAR pthread_mutex_t *mutex)
 
           else
 #endif /* CONFIG_PTHREAD_MUTEX_UNSAFE */
-
             {
               ret = EBUSY;
             }
@@ -206,8 +202,7 @@ int pthread_mutex_trylock(FAR pthread_mutex_t *mutex)
 
       else
         {
-              ret = errcode;
-            }
+          ret = status;
         }
 
       sched_unlock();