From ff87e2e02a5b40807cdb0ec7ef0ba48516e5ac64 Mon Sep 17 00:00:00 2001
From: Gregory Nutt <gnutt@nuttx.org>
Date: Sat, 6 Dec 2014 07:18:48 -0600
Subject: [PATCH] In message queue created return ENOSPC error if size exceeds
 the configured size of pre-allocatd messages; Use ENOSPC vs ENOMEM per
 OpenGroup.org.  From  Pierre-Noel Bouteville

---
 fs/mqueue/mq_open.c         | 13 ++-----------
 sched/mqueue/mq_msgqalloc.c | 32 +++++++++++++++++++-------------
 sched/mqueue/mqueue.h       |  2 +-
 3 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/fs/mqueue/mq_open.c b/fs/mqueue/mq_open.c
index 0769712873..f2b95b9b5a 100644
--- a/fs/mqueue/mq_open.c
+++ b/fs/mqueue/mq_open.c
@@ -52,17 +52,8 @@
 #include "inode/inode.h"
 #include "mqueue/mqueue.h"
 
-#if 0
-#include <sys/types.h>
-#include <stdint.h>
-#include <debug.h>
-
-
-#include "sched/sched.h"
-#endif
-
 /****************************************************************************
- * Pre-procesor Definitions
+ * Pre-processor Definitions
  ****************************************************************************/
 
 /****************************************************************************
@@ -220,7 +211,7 @@ mqd_t mq_open(FAR const char *mq_name, int oflags, ...)
       msgq = (FAR struct mqueue_inode_s*)mq_msgqalloc(mode, attr);
       if (!msgq)
         {
-          errcode = ENOMEM;
+          errcode = ENOSPC;
           goto errout_with_inode;
         }
 
diff --git a/sched/mqueue/mq_msgqalloc.c b/sched/mqueue/mq_msgqalloc.c
index 6b5cea76e4..280f1f1821 100644
--- a/sched/mqueue/mq_msgqalloc.c
+++ b/sched/mqueue/mq_msgqalloc.c
@@ -40,6 +40,7 @@
 #include <nuttx/config.h>
 
 #include <mqueue.h>
+#include <assert.h>
 
 #include <nuttx/kmalloc.h>
 #include <nuttx/sched.h>
@@ -77,16 +78,16 @@
  *
  * Description:
  *   This function implements a part of the POSIX message queue open logic.
- *   It allocates and initializes a structu mqueue_inode_s structure.
+ *   It allocates and initializes a struct mqueue_inode_s structure.
  *
  * Parameters:
  *   mode   - mode_t value is ignored
  *   attr   - The mq_maxmsg attribute is used at the time that the message
  *            queue is created to determine the maximum number of
- *             messages that may be placed in the message queue.
+ *            messages that may be placed in the message queue.
  *
  * Return Value:
- *   The allocated and initalized message queue structure or NULL in the
+ *   The allocated and initialized message queue structure or NULL in the
  *   event of a failure.
  *
  ****************************************************************************/
@@ -96,9 +97,21 @@ FAR struct mqueue_inode_s *mq_msgqalloc(mode_t mode,
 {
   FAR struct mqueue_inode_s *msgq;
 
+  /* Check if the caller is attempting to allocate a message for messages
+   * larger than the configured maximum message size.
+   */
+
+  DEBUGASSERT(!attr || attr->mq_msgsize <= MQ_MAX_BYTES);
+  if (attr && attr->mq_msgsize > MQ_MAX_BYTES)
+    {
+      return NULL;
+    }
+
   /* Allocate memory for the new message queue. */
 
-  msgq = (FAR struct mqueue_inode_s*)kmm_zalloc(sizeof(struct mqueue_inode_s));
+  msgq = (FAR struct mqueue_inode_s*)
+    kmm_zalloc(sizeof(struct mqueue_inode_s));
+
   if (msgq)
     {
       /* Initialize the new named message queue */
@@ -106,15 +119,8 @@ FAR struct mqueue_inode_s *mq_msgqalloc(mode_t mode,
       sq_init(&msgq->msglist);
       if (attr)
         {
-          msgq->maxmsgs = (int16_t)attr->mq_maxmsg;
-          if (attr->mq_msgsize <= MQ_MAX_BYTES)
-            {
-              msgq->maxmsgsize = (int16_t)attr->mq_msgsize;
-            }
-          else
-            {
-              msgq->maxmsgsize = MQ_MAX_BYTES;
-            }
+          msgq->maxmsgs    = (int16_t)attr->mq_maxmsg;
+          msgq->maxmsgsize = (int16_t)attr->mq_msgsize;
         }
       else
         {
diff --git a/sched/mqueue/mqueue.h b/sched/mqueue/mqueue.h
index 03bc8d050e..bd771e140e 100644
--- a/sched/mqueue/mqueue.h
+++ b/sched/mqueue/mqueue.h
@@ -98,7 +98,7 @@ struct mqueue_msg_s
 #else
   uint16_t msglen;                /* Message data length */
 #endif
-  uint8_t mail[MQ_MAX_BYTES];     /* Message data */
+  char mail[MQ_MAX_BYTES];        /* Message data */
 };
 
 /****************************************************************************