rpmsg: support the fdsan feature for rpmsg dev/mtd/blk

Directly return -ENOTTY in rpmsgxxx_ioctl() when the command is
not supported to avoid fdsan command FIOC_SETTAG and FIOC_GETTAG
pass to the rpmsg dev/mtd/blk server with CONFIG_FDSAN enabled.

Signed-off-by: wangbowen6 <wangbowen6@xiaomi.com>

drivers/misc/rpmsgblk.c

@@ -86,7 +86,7 @@ static ssize_t rpmsgblk_write(FAR struct inode *inode,
                               blkcnt_t start_sector, unsigned int nsectors);
 static int     rpmsgblk_geometry(FAR struct inode *inode,
                                  FAR struct geometry *geometry);
-static size_t  rpmsgblk_ioctl_arglen(int cmd);
+static ssize_t rpmsgblk_ioctl_arglen(int cmd);
 static int     rpmsgblk_ioctl(FAR struct inode *inode, int cmd,
                               unsigned long arg);
 #ifndef CONFIG_DISABLE_PSEUDOFS_OPERATIONS
@@ -499,7 +499,7 @@ out:
  *
  ****************************************************************************/
 
-static size_t rpmsgblk_ioctl_arglen(int cmd)
+static ssize_t rpmsgblk_ioctl_arglen(int cmd)
 {
   switch (cmd)
     {
@@ -527,7 +527,7 @@ static size_t rpmsgblk_ioctl_arglen(int cmd)
       case BIOC_BLKSSZGET:
         return sizeof(blksize_t);
       default:
-        return 0;
+        return -ENOTTY;
     }
 }
 
@@ -553,7 +553,7 @@ static int rpmsgblk_ioctl(FAR struct inode *inode, int cmd,
   FAR struct rpmsgblk_s *priv = (FAR struct rpmsgblk_s *)inode->i_private;
   FAR struct rpmsgblk_ioctl_s *msg;
   uint32_t space;
-  size_t arglen;
+  ssize_t arglen;
   size_t msglen;
 
   /* Sanity checks */
@@ -563,6 +563,11 @@ static int rpmsgblk_ioctl(FAR struct inode *inode, int cmd,
   /* Call our internal routine to perform the ioctl */
 
   arglen = rpmsgblk_ioctl_arglen(cmd);
+  if (arglen < 0)
+    {
+      return arglen;
+    }
+
   msglen = sizeof(*msg) + arglen - 1;
 
   msg = rpmsgblk_get_tx_payload_buffer(priv, &space);

drivers/misc/rpmsgdev.c

@@ -96,7 +96,7 @@ static ssize_t rpmsgdev_write(FAR struct file *filep, FAR const char *buffer,
                               size_t buflen);
 static off_t   rpmsgdev_seek(FAR struct file *filep, off_t offset,
                              int whence);
-static size_t  rpmsgdev_ioctl_arglen(int cmd);
+static ssize_t rpmsgdev_ioctl_arglen(int cmd);
 static int     rpmsgdev_ioctl(FAR struct file *filep, int cmd,
                               unsigned long arg);
 static int     rpmsgdev_poll(FAR struct file *filep, FAR struct pollfd *fds,
@@ -616,7 +616,7 @@ static off_t rpmsgdev_seek(FAR struct file *filep, off_t offset, int whence)
  *
  ****************************************************************************/
 
-static size_t rpmsgdev_ioctl_arglen(int cmd)
+static ssize_t rpmsgdev_ioctl_arglen(int cmd)
 {
   switch (cmd)
     {
@@ -628,7 +628,7 @@ static size_t rpmsgdev_ioctl_arglen(int cmd)
       case FBIOGET_POWER:
         return sizeof(int);
       default:
-        return 0;
+        return -ENOTTY;
     }
 }
 
@@ -654,7 +654,7 @@ static int rpmsgdev_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
   FAR struct rpmsgdev_priv_s *priv;
   FAR struct rpmsgdev_ioctl_s *msg;
   uint32_t space;
-  size_t arglen;
+  ssize_t arglen;
   size_t msglen;
   int ret;
 
@@ -671,6 +671,11 @@ static int rpmsgdev_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
   /* Call our internal routine to perform the ioctl */
 
   arglen = rpmsgdev_ioctl_arglen(cmd);
+  if (arglen < 0)
+    {
+      return arglen;
+    }
+
   msglen = sizeof(*msg) + arglen - 1;
 
   msg = rpmsgdev_get_tx_payload_buffer(dev, &space);

drivers/mtd/rpmsgmtd.c

@@ -85,7 +85,7 @@ static ssize_t rpmsgmtd_read(FAR struct mtd_dev_s *dev, off_t offset,
 static ssize_t rpmsgmtd_write(FAR struct mtd_dev_s *dev, off_t offset,
                               size_t nbytes, FAR const uint8_t *buffer);
 #endif
-static size_t  rpmsgmtd_ioctl_arglen(int cmd);
+static ssize_t rpmsgmtd_ioctl_arglen(int cmd);
 static int     rpmsgmtd_ioctl(FAR struct mtd_dev_s *dev, int cmd,
                               unsigned long arg);
 
@@ -548,7 +548,7 @@ out:
  *
  ****************************************************************************/
 
-static size_t rpmsgmtd_ioctl_arglen(int cmd)
+static ssize_t rpmsgmtd_ioctl_arglen(int cmd)
 {
   switch (cmd)
     {
@@ -558,7 +558,7 @@ static size_t rpmsgmtd_ioctl_arglen(int cmd)
       case MTDIOC_UNPROTECT:
         return sizeof(struct mtd_protect_s);
       default:
-        return 0;
+        return -ENOTTY;
     }
 }
 
@@ -584,7 +584,7 @@ static int rpmsgmtd_ioctl(FAR struct mtd_dev_s *dev, int cmd,
   FAR struct rpmsgmtd_s *priv = (FAR struct rpmsgmtd_s *)dev;
   FAR struct rpmsgmtd_ioctl_s *msg;
   uint32_t space;
-  size_t arglen;
+  ssize_t arglen;
   size_t msglen;
 
   /* Sanity checks */
@@ -594,6 +594,11 @@ static int rpmsgmtd_ioctl(FAR struct mtd_dev_s *dev, int cmd,
   /* Call our internal routine to perform the ioctl */
 
   arglen = rpmsgmtd_ioctl_arglen(cmd);
+  if (arglen < 0)
+    {
+      return arglen;
+    }
+
   msglen = sizeof(*msg) + arglen - 1;
 
   msg = rpmsgmtd_get_tx_payload_buffer(priv, &space);

fs/rpmsgfs/rpmsgfs_client.c

@@ -380,7 +380,7 @@ fail:
   return ret;
 }
 
-static size_t rpmsgfs_ioctl_arglen(int cmd)
+static ssize_t rpmsgfs_ioctl_arglen(int cmd)
 {
   switch (cmd)
     {
@@ -389,7 +389,7 @@ static size_t rpmsgfs_ioctl_arglen(int cmd)
       case FIONREAD:
         return sizeof(int);
       default:
-        return 0;
+        return -ENOTTY;
     }
 }
 
@@ -567,12 +567,17 @@ off_t rpmsgfs_client_lseek(FAR void *handle, int fd,
 int rpmsgfs_client_ioctl(FAR void *handle, int fd,
                          int request, unsigned long arg)
 {
-  size_t arglen = rpmsgfs_ioctl_arglen(request);
+  ssize_t arglen = rpmsgfs_ioctl_arglen(request);
   FAR struct rpmsgfs_s *priv = handle;
   FAR struct rpmsgfs_ioctl_s *msg;
   uint32_t space;
   size_t len;
 
+  if (arglen < 0)
+    {
+      return arglen;
+    }
+
   len = sizeof(*msg) + arglen;
   msg = rpmsgfs_get_tx_payload_buffer(priv, &space);
   if (msg == NULL)