From 4b1553d3ad8a640af057bd728a66ea59881bdb24 Mon Sep 17 00:00:00 2001
From: Gregory Nutt <gnutt@nuttx.org>
Date: Fri, 15 Jul 2016 14:29:32 -0600
Subject: [PATCH] PTY: Fix some tricky issues.  Now seems to be working.  A lot
 more testing is needed

---
 drivers/serial/Kconfig |  1 -
 drivers/serial/pty.c   |  7 ++++++-
 fs/vfs/fs_open.c       | 28 ++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/drivers/serial/Kconfig b/drivers/serial/Kconfig
index 68a8d59739..ad738a4824 100644
--- a/drivers/serial/Kconfig
+++ b/drivers/serial/Kconfig
@@ -2015,7 +2015,6 @@ endmenu # SCI1 Configuration
 config PSEUDOTERM
 	bool "Pseudo-Terminal (PTY) suppport"
 	default n
-	depends on EXPERIMENTAL
 	---help---
 		Enable support support for master and slave pseudo-terminal devices.
 
diff --git a/drivers/serial/pty.c b/drivers/serial/pty.c
index f55dbddf65..ec2db0002e 100644
--- a/drivers/serial/pty.c
+++ b/drivers/serial/pty.c
@@ -263,8 +263,12 @@ static int pty_open(FAR struct file *filep)
        pty_semtake(devpair);
     }
 
+#ifndef CONFIG_PSEUDOTERM_SUSV1
   /* If one side of the driver has been unlinked, then refuse further
    * opens.
+   *
+   * NOTE: We ignore this case in the SUSv1 case.  In the SUSv1 case, the
+   * master side is always unlinked.
    */
 
   if (devpair->pp_unlinked)
@@ -272,6 +276,7 @@ static int pty_open(FAR struct file *filep)
       ret = -EIDRM;
     }
   else
+#endif
     {
       /* Increment the count of open references on the driver */
 
@@ -505,7 +510,7 @@ static int pty_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
 
 #ifndef CONFIG_DISABLE_POLL
 static int pty_poll(FAR struct file *filep, FAR struct pollfd *fds,
-                        bool setup)
+                    bool setup)
 {
   FAR struct inode *inode;
   FAR struct pty_dev_s *dev;
diff --git a/fs/vfs/fs_open.c b/fs/vfs/fs_open.c
index 81a2ef7f7c..462aac6734 100644
--- a/fs/vfs/fs_open.c
+++ b/fs/vfs/fs_open.c
@@ -233,6 +233,34 @@ int open(const char *path, int oflags, ...)
       goto errout_with_fd;
     }
 
+#ifdef CONFIG_PSEUDOTERM_SUSV1
+  /* If the return value from the open method is > 0, then we may assume that
+   * it is actually a file descriptor.  This kind of logic is only needed for
+   * /dev/ptmx:  When dev ptmx is opened, it does not return a file descriptor
+   * associated with the /dev/ptmx inode, but rather with the master device.
+   *
+   * REVISIT: This is a kludge.  It is dangerous because (1) Zero is also a
+   * valid file descriptor, and (2) there could potentially be driver open
+   * methods that return values > 0 for normal success conditions.
+   */
+
+  if (ret > 0)
+    {
+      /* Release file descriptor and inode that we allocated.  We don't
+       * need those.
+       */
+
+      files_release(fd);
+      inode_release(inode);
+
+      /* Instead, return the descriptor associated with the master side
+       * device.
+       */
+
+      fd = ret;
+    }
+#endif
+
   return fd;
 
 errout_with_fd: