PTY: Fix some tricky issues. Now seems to be working. A lot more testing is needed

drivers/serial/Kconfig

@@ -2015,7 +2015,6 @@ endmenu # SCI1 Configuration
 config PSEUDOTERM
 	bool "Pseudo-Terminal (PTY) suppport"
 	default n
-	depends on EXPERIMENTAL
 	---help---
 		Enable support support for master and slave pseudo-terminal devices.
 

drivers/serial/pty.c

@@ -263,8 +263,12 @@ static int pty_open(FAR struct file *filep)
        pty_semtake(devpair);
     }
 
+#ifndef CONFIG_PSEUDOTERM_SUSV1
   /* If one side of the driver has been unlinked, then refuse further
    * opens.
+   *
+   * NOTE: We ignore this case in the SUSv1 case.  In the SUSv1 case, the
+   * master side is always unlinked.
    */
 
   if (devpair->pp_unlinked)
@@ -272,6 +276,7 @@ static int pty_open(FAR struct file *filep)
       ret = -EIDRM;
     }
   else
+#endif
     {
       /* Increment the count of open references on the driver */
 
@@ -505,7 +510,7 @@ static int pty_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
 
 #ifndef CONFIG_DISABLE_POLL
 static int pty_poll(FAR struct file *filep, FAR struct pollfd *fds,
-                        bool setup)
+                    bool setup)
 {
   FAR struct inode *inode;
   FAR struct pty_dev_s *dev;

fs/vfs/fs_open.c

@@ -233,6 +233,34 @@ int open(const char *path, int oflags, ...)
       goto errout_with_fd;
     }
 
+#ifdef CONFIG_PSEUDOTERM_SUSV1
+  /* If the return value from the open method is > 0, then we may assume that
+   * it is actually a file descriptor.  This kind of logic is only needed for
+   * /dev/ptmx:  When dev ptmx is opened, it does not return a file descriptor
+   * associated with the /dev/ptmx inode, but rather with the master device.
+   *
+   * REVISIT: This is a kludge.  It is dangerous because (1) Zero is also a
+   * valid file descriptor, and (2) there could potentially be driver open
+   * methods that return values > 0 for normal success conditions.
+   */
+
+  if (ret > 0)
+    {
+      /* Release file descriptor and inode that we allocated.  We don't
+       * need those.
+       */
+
+      files_release(fd);
+      inode_release(inode);
+
+      /* Instead, return the descriptor associated with the master side
+       * device.
+       */
+
+      fd = ret;
+    }
+#endif
+
   return fd;
 
 errout_with_fd: