Squashed commit of the following:

task_spawn() and posix_spawn() are NuttX OS interfaces.  In PROTECTED and KERNEL build modes, then can be reached from applications only via a system call.  Currently, the number of parameters in a system call is limited to six; these spawn function have seven parameters.  Rather than extend the maximum number of parameters across all architectures, I opted instead to marshal the seven parameters into a structure.
     *

In order to support builtin in function in protected mode, a task_spawn() system call must be supported.  Unfortunately this is overly complex because there is a (soft) limit of 6 parameters in a system call; task_spawn has seven paramters.  This is a soft limit but still difficult to extend because it involves assembly language changes to numerous architectures.  Better to get more creative.

include/nuttx/spawn.h

@@ -1,7 +1,7 @@
 /****************************************************************************
  * include/nuttx/spawn.h
  *
- *   Copyright (C) 2013 Gregory Nutt. All rights reserved.
+ *   Copyright (C) 2013, 2019 Gregory Nutt. All rights reserved.
  *   Author: Gregory Nutt <gnutt@nuttx.org>
  *
  * Redistribution and use in source and binary forms, with or without
@@ -51,6 +51,7 @@
 /****************************************************************************
  * Type Definitions
  ****************************************************************************/
+
 /* This enumerator identifies a file action */
 
 enum spawn_file_actions_e
@@ -102,6 +103,27 @@ struct spawn_open_file_action_s
 #define SIZEOF_OPEN_FILE_ACTION_S(n) \
   (sizeof(struct spawn_open_file_action_s) + (n))
 
+#ifdef CONFIG_LIB_SYSCALL
+/* task_spawn() and posix_spawn() are NuttX OS interfaces.  In PROTECTED and
+ * KERNEL build modes, then can be reached from applications only via a
+ * system call.  Currently, the number of parameters in a system call is
+ * limited to six; these spawn function have seven parameters.  Rather than
+ * extend the maximum number of parameters across all architectures, I opted
+ * instead to marshal the seven parameters into the following structure:
+ */
+
+struct spawn_syscall_parms_s
+{
+  FAR pid_t *pid;
+  FAR const char *name;
+  main_t entry;
+  FAR const posix_spawn_file_actions_t *file_actions;
+  FAR const posix_spawnattr_t *attr;
+  FAR char * const *argv;
+  FAR char * const *envp;
+};
+#endif
+
 /****************************************************************************
  * Public Function Prototypes
  ****************************************************************************/
@@ -113,6 +135,9 @@ extern "C"
 
 void add_file_action(FAR posix_spawn_file_actions_t *file_action,
                      FAR struct spawn_general_file_action_s *entry);
+#ifdef CONFIG_LIB_SYSCALL
+int nx_task_spawn(FAR const struct spawn_syscall_parms_s *parms);
+#endif
 
 #ifdef __cplusplus
 }

include/sys/syscall.h

@@ -128,7 +128,8 @@
 
 #ifndef CONFIG_BUILD_KERNEL
 #  define SYS_task_create              __SYS_task_create
-#  define __SYS_task_delete            (__SYS_task_create + 1)
+#  define SYS_nx_task_spawn            (__SYS_task_create + 1)
+#  define __SYS_task_delete            (__SYS_task_create + 2)
 
 /* pgalloc() is only available with address environments with the page
  * allocator selected.  MMU support from the CPU is also required.

libs/libc/spawn/Make.defs

@@ -50,6 +50,10 @@ ifneq ($(CONFIG_BUILD_KERNEL),y)
 CSRCS += lib_psa_getstacksize.c lib_psa_setstacksize.c
 endif
 
+ifeq ($(CONFIG_BUILD_PROTECTED),y)
+CSRCS += lib_task_spawn.c
+endif
+
 ifeq ($(CONFIG_DEBUG_FEATURES),y)
 CSRCS += lib_psa_dump.c
 endif

libs/libc/spawn/lib_task_spawn.c

@@ -0,0 +1,94 @@
+/****************************************************************************
+ * libs/libc/string/lib_psfa_init.c
+ *
+ *   Copyright (C) 2013 Gregory Nutt. All rights reserved.
+ *   Author: Gregory Nutt <gnutt@nuttx.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name NuttX nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <spawn.h>
+#include <nuttx/spawn.h>
+
+#if defined(CONFIG_BUILD_PROTECTED) && defined(CONFIG_LIB_SYSCALL)
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: task_spawn
+ *
+ * Description:
+ *   This function marshals task_spwan() parameters and invokes the
+ *   nx_task_spawn() system call.
+ *
+ *   task_spawn() and posix_spawn() are NuttX OS interfaces.  In PROTECTED
+ *   and KERNEL build modes, then can be reached from applications only via
+ *   a system call.  Currently, the number of parameters in a system call
+ *   is limited to six; these spawn function have seven parameters.  Rather
+ *   than extend the maximum number of parameters across all architectures,
+ *   I opted instead to marshal the seven parameters into a structure.
+ *
+ *
+ * Input Parameters:
+ *   file_actions - The address of the posix_spawn_file_actions_t to be
+ *   initialized.
+ *
+ * Returned Value:
+ *   On success, these functions return 0; on failure they return an error
+ *   number from <errno.h>.
+ *
+ ****************************************************************************/
+
+int task_spawn(FAR pid_t *pid, FAR const char *name, main_t entry,
+               FAR const posix_spawn_file_actions_t *file_actions,
+               FAR const posix_spawnattr_t *attr,
+               FAR char *const argv[], FAR char *const envp[])
+{
+  struct spawn_syscall_parms_s parms;
+
+  parms.pid          = pid;
+  parms.name         = name;
+  parms.entry        = entry;
+  parms.file_actions = file_actions;
+  parms.attr         = attr;
+  parms.argv         = (FAR char * const *)argv;
+  parms.envp         = (FAR char * const *)envp;
+
+  return nx_task_spawn(&parms);
+}
+
+#endif /* CONFIG_BUILD_PROTECTED && CONFIG_LIB_SYSCALL */

sched/task/task_spawn.c

@@ -46,6 +46,7 @@
 
 #include <nuttx/sched.h>
 #include <nuttx/kthread.h>
+#include <nuttx/spawn.h>
 
 #include "sched/sched.h"
 #include "group/group.h"
@@ -435,4 +436,36 @@ errout_with_lock:
   return ret;
 }
 
+/****************************************************************************
+ * Name: nx_task_spawn
+ *
+ * Description:
+ *   This function de-marshals parameters and invokes task_spawn().
+ *
+ *   task_spawn() and posix_spawn() are NuttX OS interfaces.  In PROTECTED
+ *   and KERNEL build modes, then can be reached from applications only via
+ *   a system call.  Currently, the number of parameters in a system call
+ *   is limited to six; these spawn function have seven parameters.  Rather
+ *   than extend the maximum number of parameters across all architectures,
+ *   I opted instead to marshal the seven parameters into a structure.
+ *
+ * Input Parameters:
+ *   parms - The marshaled task_spawn() parameters.
+ *
+ * Returned Value:
+ *   On success, these functions return 0; on failure they return an error
+ *   number from <errno.h> (see the comments associated with task_spawn()).
+ *
+ ****************************************************************************/
+
+#if defined(CONFIG_BUILD_PROTECTED) && defined(CONFIG_LIB_SYSCALL)
+int nx_task_spawn(FAR const struct spawn_syscall_parms_s *parms)
+{
+  DEBUGASSERT(parms != NULL);
+  return task_spawn(parms->pid, parms->name, parms->entry,
+                    parms->file_actions, parms->attr,
+                    parms->argv, parms->envp);
+}
+#endif
+
 #endif /* CONFIG_BUILD_KERNEL */

syscall/syscall.csv

@@ -63,6 +63,7 @@
 "mq_timedreceive","mqueue.h","!defined(CONFIG_DISABLE_MQUEUE)","ssize_t","mqd_t","char*","size_t","FAR unsigned int*","const struct timespec*"
 "mq_timedsend","mqueue.h","!defined(CONFIG_DISABLE_MQUEUE)","int","mqd_t","const char*","size_t","unsigned int","const struct timespec*"
 "mq_unlink","mqueue.h","!defined(CONFIG_DISABLE_MQUEUE)","int","const char*"
+"nx_task_spawn","nuttx/spawn.h","!defined(CONFIG_BUILD_KERNEL)","int","FAR const struct spawn_syscall_parms_s *"
 "nx_vsyslog","nuttx/syslog/syslog.h","","int","int","FAR const IPTR char*","FAR va_list*"
 "on_exit","stdlib.h","defined(CONFIG_SCHED_ONEXIT)","int","CODE void (*)(int, FAR void *)","FAR void *"
 "open","fcntl.h","","int","const char*","int","..."
@@ -162,7 +163,6 @@
 "stat","sys/stat.h","","int","const char*","FAR struct stat*"
 "statfs","sys/statfs.h","","int","FAR const char*","FAR struct statfs*"
 "task_create","sched.h","!defined(CONFIG_BUILD_KERNEL)", "int","FAR const char*","int","int","main_t","FAR char * const []|FAR char * const *"
-#"task_create","sched.h","","int","const char*","int","main_t","FAR char * const []|FAR char * const *"
 "task_delete","sched.h","","int","pid_t"
 "task_restart","sched.h","","int","pid_t"
 "task_setcancelstate","sched.h","","int","int","FAR int*"
@@ -180,7 +180,6 @@
 "unlink","unistd.h","!defined(CONFIG_DISABLE_MOUNTPOINT)","int","FAR const char*"
 "unsetenv","stdlib.h","!defined(CONFIG_DISABLE_ENVIRON)","int","const char*"
 "up_assert","assert.h","","void","FAR const uint8_t*","int"
-#"up_assert","assert.h","","void"
 "vfork","unistd.h","defined(CONFIG_SCHED_WAITPID) && defined(CONFIG_ARCH_HAVE_VFORK)","pid_t"
 "wait","sys/wait.h","defined(CONFIG_SCHED_WAITPID) && defined(CONFIG_SCHED_HAVE_PARENT)","pid_t","int*"
 "waitid","sys/wait.h","defined(CONFIG_SCHED_WAITPID) && defined(CONFIG_SCHED_HAVE_PARENT)","int","idtype_t","id_t"," FAR siginfo_t *","int"

syscall/syscall_lookup.h

@@ -90,6 +90,7 @@ SYSCALL_LOOKUP(sem_unlink,                 1, STUB_sem_unlink)
 
 #ifndef CONFIG_BUILD_KERNEL
 SYSCALL_LOOKUP(task_create,                5, STUB_task_create)
+SYSCALL_LOOKUP(nx_task_spawn,              1, STUB_nx_task_spawn)
 #else
 SYSCALL_LOOKUP(pgalloc,                    2, STUB_pgalloc)
 #endif

syscall/syscall_stublookup.c

@@ -90,7 +90,8 @@ uintptr_t STUB_getgid(int nbr);
 uintptr_t STUB_sem_close(int nbr, uintptr_t parm1);
 uintptr_t STUB_sem_destroy(int nbr, uintptr_t parm1);
 uintptr_t STUB_sem_open(int nbr, uintptr_t parm1, uintptr_t parm2,
-            uintptr_t parm3, uintptr_t parm4, uintptr_t parm5, uintptr_t parm6);
+            uintptr_t parm3, uintptr_t parm4, uintptr_t parm5,
+            uintptr_t parm6);
 uintptr_t STUB_sem_post(int nbr, uintptr_t parm1);
 uintptr_t STUB_sem_setprotocol(int nbr, uintptr_t parm1, uintptr_t parm2);
 uintptr_t STUB_sem_timedwait(int nbr, uintptr_t parm1, uintptr_t parm2);
@@ -101,6 +102,7 @@ uintptr_t STUB_sem_wait(int nbr, uintptr_t parm1);
 uintptr_t STUB_pgalloc(int nbr, uintptr_t parm1, uintptr_t parm2);
 uintptr_t STUB_task_create(int nbr, uintptr_t parm1, uintptr_t parm2,
             uintptr_t parm3, uintptr_t parm4, uintptr_t parm5);
+uintptr_t STUB_nx_task_spawn(int nbr, uintptr_t parm1);
 uintptr_t STUB_task_delete(int nbr, uintptr_t parm1);
 uintptr_t STUB_task_restart(int nbr, uintptr_t parm1);
 uintptr_t STUB_task_setcancelstate(int nbr, uintptr_t parm1,