5049299: (process) Use posix_spawn, not fork, on S10 to avoid swap exhaustion

Reviewed-by: alanb, dholmes, martin, erikj, coffeys

make/java/java/Exportedfiles.gmk

@@ -50,6 +50,7 @@ FILES_export = \
     java/lang/SecurityManager.java \
     java/lang/Shutdown.java \
     java/lang/Package.java \
+    java/lang/UNIXProcess.java \
     java/lang/ref/Finalizer.java \
     java/lang/reflect/AccessibleObject.java \
     java/lang/reflect/Field.java \

make/java/java/Makefile

@@ -87,6 +87,7 @@ FILES_java += 	java/lang/UNIXProcess.java \
                 java/util/prefs/FileSystemPreferencesFactory.java \
 
 FILES_c    +=   UNIXProcess_md.c \
+		childproc.c \
 		UnixFileSystem_md.c \
 		canonicalize_md.c \
 		TimeZone.c \
@@ -468,3 +469,36 @@ $(GENSRCDIR)/sun/util/CoreResourceBundleControl.java: \
 clean::
 	$(RM) $(GENSRCDIR)/sun/util/CoreResourceBundleControl.java
 
+
+HELPER_EXE = $(LIBDIR)/$(LIBARCH)/jspawnhelper
+BUILDHELPER =
+ifeq ($(PLATFORM), solaris)
+    BUILDHELPER = 1
+endif
+ifeq ($(PLATFORM), macosx)
+    HELPER_EXE = $(LIBDIR)/jspawnhelper
+    BUILDHELPER = 1
+endif
+
+ARCHFLAG = 
+ifeq ($(ARCH_DATA_MODEL), 64)
+ARCHFLAG = -m64
+endif
+
+ifdef BUILDHELPER
+
+HELPER_EXE_FILES_c = jspawnhelper.c
+HELPER_EXE_FILES_o = $(OBJDIR)/jspawnhelper.o \
+		$(OBJDIR)/childproc.o
+
+$(HELPER_EXE): $(HELPER_EXE_FILES_o)
+	$(CC) $(ARCHFLAG) $(HELPER_EXE_FILES_o) \
+		-o $(TEMPDIR)/jspawnhelper
+	$(CP) $(TEMPDIR)/jspawnhelper $(HELPER_EXE)
+
+build: $(HELPER_EXE)
+
+clean clobber:: 
+ 	$(RM) $(HELPER_EXE_FILES_o) $(HELPER_EXE)
+
+endif #BUILDHELPER
\ No newline at end of file

makefiles/CompileLaunchers.gmk

@@ -40,7 +40,7 @@ include Tools.gmk
 
 BUILD_LAUNCHERS=
 
-# When building a legacy overlay image (on solaris 64 bit), the launchers 
+# When building a legacy overlay image (on solaris 64 bit), the launchers
 # need to be built with a different rpath and a different output dir.
 ifeq ($(OVERLAY_IMAGES),true)
     ORIGIN_ROOT:=/../..
@@ -63,7 +63,7 @@ endif
 #
 ifneq ($(findstring $(OPENJDK_TARGET_OS), linux solaris),)
     ORIGIN_ARG+=$(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/lib$(OPENJDK_TARGET_CPU_LIBDIR)) \
-                $(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/jre/lib$(OPENJDK_TARGET_CPU_LIBDIR)) 
+                $(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/jre/lib$(OPENJDK_TARGET_CPU_LIBDIR))
 endif
 
 define SetupLauncher
@@ -189,7 +189,7 @@ define SetupLauncher
     ifeq ($(OPENJDK_TARGET_OS),windows)
         $$(BUILD_LAUNCHER_$1) : $(JDK_OUTPUTDIR)/objs/libjava/java.lib \
 				$$($1_WINDOWS_JLI_LIB)
-    endif    	 
+    endif
 endef
 
 ##########################################################################################
@@ -441,7 +441,7 @@ ifeq ($(OPENJDK_TARGET_OS),solaris)
 endif
 # On windows, unpack200 is linked completely differently to all other
 # executables, using the compiler with the compiler arguments.
-# It's also linked incrementally, producing a .ilk file that needs to 
+# It's also linked incrementally, producing a .ilk file that needs to
 # be kept away.
 ifeq ($(OPENJDK_TARGET_OS),windows)
     BUILD_UNPACKEXE_LDEXE:=$(CC)
@@ -490,7 +490,7 @@ $(BUILD_UNPACKEXE) : $(UNPACKEXE_ZIPOBJS)
 endif
 
 # Build into object dir and copy executable afterwards to avoid .ilk file in
-# image. The real fix would be clean up linking of unpack200 using 
+# image. The real fix would be clean up linking of unpack200 using
 # -link -incremental:no
 # like all other launchers.
 $(JDK_OUTPUTDIR)/bin$(OUTPUT_SUBDIR)/unpack200$(EXE_SUFFIX): $(BUILD_UNPACKEXE)
@@ -501,7 +501,7 @@ BUILD_LAUNCHERS += $(JDK_OUTPUTDIR)/bin$(OUTPUT_SUBDIR)/unpack200$(EXE_SUFFIX)
 ##########################################################################################
 
 
-BUILD_JEXEC := 
+BUILD_JEXEC :=
 BUILD_JEXEC_SRC :=
 BUILD_JEXEC_INC :=
 BUILD_JEXEC_DST_DIR := $(JDK_OUTPUTDIR)/lib$(OPENJDK_TARGET_CPU_LIBDIR)
@@ -581,11 +581,11 @@ ifeq ($(OPENJDK_TARGET_OS), solaris)
   endif
 endif
 
-# TODO: 
+# TODO:
 # On windows java-rmi.cgi shouldn't be bundled since Java 1.2, but has been built all
 # this time anyway. Since jdk6, it has been built from the wrong source and resulted
-# in a (almost) copy of the standard java launcher named "java-rmi.exe" ending up in 
-# the final images bin dir. This weird behavior is mimicked here in the converted 
+# in a (almost) copy of the standard java launcher named "java-rmi.exe" ending up in
+# the final images bin dir. This weird behavior is mimicked here in the converted
 # makefiles for now. Should probably just be deleted.
 # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6512052
 ifeq ($(OPENJDK_TARGET_OS),windows)
@@ -597,6 +597,44 @@ else
 	$(CHMOD) a+x $@
 endif
 
+##########################################################################################
+
+BUILD_JSPAWNHELPER :=
+BUILD_JSPAWNHELPER_SRC := $(JDK_TOPDIR)/src/solaris/native/java/lang
+BUILD_JSPAWNHELPER_DST_DIR := $(JDK_OUTPUTDIR)/lib$(OPENJDK_TARGET_CPU_LIBDIR)
+LINK_JSPAWNHELPER_OBJECTS := $(JDK_OUTPUTDIR)/objs/libjava/childproc.o
+LINK_JSPAWNHELPER_FLAGS :=
+
+ifneq ($(findstring $(OPENJDK_TARGET_OS), macosx solaris),)
+    BUILD_JSPAWNHELPER := 1
+endif
+
+ifeq ($(OPENJDK_TARGET_OS), macosx)
+    BUILD_JSPAWNHELPER_DST_DIR := $(JDK_OUTPUTDIR)/lib
+endif
+
+ifeq ($(OPENJDK_TARGET_CPU_BITS), 64)
+    LINK_JSPAWNHELPER_FLAGS += -m64
+endif
+
+ifeq ($(BUILD_JSPAWNHELPER), 1)
+    $(eval $(call SetupNativeCompilation,BUILD_JSPAWNHELPER,\
+        SRC:=$(BUILD_JSPAWNHELPER_SRC),\
+        INCLUDE_FILES:=jspawnhelper.c,\
+        LANG:=C,\
+        OPTIMIZATION := LOW, \
+        CFLAGS:=$(CFLAGS_JDKEXE), \
+        LDFLAGS:=$(LDFLAGS_JDKEXE) $(LINK_JSPAWNHELPER_FLAGS), \
+        LDFLAGS_SUFFIX:= $(LINK_JSPAWNHELPER_OBJECTS), \
+        OBJECT_DIR:=$(JDK_OUTPUTDIR)/objs/jspawnhelper,\
+        OUTPUT_DIR:=$(BUILD_JSPAWNHELPER_DST_DIR),\
+        PROGRAM:=jspawnhelper))
+
+    $(BUILD_JSPAWNHELPER): $(LINK_JSPAWNHELPER_OBJECTS)
+
+    BUILD_LAUNCHERS += $(BUILD_JSPAWNHELPER)
+endif
+
 ##########################################################################################
 # jabswitch
 

makefiles/CompileNativeLibraries.gmk

@@ -205,7 +205,7 @@ ifneq (,$(JDK_UPDATE_VERSION))
 	LIBJAVA_CFLAGS += -DJDK_UPDATE_VERSION='"$(JDK_UPDATE_VERSION)"'
 endif
 
-LIBJAVA_EXCLUDE_FILES:=check_code.c check_format.c
+LIBJAVA_EXCLUDE_FILES:=check_code.c check_format.c jspawnhelper.c
 
 ifneq ($(OPENJDK_TARGET_OS),macosx)
 	LIBJAVA_EXCLUDE_FILES += java_props_macosx.c

src/solaris/classes/java/lang/UNIXProcess.java.bsd

@@ -64,11 +64,59 @@ final class UNIXProcess extends Process {
     private /* final */ InputStream  stdout;
     private /* final */ InputStream  stderr;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        POSIX_SPAWN(2);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* On BSD, the default is to spawn */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+
+                helperpath = toCString(javahome + "/lib/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "posix_spawn");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - posix_spawn(2)
      *
      * @param fds an array of three file descriptors.
      *        Indexes 0, 1, and 2 correspond to standard input,
@@ -81,7 +129,8 @@ final class UNIXProcess extends Process {
      *        output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -133,7 +182,9 @@ final class UNIXProcess extends Process {
                 final boolean redirectErrorStream)
             throws IOException {
 
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,
@@ -212,13 +263,13 @@ final class UNIXProcess extends Process {
         }
         return exitcode;
     }
-    
+
     @Override
-    public synchronized boolean waitFor(long timeout, TimeUnit unit) 
-        throws InterruptedException 
+    public synchronized boolean waitFor(long timeout, TimeUnit unit)
+        throws InterruptedException
     {
         if (hasExited) return true;
-        if (timeout <= 0) return false;	
+        if (timeout <= 0) return false;
 
         long timeoutAsNanos = unit.toNanos(timeout);
         long startTime = System.nanoTime();

src/solaris/classes/java/lang/UNIXProcess.java.linux

@@ -64,11 +64,61 @@ final class UNIXProcess extends Process {
     private /* final */ InputStream  stdout;
     private /* final */ InputStream  stderr;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        VFORK(3);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* default is VFORK on Linux */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+                String osArch = System.getProperty("os.arch");
+
+                helperpath = toCString(javahome + "/lib/" + osArch + "/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "vfork");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - clone(2) and exec(2)
+     * - vfork(2) and exec(2)
      *
      * @param fds an array of three file descriptors.
      *        Indexes 0, 1, and 2 correspond to standard input,
@@ -81,7 +131,8 @@ final class UNIXProcess extends Process {
      *        output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -133,7 +184,9 @@ final class UNIXProcess extends Process {
                 final boolean redirectErrorStream)
             throws IOException {
 
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,

src/solaris/classes/java/lang/UNIXProcess.java.solaris

@@ -27,6 +27,8 @@ package java.lang;
 
 import java.io.*;
 import java.util.concurrent.TimeUnit;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 
 /* java.lang.Process subclass in the UNIX environment.
  *
@@ -46,11 +48,65 @@ final class UNIXProcess extends Process {
     private DeferredCloseInputStream stdout_inner_stream;
     private InputStream stderr_stream;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        POSIX_SPAWN(2);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* On Solaris, the default is to spawn */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+                String osArch = System.getProperty("os.arch");
+                if (osArch.equals("x86")) {
+                    osArch = "i386";
+                } else if (osArch.equals("x86_64")) {
+                    osArch = "amd64";
+                }
+
+                helperpath = toCString(javahome + "/lib/" + osArch + "/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "posix_spawn");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - posix_spawn(2)
      *
      * @param std_fds array of file descriptors.  Indexes 0, 1, and
      *        2 correspond to standard input, standard output and
@@ -62,7 +118,8 @@ final class UNIXProcess extends Process {
      *        if and only if it is <em>not</em> -1 on output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -77,7 +134,9 @@ final class UNIXProcess extends Process {
                 final int[] std_fds,
                 final boolean redirectErrorStream)
     throws IOException {
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,

src/solaris/native/java/lang/UNIXProcess_md.c

@@ -43,32 +43,15 @@
 #include <sys/wait.h>
 #include <signal.h>
 #include <string.h>
-#include <errno.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <limits.h>
-
-#ifdef __APPLE__
-#include <crt_externs.h>
-#define environ (*_NSGetEnviron())
-#else
-/* This is one of the rare times it's more portable to declare an
- * external symbol explicitly, rather than via a system header.
- * The declaration is standardized as part of UNIX98, but there is
- * no standard (not even de-facto) header file where the
- * declaration is to be found.  See:
- * http://www.opengroup.org/onlinepubs/009695399/functions/environ.html
- * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_02.html
- *
- * "All identifiers in this volume of IEEE Std 1003.1-2001, except
- * environ, are defined in at least one of the headers" (!)
- */
-extern char **environ;
+
+#if defined(__solaris__) || defined(_ALLBSD_SOURCE)
+#include <spawn.h>
 #endif
 
+#include "childproc.h"
+
 /*
- * There are 3 possible strategies we might use to "fork":
+ * There are 4 possible strategies we might use to "fork":
  *
  * - fork(2).  Very portable and reliable but subject to
  *   failure due to overcommit (see the documentation on
@@ -103,68 +86,21 @@ extern char **environ;
  *     http://sources.redhat.com/bugzilla/show_bug.cgi?id=10311
  *   but the glibc maintainers closed it as WONTFIX.
  *
+ * - posix_spawn(). While posix_spawn() is a fairly elaborate and
+ *   complicated system call, it can't quite do everything that the old
+ *   fork()/exec() combination can do, so the only feasible way to do
+ *   this, is to use posix_spawn to launch a new helper executable
+ *   "jprochelper", which in turn execs the target (after cleaning
+ *   up file-descriptors etc.) The end result is the same as before,
+ *   a child process linked to the parent in the same way, but it
+ *   avoids the problem of duplicating the parent (VM) process
+ *   address space temporarily, before launching the target command.
+ *
  * Based on the above analysis, we are currently using vfork() on
- * Linux and fork() on other Unix systems, but the code to use clone()
- * remains.
+ * Linux and spawn() on other Unix systems, but the code to use clone()
+ * and fork() remains.
  */
 
-#define START_CHILD_USE_CLONE 0  /* clone() currently disabled; see above. */
-
-#ifndef START_CHILD_USE_CLONE
-  #ifdef __linux__
-    #define START_CHILD_USE_CLONE 1
-  #else
-    #define START_CHILD_USE_CLONE 0
-  #endif
-#endif
-
-/* By default, use vfork() on Linux. */
-#ifndef START_CHILD_USE_VFORK
-  #ifdef __linux__
-    #define START_CHILD_USE_VFORK 1
-  #else
-    #define START_CHILD_USE_VFORK 0
-  #endif
-#endif
-
-#if START_CHILD_USE_CLONE
-#include <sched.h>
-#define START_CHILD_SYSTEM_CALL "clone"
-#elif START_CHILD_USE_VFORK
-#define START_CHILD_SYSTEM_CALL "vfork"
-#else
-#define START_CHILD_SYSTEM_CALL "fork"
-#endif
-
-#ifndef STDIN_FILENO
-#define STDIN_FILENO 0
-#endif
-
-#ifndef STDOUT_FILENO
-#define STDOUT_FILENO 1
-#endif
-
-#ifndef STDERR_FILENO
-#define STDERR_FILENO 2
-#endif
-
-#ifndef SA_NOCLDSTOP
-#define SA_NOCLDSTOP 0
-#endif
-
-#ifndef SA_RESTART
-#define SA_RESTART 0
-#endif
-
-#define FAIL_FILENO (STDERR_FILENO + 1)
-
-/* TODO: Refactor. */
-#define RESTARTABLE(_cmd, _result) do { \
-  do { \
-    _result = _cmd; \
-  } while((_result == -1) && (errno == EINTR)); \
-} while(0)
-
 
 static void
 setSIGCHLDHandler(JNIEnv *env)
@@ -266,17 +202,10 @@ effectivePathv(JNIEnv *env)
     return pathv;
 }
 
-/**
- * The cached and split version of the JDK's effective PATH.
- * (We don't support putenv("PATH=...") in native code)
- */
-static const char * const *parentPathv;
-
 JNIEXPORT void JNICALL
 Java_java_lang_UNIXProcess_init(JNIEnv *env, jclass clazz)
 {
     parentPathv = effectivePathv(env);
-
     setSIGCHLDHandler(env);
 }
 
@@ -343,96 +272,6 @@ Java_java_lang_UNIXProcess_waitForProcessExit(JNIEnv* env,
     }
 }
 
-static ssize_t
-restartableWrite(int fd, const void *buf, size_t count)
-{
-    ssize_t result;
-    RESTARTABLE(write(fd, buf, count), result);
-    return result;
-}
-
-static int
-restartableDup2(int fd_from, int fd_to)
-{
-    int err;
-    RESTARTABLE(dup2(fd_from, fd_to), err);
-    return err;
-}
-
-static int
-restartableClose(int fd)
-{
-    int err;
-    RESTARTABLE(close(fd), err);
-    return err;
-}
-
-static int
-closeSafely(int fd)
-{
-    return (fd == -1) ? 0 : restartableClose(fd);
-}
-
-static int
-isAsciiDigit(char c)
-{
-  return c >= '0' && c <= '9';
-}
-
-#ifdef _ALLBSD_SOURCE
-#define FD_DIR "/dev/fd"
-#define dirent64 dirent
-#define readdir64 readdir
-#else
-#define FD_DIR "/proc/self/fd"
-#endif
-
-static int
-closeDescriptors(void)
-{
-    DIR *dp;
-    struct dirent64 *dirp;
-    int from_fd = FAIL_FILENO + 1;
-
-    /* We're trying to close all file descriptors, but opendir() might
-     * itself be implemented using a file descriptor, and we certainly
-     * don't want to close that while it's in use.  We assume that if
-     * opendir() is implemented using a file descriptor, then it uses
-     * the lowest numbered file descriptor, just like open().  So we
-     * close a couple explicitly.  */
-
-    restartableClose(from_fd);          /* for possible use by opendir() */
-    restartableClose(from_fd + 1);      /* another one for good luck */
-
-    if ((dp = opendir(FD_DIR)) == NULL)
-        return 0;
-
-    /* We use readdir64 instead of readdir to work around Solaris bug
-     * 6395699: /proc/self/fd fails to report file descriptors >= 1024 on Solaris 9
-     */
-    while ((dirp = readdir64(dp)) != NULL) {
-        int fd;
-        if (isAsciiDigit(dirp->d_name[0]) &&
-            (fd = strtol(dirp->d_name, NULL, 10)) >= from_fd + 2)
-            restartableClose(fd);
-    }
-
-    closedir(dp);
-
-    return 1;
-}
-
-static int
-moveDescriptor(int fd_from, int fd_to)
-{
-    if (fd_from != fd_to) {
-        if ((restartableDup2(fd_from, fd_to) == -1) ||
-            (restartableClose(fd_from) == -1))
-            return -1;
-    }
-    return 0;
-}
-
 static const char *
 getBytes(JNIEnv *env, jbyteArray arr)
 {
@@ -447,19 +286,6 @@ releaseBytes(JNIEnv *env, jbyteArray arr, const char* parr)
         (*env)->ReleaseByteArrayElements(env, arr, (jbyte*) parr, JNI_ABORT);
 }
 
-static void
-initVectorFromBlock(const char**vector, const char* block, int count)
-{
-    int i;
-    const char *p;
-    for (i = 0, p = block; i < count; i++) {
-        /* Invariant: p always points to the start of a C string. */
-        vector[i] = p;
-        while (*(p++));
-    }
-    vector[count] = NULL;
-}
-
 static void
 throwIOException(JNIEnv *env, int errnum, const char *defaultDetail)
 {
@@ -503,280 +329,74 @@ debugPrint(char *format, ...)
 }
 #endif /* DEBUG_PROCESS */
 
-/**
- * Exec FILE as a traditional Bourne shell script (i.e. one without #!).
- * If we could do it over again, we would probably not support such an ancient
- * misfeature, but compatibility wins over sanity.  The original support for
- * this was imported accidentally from execvp().
- */
 static void
-execve_as_traditional_shell_script(const char *file,
-                                   const char *argv[],
-                                   const char *const envp[])
-{
-    /* Use the extra word of space provided for us in argv by caller. */
-    const char *argv0 = argv[0];
-    const char *const *end = argv;
-    while (*end != NULL)
-        ++end;
-    memmove(argv+2, argv+1, (end-argv) * sizeof (*end));
-    argv[0] = "/bin/sh";
-    argv[1] = file;
-    execve(argv[0], (char **) argv, (char **) envp);
-    /* Can't even exec /bin/sh?  Big trouble, but let's soldier on... */
-    memmove(argv+1, argv+2, (end-argv) * sizeof (*end));
-    argv[0] = argv0;
-}
-
-/**
- * Like execve(2), except that in case of ENOEXEC, FILE is assumed to
- * be a shell script and the system default shell is invoked to run it.
- */
-static void
-execve_with_shell_fallback(const char *file,
-                           const char *argv[],
-                           const char *const envp[])
+copyPipe(int from[2], int to[2])
 {
-#if START_CHILD_USE_CLONE || START_CHILD_USE_VFORK
-    /* shared address space; be very careful. */
-    execve(file, (char **) argv, (char **) envp);
-    if (errno == ENOEXEC)
-        execve_as_traditional_shell_script(file, argv, envp);
-#else
-    /* unshared address space; we can mutate environ. */
-    environ = (char **) envp;
-    execvp(file, (char **) argv);
-#endif
+    to[0] = from[0];
+    to[1] = from[1];
 }
 
-/**
- * 'execvpe' should have been included in the Unix standards,
- * and is a GNU extension in glibc 2.10.
+/* arg is an array of pointers to 0 terminated strings. array is terminated
+ * by a null element.
  *
- * JDK_execvpe is identical to execvp, except that the child environment is
- * specified via the 3rd argument instead of being inherited from environ.
+ * *nelems and *nbytes receive the number of elements of array (incl 0)
+ * and total number of bytes (incl. 0)
+ * Note. An empty array will have one null element
+ * But if arg is null, then *nelems set to 0, and *nbytes to 0
  */
-static void
-JDK_execvpe(const char *file,
-            const char *argv[],
-            const char *const envp[])
+static void arraysize(const char * const *arg, int *nelems, int *nbytes)
 {
-    if (envp == NULL || (char **) envp == environ) {
-        execvp(file, (char **) argv);
-        return;
-    }
-
-    if (*file == '\0') {
-        errno = ENOENT;
+    int i, bytes, count;
+    const char * const *a = arg;
+    char *p;
+    int *q;
+    if (arg == 0) {
+        *nelems = 0;
+        *nbytes = 0;
         return;
     }
-
-    if (strchr(file, '/') != NULL) {
-        execve_with_shell_fallback(file, argv, envp);
-    } else {
-        /* We must search PATH (parent's, not child's) */
-        char expanded_file[PATH_MAX];
-        int filelen = strlen(file);
-        int sticky_errno = 0;
-        const char * const * dirs;
-        for (dirs = parentPathv; *dirs; dirs++) {
-            const char * dir = *dirs;
-            int dirlen = strlen(dir);
-            if (filelen + dirlen + 2 >= PATH_MAX) {
-                errno = ENAMETOOLONG;
-                continue;
-            }
-            memcpy(expanded_file, dir, dirlen);
-            if (expanded_file[dirlen - 1] != '/')
-                expanded_file[dirlen++] = '/';
-            memcpy(expanded_file + dirlen, file, filelen);
-            expanded_file[dirlen + filelen] = '\0';
-            execve_with_shell_fallback(expanded_file, argv, envp);
-            /* There are 3 responses to various classes of errno:
-             * return immediately, continue (especially for ENOENT),
-             * or continue with "sticky" errno.
-             *
-             * From exec(3):
-             *
-             * If permission is denied for a file (the attempted
-             * execve returned EACCES), these functions will continue
-             * searching the rest of the search path.  If no other
-             * file is found, however, they will return with the
-             * global variable errno set to EACCES.
-             */
-            switch (errno) {
-            case EACCES:
-                sticky_errno = errno;
-                /* FALLTHRU */
-            case ENOENT:
-            case ENOTDIR:
-#ifdef ELOOP
-            case ELOOP:
-#endif
-#ifdef ESTALE
-            case ESTALE:
-#endif
-#ifdef ENODEV
-            case ENODEV:
-#endif
-#ifdef ETIMEDOUT
-            case ETIMEDOUT:
-#endif
-                break; /* Try other directories in PATH */
-            default:
-                return;
-            }
-        }
-        if (sticky_errno != 0)
-            errno = sticky_errno;
+    /* count the array elements and number of bytes */
+    for (count=0, bytes=0; *a != 0; count++, a++) {
+        bytes += strlen(*a)+1;
     }
+    *nbytes = bytes;
+    *nelems = count+1;
 }
 
-/*
- * Reads nbyte bytes from file descriptor fd into buf,
- * The read operation is retried in case of EINTR or partial reads.
- *
- * Returns number of bytes read (normally nbyte, but may be less in
- * case of EOF).  In case of read errors, returns -1 and sets errno.
- */
-static ssize_t
-readFully(int fd, void *buf, size_t nbyte)
-{
-    ssize_t remaining = nbyte;
-    for (;;) {
-        ssize_t n = read(fd, buf, remaining);
-        if (n == 0) {
-            return nbyte - remaining;
-        } else if (n > 0) {
-            remaining -= n;
-            if (remaining <= 0)
-                return nbyte;
-            /* We were interrupted in the middle of reading the bytes.
-             * Unlikely, but possible. */
-            buf = (void *) (((char *)buf) + n);
-        } else if (errno == EINTR) {
-            /* Strange signals like SIGJVM1 are possible at any time.
-             * See http://www.dreamsongs.com/WorseIsBetter.html */
-        } else {
-            return -1;
-        }
-    }
-}
-
-typedef struct _ChildStuff
-{
-    int in[2];
-    int out[2];
-    int err[2];
-    int fail[2];
-    int fds[3];
-    const char **argv;
-    const char **envv;
-    const char *pdir;
-    jboolean redirectErrorStream;
-#if START_CHILD_USE_CLONE
-    void *clone_stack;
-#endif
-} ChildStuff;
-
-static void
-copyPipe(int from[2], int to[2])
-{
-    to[0] = from[0];
-    to[1] = from[1];
-}
-
-/**
- * Child process after a successful fork() or clone().
- * This function must not return, and must be prepared for either all
- * of its address space to be shared with its parent, or to be a copy.
- * It must not modify global variables such as "environ".
+/* copy the strings from arg[] into buf, starting at given offset
+ * return new offset to next free byte
  */
-static int
-childProcess(void *arg)
-{
-    const ChildStuff* p = (const ChildStuff*) arg;
-
-    /* Close the parent sides of the pipes.
-       Closing pipe fds here is redundant, since closeDescriptors()
-       would do it anyways, but a little paranoia is a good thing. */
-    if ((closeSafely(p->in[1])   == -1) ||
-        (closeSafely(p->out[0])  == -1) ||
-        (closeSafely(p->err[0])  == -1) ||
-        (closeSafely(p->fail[0]) == -1))
-        goto WhyCantJohnnyExec;
-
-    /* Give the child sides of the pipes the right fileno's. */
-    /* Note: it is possible for in[0] == 0 */
-    if ((moveDescriptor(p->in[0] != -1 ?  p->in[0] : p->fds[0],
-                        STDIN_FILENO) == -1) ||
-        (moveDescriptor(p->out[1]!= -1 ? p->out[1] : p->fds[1],
-                        STDOUT_FILENO) == -1))
-        goto WhyCantJohnnyExec;
-
-    if (p->redirectErrorStream) {
-        if ((closeSafely(p->err[1]) == -1) ||
-            (restartableDup2(STDOUT_FILENO, STDERR_FILENO) == -1))
-            goto WhyCantJohnnyExec;
-    } else {
-        if (moveDescriptor(p->err[1] != -1 ? p->err[1] : p->fds[2],
-                           STDERR_FILENO) == -1)
-            goto WhyCantJohnnyExec;
-    }
-
-    if (moveDescriptor(p->fail[1], FAIL_FILENO) == -1)
-        goto WhyCantJohnnyExec;
+static int copystrings(char *buf, int offset, const char * const *arg) {
+    char *p;
+    const char * const *a;
+    int count=0;
 
-    /* close everything */
-    if (closeDescriptors() == 0) { /* failed,  close the old way */
-        int max_fd = (int)sysconf(_SC_OPEN_MAX);
-        int fd;
-        for (fd = FAIL_FILENO + 1; fd < max_fd; fd++)
-            if (restartableClose(fd) == -1 && errno != EBADF)
-                goto WhyCantJohnnyExec;
+    if (arg == 0) {
+        return offset;
     }
-
-    /* change to the new working directory */
-    if (p->pdir != NULL && chdir(p->pdir) < 0)
-        goto WhyCantJohnnyExec;
-
-    if (fcntl(FAIL_FILENO, F_SETFD, FD_CLOEXEC) == -1)
-        goto WhyCantJohnnyExec;
-
-    JDK_execvpe(p->argv[0], p->argv, p->envv);
-
- WhyCantJohnnyExec:
-    /* We used to go to an awful lot of trouble to predict whether the
-     * child would fail, but there is no reliable way to predict the
-     * success of an operation without *trying* it, and there's no way
-     * to try a chdir or exec in the parent.  Instead, all we need is a
-     * way to communicate any failure back to the parent.  Easy; we just
-     * send the errno back to the parent over a pipe in case of failure.
-     * The tricky thing is, how do we communicate the *success* of exec?
-     * We use FD_CLOEXEC together with the fact that a read() on a pipe
-     * yields EOF when the write ends (we have two of them!) are closed.
-     */
-    {
-        int errnum = errno;
-        restartableWrite(FAIL_FILENO, &errnum, sizeof(errnum));
+    for (p=buf+offset, a=arg; *a != 0; a++) {
+        int len = strlen(*a) +1;
+        memcpy(p, *a, len);
+        p += len;
+        count += len;
     }
-    restartableClose(FAIL_FILENO);
-    _exit(-1);
-    return 0;  /* Suppress warning "no return value from function" */
+    return offset+count;
 }
 
 /**
- * Start a child process running function childProcess.
- * This function only returns in the parent.
  * We are unusually paranoid; use of clone/vfork is
  * especially likely to tickle gcc/glibc bugs.
  */
 #ifdef __attribute_noinline__  /* See: sys/cdefs.h */
 __attribute_noinline__
 #endif
+
+#define START_CHILD_USE_CLONE 0  /* clone() currently disabled; see above. */
+
+#ifdef START_CHILD_USE_CLONE
 static pid_t
-startChild(ChildStuff *c) {
-#if START_CHILD_USE_CLONE
+cloneChild(ChildStuff *c) {
+#ifdef __linux__
 #define START_CHILD_CLONE_STACK_SIZE (64 * 1024)
     /*
      * See clone(2).
@@ -790,33 +410,161 @@ startChild(ChildStuff *c) {
                  c->clone_stack + START_CHILD_CLONE_STACK_SIZE,
                  CLONE_VFORK | CLONE_VM | SIGCHLD, c);
 #else
-  #if START_CHILD_USE_VFORK
+/* not available on Solaris / Mac */
+    assert(0);
+    return -1;
+#endif
+}
+#endif
+
+static pid_t
+vforkChild(ChildStuff *c) {
+    volatile pid_t resultPid;
+
     /*
      * We separate the call to vfork into a separate function to make
      * very sure to keep stack of child from corrupting stack of parent,
      * as suggested by the scary gcc warning:
      *  warning: variable 'foo' might be clobbered by 'longjmp' or 'vfork'
      */
-    volatile pid_t resultPid = vfork();
-  #else
+    resultPid = vfork();
+
+    if (resultPid == 0) {
+        childProcess(c);
+    }
+    assert(resultPid != 0);  /* childProcess never returns */
+    return resultPid;
+}
+
+static pid_t
+forkChild(ChildStuff *c) {
+    pid_t resultPid;
+
     /*
      * From Solaris fork(2): In Solaris 10, a call to fork() is
      * identical to a call to fork1(); only the calling thread is
      * replicated in the child process. This is the POSIX-specified
      * behavior for fork().
      */
-    pid_t resultPid = fork();
-  #endif
-    if (resultPid == 0)
+    resultPid = fork();
+
+    if (resultPid == 0) {
         childProcess(c);
+    }
     assert(resultPid != 0);  /* childProcess never returns */
     return resultPid;
-#endif /* ! START_CHILD_USE_CLONE */
+}
+
+#if defined(__solaris__) || defined(_ALLBSD_SOURCE)
+static pid_t
+spawnChild(JNIEnv *env, jobject process, ChildStuff *c, const char *helperpath) {
+    pid_t resultPid;
+    jboolean isCopy;
+    int i, offset, rval, bufsize, magic;
+    char *buf, buf1[16];
+    char *hlpargs[2];
+    SpawnInfo sp;
+
+    /* need to tell helper which fd is for receiving the childstuff
+     * and which fd to send response back on
+     */
+    snprintf(buf1, sizeof(buf1), "%d:%d", c->childenv[0], c->fail[1]);
+    /* put the fd string as argument to the helper cmd */
+    hlpargs[0] = buf1;
+    hlpargs[1] = 0;
+
+    /* Following items are sent down the pipe to the helper
+     * after it is spawned.
+     * All strings are null terminated. All arrays of strings
+     * have an empty string for termination.
+     * - the ChildStuff struct
+     * - the SpawnInfo struct
+     * - the argv strings array
+     * - the envv strings array
+     * - the home directory string
+     * - the parentPath string
+     * - the parentPathv array
+     */
+    /* First calculate the sizes */
+    arraysize(c->argv, &sp.nargv, &sp.argvBytes);
+    bufsize = sp.argvBytes;
+    arraysize(c->envv, &sp.nenvv, &sp.envvBytes);
+    bufsize += sp.envvBytes;
+    sp.dirlen = c->pdir == 0 ? 0 : strlen(c->pdir)+1;
+    bufsize += sp.dirlen;
+    arraysize(parentPathv, &sp.nparentPathv, &sp.parentPathvBytes);
+    bufsize += sp.parentPathvBytes;
+    /* We need to clear FD_CLOEXEC if set in the fds[].
+     * Files are created FD_CLOEXEC in Java.
+     * Otherwise, they will be closed when the target gets exec'd */
+    for (i=0; i<3; i++) {
+        if (c->fds[i] != -1) {
+            int flags = fcntl(c->fds[i], F_GETFD);
+            if (flags & FD_CLOEXEC) {
+                fcntl(c->fds[i], F_SETFD, flags & (~1));
+            }
+        }
+    }
+
+    rval = posix_spawn(&resultPid, helperpath, 0, 0, (char * const *) hlpargs, environ);
+
+    if (rval != 0) {
+        return -1;
+    }
+
+    /* now the lengths are known, copy the data */
+    buf = NEW(char, bufsize);
+    if (buf == 0) {
+        return -1;
+    }
+    offset = copystrings(buf, 0, &c->argv[0]);
+    offset = copystrings(buf, offset, &c->envv[0]);
+    memcpy(buf+offset, c->pdir, sp.dirlen);
+    offset += sp.dirlen;
+    offset = copystrings(buf, offset, parentPathv);
+    assert(offset == bufsize);
+
+    magic = magicNumber();
+
+    /* write the two structs and the data buffer */
+    write(c->childenv[1], (char *)&magic, sizeof(magic)); // magic number first
+    write(c->childenv[1], (char *)c, sizeof(*c));
+    write(c->childenv[1], (char *)&sp, sizeof(sp));
+    write(c->childenv[1], buf, bufsize);
+    free(buf);
+
+    /* In this mode an external main() in invoked which calls back into
+     * childProcess() in this file, rather than directly
+     * via the statement below */
+    return resultPid;
+}
+#endif
+
+/*
+ * Start a child process running function childProcess.
+ * This function only returns in the parent.
+ */
+static pid_t
+startChild(JNIEnv *env, jobject process, ChildStuff *c, const char *helperpath) {
+    switch (c->mode) {
+      case MODE_VFORK:
+        return vforkChild(c);
+      case MODE_FORK:
+        return forkChild(c);
+#if defined(__solaris__) || defined(_ALLBSD_SOURCE)
+      case MODE_POSIX_SPAWN:
+        return spawnChild(env, process, c, helperpath);
+#endif
+      default:
+        return -1;
+    }
 }
 
 JNIEXPORT jint JNICALL
 Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
                                        jobject process,
+                                       jint mode,
+                                       jbyteArray helperpath,
                                        jbyteArray prog,
                                        jbyteArray argBlock, jint argc,
                                        jbyteArray envBlock, jint envc,
@@ -826,32 +574,35 @@ Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
 {
     int errnum;
     int resultPid = -1;
-    int in[2], out[2], err[2], fail[2];
+    int in[2], out[2], err[2], fail[2], childenv[2];
     jint *fds = NULL;
+    const char *phelperpath = NULL;
     const char *pprog = NULL;
     const char *pargBlock = NULL;
     const char *penvBlock = NULL;
     ChildStuff *c;
 
     in[0] = in[1] = out[0] = out[1] = err[0] = err[1] = fail[0] = fail[1] = -1;
+    childenv[0] = childenv[1] = -1;
 
     if ((c = NEW(ChildStuff, 1)) == NULL) return -1;
     c->argv = NULL;
     c->envv = NULL;
     c->pdir = NULL;
-#if START_CHILD_USE_CLONE
     c->clone_stack = NULL;
-#endif
 
     /* Convert prog + argBlock into a char ** argv.
      * Add one word room for expansion of argv for use by
      * execve_as_traditional_shell_script.
+     * This word is also used when using spawn mode
      */
     assert(prog != NULL && argBlock != NULL);
+    if ((phelperpath = getBytes(env, helperpath))   == NULL) goto Catch;
     if ((pprog     = getBytes(env, prog))       == NULL) goto Catch;
     if ((pargBlock = getBytes(env, argBlock))   == NULL) goto Catch;
     if ((c->argv = NEW(const char *, argc + 3)) == NULL) goto Catch;
     c->argv[0] = pprog;
+    c->argc = argc + 2;
     initVectorFromBlock(c->argv+1, pargBlock, argc);
 
     if (envBlock != NULL) {
@@ -872,6 +623,7 @@ Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
     if ((fds[0] == -1 && pipe(in)  < 0) ||
         (fds[1] == -1 && pipe(out) < 0) ||
         (fds[2] == -1 && pipe(err) < 0) ||
+        (pipe(childenv) < 0) ||
         (pipe(fail) < 0)) {
         throwIOException(env, errno, "Bad file descriptor");
         goto Catch;
@@ -884,18 +636,29 @@ Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
     copyPipe(out,  c->out);
     copyPipe(err,  c->err);
     copyPipe(fail, c->fail);
+    copyPipe(childenv, c->childenv);
 
     c->redirectErrorStream = redirectErrorStream;
+    c->mode = mode;
 
-    resultPid = startChild(c);
+    resultPid = startChild(env, process, c, phelperpath);
     assert(resultPid != 0);
 
     if (resultPid < 0) {
-        throwIOException(env, errno, START_CHILD_SYSTEM_CALL " failed");
+        switch (c->mode) {
+          case MODE_VFORK:
+            throwIOException(env, errno, "vfork failed");
+            break;
+          case MODE_FORK:
+            throwIOException(env, errno, "fork failed");
+            break;
+          case MODE_POSIX_SPAWN:
+            throwIOException(env, errno, "spawn failed");
+            break;
+        }
         goto Catch;
     }
-
-    restartableClose(fail[1]); fail[1] = -1; /* See: WhyCantJohnnyExec */
+    close(fail[1]); fail[1] = -1; /* See: WhyCantJohnnyExec  (childproc.c)  */
 
     switch (readFully(fail[0], &errnum, sizeof(errnum))) {
     case 0: break; /* Exec succeeded */
@@ -913,18 +676,18 @@ Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
     fds[2] = (err[0] != -1) ? err[0] : -1;
 
  Finally:
-#if START_CHILD_USE_CLONE
     free(c->clone_stack);
-#endif
 
     /* Always clean up the child's side of the pipes */
     closeSafely(in [0]);
     closeSafely(out[1]);
     closeSafely(err[1]);
 
-    /* Always clean up fail descriptors */
+    /* Always clean up fail and childEnv descriptors */
     closeSafely(fail[0]);
     closeSafely(fail[1]);
+    closeSafely(childenv[0]);
+    closeSafely(childenv[1]);
 
     releaseBytes(env, prog,     pprog);
     releaseBytes(env, argBlock, pargBlock);
@@ -942,9 +705,9 @@ Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
 
  Catch:
     /* Clean up the parent's side of the pipes in case of failure only */
-    closeSafely(in [1]);
-    closeSafely(out[0]);
-    closeSafely(err[0]);
+    closeSafely(in [1]); in[1] = -1;
+    closeSafely(out[0]); out[0] = -1;
+    closeSafely(err[0]); err[0] = -1;
     goto Finally;
 }
 

src/solaris/native/java/lang/childproc.c

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "childproc.h"
+
+
+ssize_t
+restartableWrite(int fd, const void *buf, size_t count)
+{
+    ssize_t result;
+    RESTARTABLE(write(fd, buf, count), result);
+    return result;
+}
+
+int
+restartableDup2(int fd_from, int fd_to)
+{
+    int err;
+    RESTARTABLE(dup2(fd_from, fd_to), err);
+    return err;
+}
+
+int
+closeSafely(int fd)
+{
+    return (fd == -1) ? 0 : close(fd);
+}
+
+int
+isAsciiDigit(char c)
+{
+  return c >= '0' && c <= '9';
+}
+
+#ifdef _ALLBSD_SOURCE
+#define FD_DIR "/dev/fd"
+#define dirent64 dirent
+#define readdir64 readdir
+#else
+#define FD_DIR "/proc/self/fd"
+#endif
+
+int
+closeDescriptors(void)
+{
+    DIR *dp;
+    struct dirent64 *dirp;
+    int from_fd = FAIL_FILENO + 1;
+
+    /* We're trying to close all file descriptors, but opendir() might
+     * itself be implemented using a file descriptor, and we certainly
+     * don't want to close that while it's in use.  We assume that if
+     * opendir() is implemented using a file descriptor, then it uses
+     * the lowest numbered file descriptor, just like open().  So we
+     * close a couple explicitly.  */
+
+    close(from_fd);          /* for possible use by opendir() */
+    close(from_fd + 1);      /* another one for good luck */
+
+    if ((dp = opendir(FD_DIR)) == NULL)
+        return 0;
+
+    /* We use readdir64 instead of readdir to work around Solaris bug
+     * 6395699: /proc/self/fd fails to report file descriptors >= 1024 on Solaris 9
+     */
+    while ((dirp = readdir64(dp)) != NULL) {
+        int fd;
+        if (isAsciiDigit(dirp->d_name[0]) &&
+            (fd = strtol(dirp->d_name, NULL, 10)) >= from_fd + 2)
+            close(fd);
+    }
+
+    closedir(dp);
+
+    return 1;
+}
+
+int
+moveDescriptor(int fd_from, int fd_to)
+{
+    if (fd_from != fd_to) {
+        if ((restartableDup2(fd_from, fd_to) == -1) ||
+            (close(fd_from) == -1))
+            return -1;
+    }
+    return 0;
+}
+
+int
+magicNumber() {
+    return 43110;
+}
+
+/*
+ * Reads nbyte bytes from file descriptor fd into buf,
+ * The read operation is retried in case of EINTR or partial reads.
+ *
+ * Returns number of bytes read (normally nbyte, but may be less in
+ * case of EOF).  In case of read errors, returns -1 and sets errno.
+ */
+ssize_t
+readFully(int fd, void *buf, size_t nbyte)
+{
+    ssize_t remaining = nbyte;
+    for (;;) {
+        ssize_t n = read(fd, buf, remaining);
+        if (n == 0) {
+            return nbyte - remaining;
+        } else if (n > 0) {
+            remaining -= n;
+            if (remaining <= 0)
+                return nbyte;
+            /* We were interrupted in the middle of reading the bytes.
+             * Unlikely, but possible. */
+            buf = (void *) (((char *)buf) + n);
+        } else if (errno == EINTR) {
+            /* Strange signals like SIGJVM1 are possible at any time.
+             * See http://www.dreamsongs.com/WorseIsBetter.html */
+        } else {
+            return -1;
+        }
+    }
+}
+
+void
+initVectorFromBlock(const char**vector, const char* block, int count)
+{
+    int i;
+    const char *p;
+    for (i = 0, p = block; i < count; i++) {
+        /* Invariant: p always points to the start of a C string. */
+        vector[i] = p;
+        while (*(p++));
+    }
+    vector[count] = NULL;
+}
+
+/**
+ * Exec FILE as a traditional Bourne shell script (i.e. one without #!).
+ * If we could do it over again, we would probably not support such an ancient
+ * misfeature, but compatibility wins over sanity.  The original support for
+ * this was imported accidentally from execvp().
+ */
+void
+execve_as_traditional_shell_script(const char *file,
+                                   const char *argv[],
+                                   const char *const envp[])
+{
+    /* Use the extra word of space provided for us in argv by caller. */
+    const char *argv0 = argv[0];
+    const char *const *end = argv;
+    while (*end != NULL)
+        ++end;
+    memmove(argv+2, argv+1, (end-argv) * sizeof(*end));
+    argv[0] = "/bin/sh";
+    argv[1] = file;
+    execve(argv[0], (char **) argv, (char **) envp);
+    /* Can't even exec /bin/sh?  Big trouble, but let's soldier on... */
+    memmove(argv+1, argv+2, (end-argv) * sizeof(*end));
+    argv[0] = argv0;
+}
+
+/**
+ * Like execve(2), except that in case of ENOEXEC, FILE is assumed to
+ * be a shell script and the system default shell is invoked to run it.
+ */
+void
+execve_with_shell_fallback(int mode, const char *file,
+                           const char *argv[],
+                           const char *const envp[])
+{
+    if (mode == MODE_CLONE || mode == MODE_VFORK) {
+        /* shared address space; be very careful. */
+        execve(file, (char **) argv, (char **) envp);
+        if (errno == ENOEXEC)
+            execve_as_traditional_shell_script(file, argv, envp);
+    } else {
+        /* unshared address space; we can mutate environ. */
+        environ = (char **) envp;
+        execvp(file, (char **) argv);
+    }
+}
+
+/**
+ * 'execvpe' should have been included in the Unix standards,
+ * and is a GNU extension in glibc 2.10.
+ *
+ * JDK_execvpe is identical to execvp, except that the child environment is
+ * specified via the 3rd argument instead of being inherited from environ.
+ */
+void
+JDK_execvpe(int mode, const char *file,
+            const char *argv[],
+            const char *const envp[])
+{
+    if (envp == NULL || (char **) envp == environ) {
+        execvp(file, (char **) argv);
+        return;
+    }
+
+    if (*file == '\0') {
+        errno = ENOENT;
+        return;
+    }
+
+    if (strchr(file, '/') != NULL) {
+        execve_with_shell_fallback(mode, file, argv, envp);
+    } else {
+        /* We must search PATH (parent's, not child's) */
+        char expanded_file[PATH_MAX];
+        int filelen = strlen(file);
+        int sticky_errno = 0;
+        const char * const * dirs;
+        for (dirs = parentPathv; *dirs; dirs++) {
+            const char * dir = *dirs;
+            int dirlen = strlen(dir);
+            if (filelen + dirlen + 2 >= PATH_MAX) {
+                errno = ENAMETOOLONG;
+                continue;
+            }
+            memcpy(expanded_file, dir, dirlen);
+            if (expanded_file[dirlen - 1] != '/')
+                expanded_file[dirlen++] = '/';
+            memcpy(expanded_file + dirlen, file, filelen);
+            expanded_file[dirlen + filelen] = '\0';
+            execve_with_shell_fallback(mode, expanded_file, argv, envp);
+            /* There are 3 responses to various classes of errno:
+             * return immediately, continue (especially for ENOENT),
+             * or continue with "sticky" errno.
+             *
+             * From exec(3):
+             *
+             * If permission is denied for a file (the attempted
+             * execve returned EACCES), these functions will continue
+             * searching the rest of the search path.  If no other
+             * file is found, however, they will return with the
+             * global variable errno set to EACCES.
+             */
+            switch (errno) {
+            case EACCES:
+                sticky_errno = errno;
+                /* FALLTHRU */
+            case ENOENT:
+            case ENOTDIR:
+#ifdef ELOOP
+            case ELOOP:
+#endif
+#ifdef ESTALE
+            case ESTALE:
+#endif
+#ifdef ENODEV
+            case ENODEV:
+#endif
+#ifdef ETIMEDOUT
+            case ETIMEDOUT:
+#endif
+                break; /* Try other directories in PATH */
+            default:
+                return;
+            }
+        }
+        if (sticky_errno != 0)
+            errno = sticky_errno;
+    }
+}
+
+/**
+ * Child process after a successful fork() or clone().
+ * This function must not return, and must be prepared for either all
+ * of its address space to be shared with its parent, or to be a copy.
+ * It must not modify global variables such as "environ".
+ */
+int
+childProcess(void *arg)
+{
+    const ChildStuff* p = (const ChildStuff*) arg;
+
+    /* Close the parent sides of the pipes.
+       Closing pipe fds here is redundant, since closeDescriptors()
+       would do it anyways, but a little paranoia is a good thing. */
+    if ((closeSafely(p->in[1])   == -1) ||
+        (closeSafely(p->out[0])  == -1) ||
+        (closeSafely(p->err[0])  == -1) ||
+        (closeSafely(p->childenv[0])  == -1) ||
+        (closeSafely(p->childenv[1])  == -1) ||
+        (closeSafely(p->fail[0]) == -1))
+        goto WhyCantJohnnyExec;
+
+    /* Give the child sides of the pipes the right fileno's. */
+    /* Note: it is possible for in[0] == 0 */
+    if ((moveDescriptor(p->in[0] != -1 ?  p->in[0] : p->fds[0],
+                        STDIN_FILENO) == -1) ||
+        (moveDescriptor(p->out[1]!= -1 ? p->out[1] : p->fds[1],
+                        STDOUT_FILENO) == -1))
+        goto WhyCantJohnnyExec;
+
+    if (p->redirectErrorStream) {
+        if ((closeSafely(p->err[1]) == -1) ||
+            (restartableDup2(STDOUT_FILENO, STDERR_FILENO) == -1))
+            goto WhyCantJohnnyExec;
+    } else {
+        if (moveDescriptor(p->err[1] != -1 ? p->err[1] : p->fds[2],
+                           STDERR_FILENO) == -1)
+            goto WhyCantJohnnyExec;
+    }
+
+    if (moveDescriptor(p->fail[1], FAIL_FILENO) == -1)
+        goto WhyCantJohnnyExec;
+
+    /* close everything */
+    if (closeDescriptors() == 0) { /* failed,  close the old way */
+        int max_fd = (int)sysconf(_SC_OPEN_MAX);
+        int fd;
+        for (fd = FAIL_FILENO + 1; fd < max_fd; fd++)
+            if (close(fd) == -1 && errno != EBADF)
+                goto WhyCantJohnnyExec;
+    }
+
+    /* change to the new working directory */
+    if (p->pdir != NULL && chdir(p->pdir) < 0)
+        goto WhyCantJohnnyExec;
+
+    if (fcntl(FAIL_FILENO, F_SETFD, FD_CLOEXEC) == -1)
+        goto WhyCantJohnnyExec;
+
+    JDK_execvpe(p->mode, p->argv[0], p->argv, p->envv);
+
+ WhyCantJohnnyExec:
+    /* We used to go to an awful lot of trouble to predict whether the
+     * child would fail, but there is no reliable way to predict the
+     * success of an operation without *trying* it, and there's no way
+     * to try a chdir or exec in the parent.  Instead, all we need is a
+     * way to communicate any failure back to the parent.  Easy; we just
+     * send the errno back to the parent over a pipe in case of failure.
+     * The tricky thing is, how do we communicate the *success* of exec?
+     * We use FD_CLOEXEC together with the fact that a read() on a pipe
+     * yields EOF when the write ends (we have two of them!) are closed.
+     */
+    {
+        int errnum = errno;
+        restartableWrite(FAIL_FILENO, &errnum, sizeof(errnum));
+    }
+    close(FAIL_FILENO);
+    _exit(-1);
+    return 0;  /* Suppress warning "no return value from function" */
+}

src/solaris/native/java/lang/childproc.h

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CHILDPROC_MD_H
+#define CHILDPROC_MD_H
+
+#include <sys/types.h>
+
+#ifdef __APPLE__
+#include <crt_externs.h>
+#define environ (*_NSGetEnviron())
+#else
+/* This is one of the rare times it's more portable to declare an
+ * external symbol explicitly, rather than via a system header.
+ * The declaration is standardized as part of UNIX98, but there is
+ * no standard (not even de-facto) header file where the
+ * declaration is to be found.  See:
+ * http://www.opengroup.org/onlinepubs/009695399/functions/environ.html
+ * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_02.html
+ *
+ * "All identifiers in this volume of IEEE Std 1003.1-2001, except
+ * environ, are defined in at least one of the headers" (!)
+ */
+extern char **environ;
+#endif
+
+#ifdef __linux__
+#include <sched.h>
+#endif
+
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+
+#ifndef STDERR_FILENO
+#define STDERR_FILENO 2
+#endif
+
+#ifndef SA_NOCLDSTOP
+#define SA_NOCLDSTOP 0
+#endif
+
+#ifndef SA_RESTART
+#define SA_RESTART 0
+#endif
+
+#define FAIL_FILENO (STDERR_FILENO + 1)
+
+/* TODO: Refactor. */
+#define RESTARTABLE(_cmd, _result) do { \
+  do { \
+    _result = _cmd; \
+  } while((_result == -1) && (errno == EINTR)); \
+} while(0)
+
+/* These numbers must be the same as the Enum in UNIXProcess.java
+ * Must be a better way of doing this.
+ */
+#define MODE_FORK 1
+#define MODE_POSIX_SPAWN 2
+#define MODE_VFORK 3
+#define MODE_CLONE 4
+
+typedef struct _ChildStuff
+{
+    int in[2];
+    int out[2];
+    int err[2];
+    int fail[2];
+    int childenv[2];
+    int fds[3];
+    int mode;
+    const char **argv;
+    int argc;
+    const char **envv;
+    const char *pdir;
+    int redirectErrorStream;
+    void *clone_stack;
+} ChildStuff;
+
+/* following used in addition when mode is SPAWN */
+typedef struct _SpawnInfo {
+    int nargv; /* number of argv array elements  */
+    int argvBytes; /* total number of bytes in argv array */
+    int nenvv; /* number of envv array elements  */
+    int envvBytes; /* total number of bytes in envv array */
+    int dirlen; /* length of home directory string */
+    int nparentPathv; /* number of elements in parentPathv array */
+    int parentPathvBytes; /* total number of bytes in parentPathv array */
+} SpawnInfo;
+
+/**
+ * The cached and split version of the JDK's effective PATH.
+ * (We don't support putenv("PATH=...") in native code)
+ */
+const char * const *parentPathv;
+
+ssize_t restartableWrite(int fd, const void *buf, size_t count);
+int restartableDup2(int fd_from, int fd_to);
+int closeSafely(int fd);
+int isAsciiDigit(char c);
+int closeDescriptors(void);
+int moveDescriptor(int fd_from, int fd_to);
+
+int magicNumber();
+ssize_t readFully(int fd, void *buf, size_t nbyte);
+void initVectorFromBlock(const char**vector, const char* block, int count);
+void execve_as_traditional_shell_script(const char *file,
+                                        const char *argv[],
+                                        const char *const envp[]);
+void execve_with_shell_fallback(int mode, const char *file,
+                                const char *argv[],
+                                const char *const envp[]);
+void JDK_execvpe(int mode, const char *file,
+                 const char *argv[],
+                 const char *const envp[]);
+int childProcess(void *arg);
+
+#endif

src/solaris/native/java/lang/jspawnhelper.c

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "childproc.h"
+
+extern int errno;
+
+#define ALLOC(X,Y) { \
+    void *mptr; \
+    mptr = malloc (Y); \
+    if (mptr == 0) { \
+        error (fdout, ERR_MALLOC); \
+    } \
+    X = mptr; \
+}
+
+#define ERR_MALLOC 1
+#define ERR_PIPE 2
+#define ERR_ARGS 3
+
+void error (int fd, int err) {
+    write (fd, &err, sizeof(err));
+    exit (1);
+}
+
+void shutItDown() {
+    fprintf(stdout, "This command is not for general use and should ");
+    fprintf(stdout, "only be run as the result of a call to\n");
+    fprintf(stdout, "ProcessBuilder.start() or Runtime.exec() in a java ");
+    fprintf(stdout, "application\n");
+    _exit(1);
+}
+
+/*
+ * read the following off the pipefd
+ * - the ChildStuff struct
+ * - the SpawnInfo struct
+ * - the data strings for fields in ChildStuff
+ */
+void initChildStuff (int fdin, int fdout, ChildStuff *c) {
+    int n;
+    int argvBytes, nargv, envvBytes, nenvv;
+    int dirlen;
+    char *buf;
+    SpawnInfo sp;
+    int bufsize, offset=0;
+    int magic;
+    int res;
+
+    res = readFully (fdin, &magic, sizeof(magic));
+    if (res != 4 || magic != magicNumber()) {
+        error (fdout, ERR_PIPE);
+    }
+
+    if (readFully (fdin, c, sizeof(*c)) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    if (readFully (fdin, &sp, sizeof(sp)) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    bufsize = sp.argvBytes + sp.envvBytes +
+              sp.dirlen + sp.parentPathvBytes;
+
+    ALLOC(buf, bufsize);
+
+    if (readFully (fdin, buf, bufsize) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    /* Initialize argv[] */
+    ALLOC(c->argv, sizeof(char *) * sp.nargv);
+    initVectorFromBlock (c->argv, buf+offset, sp.nargv-1);
+    offset += sp.argvBytes;
+
+    /* Initialize envv[] */
+    if (sp.nenvv == 0) {
+        c->envv = 0;
+    } else {
+        ALLOC(c->envv, sizeof(char *) * sp.nenvv);
+        initVectorFromBlock (c->envv, buf+offset, sp.nenvv-1);
+        offset += sp.envvBytes;
+    }
+
+    /* Initialize pdir */
+    if (sp.dirlen == 0) {
+        c->pdir = 0;
+    } else {
+        c->pdir = buf+offset;
+        offset += sp.dirlen;
+    }
+
+    /* Initialize parentPathv[] */
+    ALLOC(parentPathv, sizeof (char *) * sp.nparentPathv)
+    initVectorFromBlock ((const char**)parentPathv, buf+offset, sp.nparentPathv-1);
+    offset += sp.parentPathvBytes;
+}
+
+int main(int argc, char *argv[]) {
+    ChildStuff c;
+    int t;
+    struct stat buf;
+    /* argv[0] contains the fd number to read all the child info */
+    int r, fdin, fdout;
+
+    r = sscanf (argv[argc-1], "%d:%d", &fdin, &fdout);
+    if (r == 2 && fcntl(fdin, F_GETFD) != -1) {
+        fstat(fdin, &buf);
+        if (!S_ISFIFO(buf.st_mode))
+            shutItDown();
+    } else {
+        shutItDown();
+    }
+    initChildStuff (fdin, fdout, &c);
+
+    childProcess (&c);
+    return 0; /* NOT REACHED */
+}

test/java/lang/ProcessBuilder/Basic.java

@@ -26,9 +26,10 @@
  * @bug 4199068 4738465 4937983 4930681 4926230 4931433 4932663 4986689
  *      5026830 5023243 5070673 4052517 4811767 6192449 6397034 6413313
  *      6464154 6523983 6206031 4960438 6631352 6631966 6850957 6850958
- *      4947220 7018606 7034570 4244896
+ *      4947220 7018606 7034570 4244896 5049299
  * @summary Basic tests for Process and Environment Variable code
  * @run main/othervm/timeout=300 Basic
+ * @run main/othervm/timeout=300 -Djdk.lang.Process.launchMechanism=fork Basic
  * @author Martin Buchholz
  */