5049299: (process) Use posix_spawn, not fork, on S10 to avoid swap exhaustion

Reviewed-by: alanb, dholmes, martin, erikj, coffeys

make/java/java/Exportedfiles.gmk

@@ -50,6 +50,7 @@ FILES_export = \
     java/lang/SecurityManager.java \
     java/lang/Shutdown.java \
     java/lang/Package.java \
+    java/lang/UNIXProcess.java \
     java/lang/ref/Finalizer.java \
     java/lang/reflect/AccessibleObject.java \
     java/lang/reflect/Field.java \

make/java/java/Makefile

@@ -87,6 +87,7 @@ FILES_java += 	java/lang/UNIXProcess.java \
                 java/util/prefs/FileSystemPreferencesFactory.java \
 
 FILES_c    +=   UNIXProcess_md.c \
+		childproc.c \
 		UnixFileSystem_md.c \
 		canonicalize_md.c \
 		TimeZone.c \
@@ -468,3 +469,36 @@ $(GENSRCDIR)/sun/util/CoreResourceBundleControl.java: \
 clean::
 	$(RM) $(GENSRCDIR)/sun/util/CoreResourceBundleControl.java
 
+
+HELPER_EXE = $(LIBDIR)/$(LIBARCH)/jspawnhelper
+BUILDHELPER =
+ifeq ($(PLATFORM), solaris)
+    BUILDHELPER = 1
+endif
+ifeq ($(PLATFORM), macosx)
+    HELPER_EXE = $(LIBDIR)/jspawnhelper
+    BUILDHELPER = 1
+endif
+
+ARCHFLAG = 
+ifeq ($(ARCH_DATA_MODEL), 64)
+ARCHFLAG = -m64
+endif
+
+ifdef BUILDHELPER
+
+HELPER_EXE_FILES_c = jspawnhelper.c
+HELPER_EXE_FILES_o = $(OBJDIR)/jspawnhelper.o \
+		$(OBJDIR)/childproc.o
+
+$(HELPER_EXE): $(HELPER_EXE_FILES_o)
+	$(CC) $(ARCHFLAG) $(HELPER_EXE_FILES_o) \
+		-o $(TEMPDIR)/jspawnhelper
+	$(CP) $(TEMPDIR)/jspawnhelper $(HELPER_EXE)
+
+build: $(HELPER_EXE)
+
+clean clobber:: 
+ 	$(RM) $(HELPER_EXE_FILES_o) $(HELPER_EXE)
+
+endif #BUILDHELPER
\ No newline at end of file

makefiles/CompileLaunchers.gmk

@@ -40,7 +40,7 @@ include Tools.gmk
 
 BUILD_LAUNCHERS=
 
-# When building a legacy overlay image (on solaris 64 bit), the launchers 
+# When building a legacy overlay image (on solaris 64 bit), the launchers
 # need to be built with a different rpath and a different output dir.
 ifeq ($(OVERLAY_IMAGES),true)
     ORIGIN_ROOT:=/../..
@@ -63,7 +63,7 @@ endif
 #
 ifneq ($(findstring $(OPENJDK_TARGET_OS), linux solaris),)
     ORIGIN_ARG+=$(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/lib$(OPENJDK_TARGET_CPU_LIBDIR)) \
-                $(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/jre/lib$(OPENJDK_TARGET_CPU_LIBDIR)) 
+                $(call SET_EXECUTABLE_ORIGIN,$(ORIGIN_ROOT)/jre/lib$(OPENJDK_TARGET_CPU_LIBDIR))
 endif
 
 define SetupLauncher
@@ -189,7 +189,7 @@ define SetupLauncher
     ifeq ($(OPENJDK_TARGET_OS),windows)
         $$(BUILD_LAUNCHER_$1) : $(JDK_OUTPUTDIR)/objs/libjava/java.lib \
 				$$($1_WINDOWS_JLI_LIB)
-    endif    	 
+    endif
 endef
 
 ##########################################################################################
@@ -441,7 +441,7 @@ ifeq ($(OPENJDK_TARGET_OS),solaris)
 endif
 # On windows, unpack200 is linked completely differently to all other
 # executables, using the compiler with the compiler arguments.
-# It's also linked incrementally, producing a .ilk file that needs to 
+# It's also linked incrementally, producing a .ilk file that needs to
 # be kept away.
 ifeq ($(OPENJDK_TARGET_OS),windows)
     BUILD_UNPACKEXE_LDEXE:=$(CC)
@@ -490,7 +490,7 @@ $(BUILD_UNPACKEXE) : $(UNPACKEXE_ZIPOBJS)
 endif
 
 # Build into object dir and copy executable afterwards to avoid .ilk file in
-# image. The real fix would be clean up linking of unpack200 using 
+# image. The real fix would be clean up linking of unpack200 using
 # -link -incremental:no
 # like all other launchers.
 $(JDK_OUTPUTDIR)/bin$(OUTPUT_SUBDIR)/unpack200$(EXE_SUFFIX): $(BUILD_UNPACKEXE)
@@ -501,7 +501,7 @@ BUILD_LAUNCHERS += $(JDK_OUTPUTDIR)/bin$(OUTPUT_SUBDIR)/unpack200$(EXE_SUFFIX)
 ##########################################################################################
 
 
-BUILD_JEXEC := 
+BUILD_JEXEC :=
 BUILD_JEXEC_SRC :=
 BUILD_JEXEC_INC :=
 BUILD_JEXEC_DST_DIR := $(JDK_OUTPUTDIR)/lib$(OPENJDK_TARGET_CPU_LIBDIR)
@@ -581,11 +581,11 @@ ifeq ($(OPENJDK_TARGET_OS), solaris)
   endif
 endif
 
-# TODO: 
+# TODO:
 # On windows java-rmi.cgi shouldn't be bundled since Java 1.2, but has been built all
 # this time anyway. Since jdk6, it has been built from the wrong source and resulted
-# in a (almost) copy of the standard java launcher named "java-rmi.exe" ending up in 
-# the final images bin dir. This weird behavior is mimicked here in the converted 
+# in a (almost) copy of the standard java launcher named "java-rmi.exe" ending up in
+# the final images bin dir. This weird behavior is mimicked here in the converted
 # makefiles for now. Should probably just be deleted.
 # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6512052
 ifeq ($(OPENJDK_TARGET_OS),windows)
@@ -597,6 +597,44 @@ else
 	$(CHMOD) a+x $@
 endif
 
+##########################################################################################
+
+BUILD_JSPAWNHELPER :=
+BUILD_JSPAWNHELPER_SRC := $(JDK_TOPDIR)/src/solaris/native/java/lang
+BUILD_JSPAWNHELPER_DST_DIR := $(JDK_OUTPUTDIR)/lib$(OPENJDK_TARGET_CPU_LIBDIR)
+LINK_JSPAWNHELPER_OBJECTS := $(JDK_OUTPUTDIR)/objs/libjava/childproc.o
+LINK_JSPAWNHELPER_FLAGS :=
+
+ifneq ($(findstring $(OPENJDK_TARGET_OS), macosx solaris),)
+    BUILD_JSPAWNHELPER := 1
+endif
+
+ifeq ($(OPENJDK_TARGET_OS), macosx)
+    BUILD_JSPAWNHELPER_DST_DIR := $(JDK_OUTPUTDIR)/lib
+endif
+
+ifeq ($(OPENJDK_TARGET_CPU_BITS), 64)
+    LINK_JSPAWNHELPER_FLAGS += -m64
+endif
+
+ifeq ($(BUILD_JSPAWNHELPER), 1)
+    $(eval $(call SetupNativeCompilation,BUILD_JSPAWNHELPER,\
+        SRC:=$(BUILD_JSPAWNHELPER_SRC),\
+        INCLUDE_FILES:=jspawnhelper.c,\
+        LANG:=C,\
+        OPTIMIZATION := LOW, \
+        CFLAGS:=$(CFLAGS_JDKEXE), \
+        LDFLAGS:=$(LDFLAGS_JDKEXE) $(LINK_JSPAWNHELPER_FLAGS), \
+        LDFLAGS_SUFFIX:= $(LINK_JSPAWNHELPER_OBJECTS), \
+        OBJECT_DIR:=$(JDK_OUTPUTDIR)/objs/jspawnhelper,\
+        OUTPUT_DIR:=$(BUILD_JSPAWNHELPER_DST_DIR),\
+        PROGRAM:=jspawnhelper))
+
+    $(BUILD_JSPAWNHELPER): $(LINK_JSPAWNHELPER_OBJECTS)
+
+    BUILD_LAUNCHERS += $(BUILD_JSPAWNHELPER)
+endif
+
 ##########################################################################################
 # jabswitch
 

makefiles/CompileNativeLibraries.gmk

@@ -205,7 +205,7 @@ ifneq (,$(JDK_UPDATE_VERSION))
 	LIBJAVA_CFLAGS += -DJDK_UPDATE_VERSION='"$(JDK_UPDATE_VERSION)"'
 endif
 
-LIBJAVA_EXCLUDE_FILES:=check_code.c check_format.c
+LIBJAVA_EXCLUDE_FILES:=check_code.c check_format.c jspawnhelper.c
 
 ifneq ($(OPENJDK_TARGET_OS),macosx)
 	LIBJAVA_EXCLUDE_FILES += java_props_macosx.c

src/solaris/classes/java/lang/UNIXProcess.java.bsd

@@ -64,11 +64,59 @@ final class UNIXProcess extends Process {
     private /* final */ InputStream  stdout;
     private /* final */ InputStream  stderr;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        POSIX_SPAWN(2);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* On BSD, the default is to spawn */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+
+                helperpath = toCString(javahome + "/lib/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "posix_spawn");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - posix_spawn(2)
      *
      * @param fds an array of three file descriptors.
      *        Indexes 0, 1, and 2 correspond to standard input,
@@ -81,7 +129,8 @@ final class UNIXProcess extends Process {
      *        output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -133,7 +182,9 @@ final class UNIXProcess extends Process {
                 final boolean redirectErrorStream)
             throws IOException {
 
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,
@@ -212,13 +263,13 @@ final class UNIXProcess extends Process {
         }
         return exitcode;
     }
-    
+
     @Override
-    public synchronized boolean waitFor(long timeout, TimeUnit unit) 
-        throws InterruptedException 
+    public synchronized boolean waitFor(long timeout, TimeUnit unit)
+        throws InterruptedException
     {
         if (hasExited) return true;
-        if (timeout <= 0) return false;	
+        if (timeout <= 0) return false;
 
         long timeoutAsNanos = unit.toNanos(timeout);
         long startTime = System.nanoTime();

src/solaris/classes/java/lang/UNIXProcess.java.linux

@@ -64,11 +64,61 @@ final class UNIXProcess extends Process {
     private /* final */ InputStream  stdout;
     private /* final */ InputStream  stderr;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        VFORK(3);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* default is VFORK on Linux */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+                String osArch = System.getProperty("os.arch");
+
+                helperpath = toCString(javahome + "/lib/" + osArch + "/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "vfork");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - clone(2) and exec(2)
+     * - vfork(2) and exec(2)
      *
      * @param fds an array of three file descriptors.
      *        Indexes 0, 1, and 2 correspond to standard input,
@@ -81,7 +131,8 @@ final class UNIXProcess extends Process {
      *        output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -133,7 +184,9 @@ final class UNIXProcess extends Process {
                 final boolean redirectErrorStream)
             throws IOException {
 
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,

src/solaris/classes/java/lang/UNIXProcess.java.solaris

@@ -27,6 +27,8 @@ package java.lang;
 
 import java.io.*;
 import java.util.concurrent.TimeUnit;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 
 /* java.lang.Process subclass in the UNIX environment.
  *
@@ -46,11 +48,65 @@ final class UNIXProcess extends Process {
     private DeferredCloseInputStream stdout_inner_stream;
     private InputStream stderr_stream;
 
+    private static enum LaunchMechanism {
+        FORK(1),
+        POSIX_SPAWN(2);
+
+        private int value;
+        LaunchMechanism(int x) {value = x;}
+    };
+
+    /* On Solaris, the default is to spawn */
+    private static final LaunchMechanism launchMechanism;
+    private static byte[] helperpath;
+
+    private static byte[] toCString(String s) {
+        if (s == null)
+            return null;
+        byte[] bytes = s.getBytes();
+        byte[] result = new byte[bytes.length + 1];
+        System.arraycopy(bytes, 0,
+                         result, 0,
+                         bytes.length);
+        result[result.length-1] = (byte)0;
+        return result;
+    }
+
+    static {
+        launchMechanism = AccessController.doPrivileged(
+                new PrivilegedAction<LaunchMechanism>()
+        {
+            public LaunchMechanism run() {
+                String javahome = System.getProperty("java.home");
+                String osArch = System.getProperty("os.arch");
+                if (osArch.equals("x86")) {
+                    osArch = "i386";
+                } else if (osArch.equals("x86_64")) {
+                    osArch = "amd64";
+                }
+
+                helperpath = toCString(javahome + "/lib/" + osArch + "/jspawnhelper");
+                String s = System.getProperty(
+                    "jdk.lang.Process.launchMechanism", "posix_spawn");
+
+                try {
+                    return LaunchMechanism.valueOf(s.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new Error(s + " is not a supported " +
+                        "process launch mechanism on this platform.");
+                }
+            }
+        });
+    }
+
     /* this is for the reaping thread */
     private native int waitForProcessExit(int pid);
 
     /**
-     * Create a process using fork(2) and exec(2).
+     * Create a process. Depending on the mode flag, this is done by
+     * one of the following mechanisms.
+     * - fork(2) and exec(2)
+     * - posix_spawn(2)
      *
      * @param std_fds array of file descriptors.  Indexes 0, 1, and
      *        2 correspond to standard input, standard output and
@@ -62,7 +118,8 @@ final class UNIXProcess extends Process {
      *        if and only if it is <em>not</em> -1 on output.
      * @return the pid of the subprocess
      */
-    private native int forkAndExec(byte[] prog,
+    private native int forkAndExec(int mode, byte[] helperpath,
+                                   byte[] prog,
                                    byte[] argBlock, int argc,
                                    byte[] envBlock, int envc,
                                    byte[] dir,
@@ -77,7 +134,9 @@ final class UNIXProcess extends Process {
                 final int[] std_fds,
                 final boolean redirectErrorStream)
     throws IOException {
-        pid = forkAndExec(prog,
+        pid = forkAndExec(launchMechanism.value,
+                          helperpath,
+                          prog,
                           argBlock, argc,
                           envBlock, envc,
                           dir,

src/solaris/native/java/lang/childproc.c

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "childproc.h"
+
+
+ssize_t
+restartableWrite(int fd, const void *buf, size_t count)
+{
+    ssize_t result;
+    RESTARTABLE(write(fd, buf, count), result);
+    return result;
+}
+
+int
+restartableDup2(int fd_from, int fd_to)
+{
+    int err;
+    RESTARTABLE(dup2(fd_from, fd_to), err);
+    return err;
+}
+
+int
+closeSafely(int fd)
+{
+    return (fd == -1) ? 0 : close(fd);
+}
+
+int
+isAsciiDigit(char c)
+{
+  return c >= '0' && c <= '9';
+}
+
+#ifdef _ALLBSD_SOURCE
+#define FD_DIR "/dev/fd"
+#define dirent64 dirent
+#define readdir64 readdir
+#else
+#define FD_DIR "/proc/self/fd"
+#endif
+
+int
+closeDescriptors(void)
+{
+    DIR *dp;
+    struct dirent64 *dirp;
+    int from_fd = FAIL_FILENO + 1;
+
+    /* We're trying to close all file descriptors, but opendir() might
+     * itself be implemented using a file descriptor, and we certainly
+     * don't want to close that while it's in use.  We assume that if
+     * opendir() is implemented using a file descriptor, then it uses
+     * the lowest numbered file descriptor, just like open().  So we
+     * close a couple explicitly.  */
+
+    close(from_fd);          /* for possible use by opendir() */
+    close(from_fd + 1);      /* another one for good luck */
+
+    if ((dp = opendir(FD_DIR)) == NULL)
+        return 0;
+
+    /* We use readdir64 instead of readdir to work around Solaris bug
+     * 6395699: /proc/self/fd fails to report file descriptors >= 1024 on Solaris 9
+     */
+    while ((dirp = readdir64(dp)) != NULL) {
+        int fd;
+        if (isAsciiDigit(dirp->d_name[0]) &&
+            (fd = strtol(dirp->d_name, NULL, 10)) >= from_fd + 2)
+            close(fd);
+    }
+
+    closedir(dp);
+
+    return 1;
+}
+
+int
+moveDescriptor(int fd_from, int fd_to)
+{
+    if (fd_from != fd_to) {
+        if ((restartableDup2(fd_from, fd_to) == -1) ||
+            (close(fd_from) == -1))
+            return -1;
+    }
+    return 0;
+}
+
+int
+magicNumber() {
+    return 43110;
+}
+
+/*
+ * Reads nbyte bytes from file descriptor fd into buf,
+ * The read operation is retried in case of EINTR or partial reads.
+ *
+ * Returns number of bytes read (normally nbyte, but may be less in
+ * case of EOF).  In case of read errors, returns -1 and sets errno.
+ */
+ssize_t
+readFully(int fd, void *buf, size_t nbyte)
+{
+    ssize_t remaining = nbyte;
+    for (;;) {
+        ssize_t n = read(fd, buf, remaining);
+        if (n == 0) {
+            return nbyte - remaining;
+        } else if (n > 0) {
+            remaining -= n;
+            if (remaining <= 0)
+                return nbyte;
+            /* We were interrupted in the middle of reading the bytes.
+             * Unlikely, but possible. */
+            buf = (void *) (((char *)buf) + n);
+        } else if (errno == EINTR) {
+            /* Strange signals like SIGJVM1 are possible at any time.
+             * See http://www.dreamsongs.com/WorseIsBetter.html */
+        } else {
+            return -1;
+        }
+    }
+}
+
+void
+initVectorFromBlock(const char**vector, const char* block, int count)
+{
+    int i;
+    const char *p;
+    for (i = 0, p = block; i < count; i++) {
+        /* Invariant: p always points to the start of a C string. */
+        vector[i] = p;
+        while (*(p++));
+    }
+    vector[count] = NULL;
+}
+
+/**
+ * Exec FILE as a traditional Bourne shell script (i.e. one without #!).
+ * If we could do it over again, we would probably not support such an ancient
+ * misfeature, but compatibility wins over sanity.  The original support for
+ * this was imported accidentally from execvp().
+ */
+void
+execve_as_traditional_shell_script(const char *file,
+                                   const char *argv[],
+                                   const char *const envp[])
+{
+    /* Use the extra word of space provided for us in argv by caller. */
+    const char *argv0 = argv[0];
+    const char *const *end = argv;
+    while (*end != NULL)
+        ++end;
+    memmove(argv+2, argv+1, (end-argv) * sizeof(*end));
+    argv[0] = "/bin/sh";
+    argv[1] = file;
+    execve(argv[0], (char **) argv, (char **) envp);
+    /* Can't even exec /bin/sh?  Big trouble, but let's soldier on... */
+    memmove(argv+1, argv+2, (end-argv) * sizeof(*end));
+    argv[0] = argv0;
+}
+
+/**
+ * Like execve(2), except that in case of ENOEXEC, FILE is assumed to
+ * be a shell script and the system default shell is invoked to run it.
+ */
+void
+execve_with_shell_fallback(int mode, const char *file,
+                           const char *argv[],
+                           const char *const envp[])
+{
+    if (mode == MODE_CLONE || mode == MODE_VFORK) {
+        /* shared address space; be very careful. */
+        execve(file, (char **) argv, (char **) envp);
+        if (errno == ENOEXEC)
+            execve_as_traditional_shell_script(file, argv, envp);
+    } else {
+        /* unshared address space; we can mutate environ. */
+        environ = (char **) envp;
+        execvp(file, (char **) argv);
+    }
+}
+
+/**
+ * 'execvpe' should have been included in the Unix standards,
+ * and is a GNU extension in glibc 2.10.
+ *
+ * JDK_execvpe is identical to execvp, except that the child environment is
+ * specified via the 3rd argument instead of being inherited from environ.
+ */
+void
+JDK_execvpe(int mode, const char *file,
+            const char *argv[],
+            const char *const envp[])
+{
+    if (envp == NULL || (char **) envp == environ) {
+        execvp(file, (char **) argv);
+        return;
+    }
+
+    if (*file == '\0') {
+        errno = ENOENT;
+        return;
+    }
+
+    if (strchr(file, '/') != NULL) {
+        execve_with_shell_fallback(mode, file, argv, envp);
+    } else {
+        /* We must search PATH (parent's, not child's) */
+        char expanded_file[PATH_MAX];
+        int filelen = strlen(file);
+        int sticky_errno = 0;
+        const char * const * dirs;
+        for (dirs = parentPathv; *dirs; dirs++) {
+            const char * dir = *dirs;
+            int dirlen = strlen(dir);
+            if (filelen + dirlen + 2 >= PATH_MAX) {
+                errno = ENAMETOOLONG;
+                continue;
+            }
+            memcpy(expanded_file, dir, dirlen);
+            if (expanded_file[dirlen - 1] != '/')
+                expanded_file[dirlen++] = '/';
+            memcpy(expanded_file + dirlen, file, filelen);
+            expanded_file[dirlen + filelen] = '\0';
+            execve_with_shell_fallback(mode, expanded_file, argv, envp);
+            /* There are 3 responses to various classes of errno:
+             * return immediately, continue (especially for ENOENT),
+             * or continue with "sticky" errno.
+             *
+             * From exec(3):
+             *
+             * If permission is denied for a file (the attempted
+             * execve returned EACCES), these functions will continue
+             * searching the rest of the search path.  If no other
+             * file is found, however, they will return with the
+             * global variable errno set to EACCES.
+             */
+            switch (errno) {
+            case EACCES:
+                sticky_errno = errno;
+                /* FALLTHRU */
+            case ENOENT:
+            case ENOTDIR:
+#ifdef ELOOP
+            case ELOOP:
+#endif
+#ifdef ESTALE
+            case ESTALE:
+#endif
+#ifdef ENODEV
+            case ENODEV:
+#endif
+#ifdef ETIMEDOUT
+            case ETIMEDOUT:
+#endif
+                break; /* Try other directories in PATH */
+            default:
+                return;
+            }
+        }
+        if (sticky_errno != 0)
+            errno = sticky_errno;
+    }
+}
+
+/**
+ * Child process after a successful fork() or clone().
+ * This function must not return, and must be prepared for either all
+ * of its address space to be shared with its parent, or to be a copy.
+ * It must not modify global variables such as "environ".
+ */
+int
+childProcess(void *arg)
+{
+    const ChildStuff* p = (const ChildStuff*) arg;
+
+    /* Close the parent sides of the pipes.
+       Closing pipe fds here is redundant, since closeDescriptors()
+       would do it anyways, but a little paranoia is a good thing. */
+    if ((closeSafely(p->in[1])   == -1) ||
+        (closeSafely(p->out[0])  == -1) ||
+        (closeSafely(p->err[0])  == -1) ||
+        (closeSafely(p->childenv[0])  == -1) ||
+        (closeSafely(p->childenv[1])  == -1) ||
+        (closeSafely(p->fail[0]) == -1))
+        goto WhyCantJohnnyExec;
+
+    /* Give the child sides of the pipes the right fileno's. */
+    /* Note: it is possible for in[0] == 0 */
+    if ((moveDescriptor(p->in[0] != -1 ?  p->in[0] : p->fds[0],
+                        STDIN_FILENO) == -1) ||
+        (moveDescriptor(p->out[1]!= -1 ? p->out[1] : p->fds[1],
+                        STDOUT_FILENO) == -1))
+        goto WhyCantJohnnyExec;
+
+    if (p->redirectErrorStream) {
+        if ((closeSafely(p->err[1]) == -1) ||
+            (restartableDup2(STDOUT_FILENO, STDERR_FILENO) == -1))
+            goto WhyCantJohnnyExec;
+    } else {
+        if (moveDescriptor(p->err[1] != -1 ? p->err[1] : p->fds[2],
+                           STDERR_FILENO) == -1)
+            goto WhyCantJohnnyExec;
+    }
+
+    if (moveDescriptor(p->fail[1], FAIL_FILENO) == -1)
+        goto WhyCantJohnnyExec;
+
+    /* close everything */
+    if (closeDescriptors() == 0) { /* failed,  close the old way */
+        int max_fd = (int)sysconf(_SC_OPEN_MAX);
+        int fd;
+        for (fd = FAIL_FILENO + 1; fd < max_fd; fd++)
+            if (close(fd) == -1 && errno != EBADF)
+                goto WhyCantJohnnyExec;
+    }
+
+    /* change to the new working directory */
+    if (p->pdir != NULL && chdir(p->pdir) < 0)
+        goto WhyCantJohnnyExec;
+
+    if (fcntl(FAIL_FILENO, F_SETFD, FD_CLOEXEC) == -1)
+        goto WhyCantJohnnyExec;
+
+    JDK_execvpe(p->mode, p->argv[0], p->argv, p->envv);
+
+ WhyCantJohnnyExec:
+    /* We used to go to an awful lot of trouble to predict whether the
+     * child would fail, but there is no reliable way to predict the
+     * success of an operation without *trying* it, and there's no way
+     * to try a chdir or exec in the parent.  Instead, all we need is a
+     * way to communicate any failure back to the parent.  Easy; we just
+     * send the errno back to the parent over a pipe in case of failure.
+     * The tricky thing is, how do we communicate the *success* of exec?
+     * We use FD_CLOEXEC together with the fact that a read() on a pipe
+     * yields EOF when the write ends (we have two of them!) are closed.
+     */
+    {
+        int errnum = errno;
+        restartableWrite(FAIL_FILENO, &errnum, sizeof(errnum));
+    }
+    close(FAIL_FILENO);
+    _exit(-1);
+    return 0;  /* Suppress warning "no return value from function" */
+}

src/solaris/native/java/lang/childproc.h

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CHILDPROC_MD_H
+#define CHILDPROC_MD_H
+
+#include <sys/types.h>
+
+#ifdef __APPLE__
+#include <crt_externs.h>
+#define environ (*_NSGetEnviron())
+#else
+/* This is one of the rare times it's more portable to declare an
+ * external symbol explicitly, rather than via a system header.
+ * The declaration is standardized as part of UNIX98, but there is
+ * no standard (not even de-facto) header file where the
+ * declaration is to be found.  See:
+ * http://www.opengroup.org/onlinepubs/009695399/functions/environ.html
+ * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_02.html
+ *
+ * "All identifiers in this volume of IEEE Std 1003.1-2001, except
+ * environ, are defined in at least one of the headers" (!)
+ */
+extern char **environ;
+#endif
+
+#ifdef __linux__
+#include <sched.h>
+#endif
+
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+
+#ifndef STDERR_FILENO
+#define STDERR_FILENO 2
+#endif
+
+#ifndef SA_NOCLDSTOP
+#define SA_NOCLDSTOP 0
+#endif
+
+#ifndef SA_RESTART
+#define SA_RESTART 0
+#endif
+
+#define FAIL_FILENO (STDERR_FILENO + 1)
+
+/* TODO: Refactor. */
+#define RESTARTABLE(_cmd, _result) do { \
+  do { \
+    _result = _cmd; \
+  } while((_result == -1) && (errno == EINTR)); \
+} while(0)
+
+/* These numbers must be the same as the Enum in UNIXProcess.java
+ * Must be a better way of doing this.
+ */
+#define MODE_FORK 1
+#define MODE_POSIX_SPAWN 2
+#define MODE_VFORK 3
+#define MODE_CLONE 4
+
+typedef struct _ChildStuff
+{
+    int in[2];
+    int out[2];
+    int err[2];
+    int fail[2];
+    int childenv[2];
+    int fds[3];
+    int mode;
+    const char **argv;
+    int argc;
+    const char **envv;
+    const char *pdir;
+    int redirectErrorStream;
+    void *clone_stack;
+} ChildStuff;
+
+/* following used in addition when mode is SPAWN */
+typedef struct _SpawnInfo {
+    int nargv; /* number of argv array elements  */
+    int argvBytes; /* total number of bytes in argv array */
+    int nenvv; /* number of envv array elements  */
+    int envvBytes; /* total number of bytes in envv array */
+    int dirlen; /* length of home directory string */
+    int nparentPathv; /* number of elements in parentPathv array */
+    int parentPathvBytes; /* total number of bytes in parentPathv array */
+} SpawnInfo;
+
+/**
+ * The cached and split version of the JDK's effective PATH.
+ * (We don't support putenv("PATH=...") in native code)
+ */
+const char * const *parentPathv;
+
+ssize_t restartableWrite(int fd, const void *buf, size_t count);
+int restartableDup2(int fd_from, int fd_to);
+int closeSafely(int fd);
+int isAsciiDigit(char c);
+int closeDescriptors(void);
+int moveDescriptor(int fd_from, int fd_to);
+
+int magicNumber();
+ssize_t readFully(int fd, void *buf, size_t nbyte);
+void initVectorFromBlock(const char**vector, const char* block, int count);
+void execve_as_traditional_shell_script(const char *file,
+                                        const char *argv[],
+                                        const char *const envp[]);
+void execve_with_shell_fallback(int mode, const char *file,
+                                const char *argv[],
+                                const char *const envp[]);
+void JDK_execvpe(int mode, const char *file,
+                 const char *argv[],
+                 const char *const envp[]);
+int childProcess(void *arg);
+
+#endif

src/solaris/native/java/lang/jspawnhelper.c

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "childproc.h"
+
+extern int errno;
+
+#define ALLOC(X,Y) { \
+    void *mptr; \
+    mptr = malloc (Y); \
+    if (mptr == 0) { \
+        error (fdout, ERR_MALLOC); \
+    } \
+    X = mptr; \
+}
+
+#define ERR_MALLOC 1
+#define ERR_PIPE 2
+#define ERR_ARGS 3
+
+void error (int fd, int err) {
+    write (fd, &err, sizeof(err));
+    exit (1);
+}
+
+void shutItDown() {
+    fprintf(stdout, "This command is not for general use and should ");
+    fprintf(stdout, "only be run as the result of a call to\n");
+    fprintf(stdout, "ProcessBuilder.start() or Runtime.exec() in a java ");
+    fprintf(stdout, "application\n");
+    _exit(1);
+}
+
+/*
+ * read the following off the pipefd
+ * - the ChildStuff struct
+ * - the SpawnInfo struct
+ * - the data strings for fields in ChildStuff
+ */
+void initChildStuff (int fdin, int fdout, ChildStuff *c) {
+    int n;
+    int argvBytes, nargv, envvBytes, nenvv;
+    int dirlen;
+    char *buf;
+    SpawnInfo sp;
+    int bufsize, offset=0;
+    int magic;
+    int res;
+
+    res = readFully (fdin, &magic, sizeof(magic));
+    if (res != 4 || magic != magicNumber()) {
+        error (fdout, ERR_PIPE);
+    }
+
+    if (readFully (fdin, c, sizeof(*c)) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    if (readFully (fdin, &sp, sizeof(sp)) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    bufsize = sp.argvBytes + sp.envvBytes +
+              sp.dirlen + sp.parentPathvBytes;
+
+    ALLOC(buf, bufsize);
+
+    if (readFully (fdin, buf, bufsize) == -1) {
+        error (fdout, ERR_PIPE);
+    }
+
+    /* Initialize argv[] */
+    ALLOC(c->argv, sizeof(char *) * sp.nargv);
+    initVectorFromBlock (c->argv, buf+offset, sp.nargv-1);
+    offset += sp.argvBytes;
+
+    /* Initialize envv[] */
+    if (sp.nenvv == 0) {
+        c->envv = 0;
+    } else {
+        ALLOC(c->envv, sizeof(char *) * sp.nenvv);
+        initVectorFromBlock (c->envv, buf+offset, sp.nenvv-1);
+        offset += sp.envvBytes;
+    }
+
+    /* Initialize pdir */
+    if (sp.dirlen == 0) {
+        c->pdir = 0;
+    } else {
+        c->pdir = buf+offset;
+        offset += sp.dirlen;
+    }
+
+    /* Initialize parentPathv[] */
+    ALLOC(parentPathv, sizeof (char *) * sp.nparentPathv)
+    initVectorFromBlock ((const char**)parentPathv, buf+offset, sp.nparentPathv-1);
+    offset += sp.parentPathvBytes;
+}
+
+int main(int argc, char *argv[]) {
+    ChildStuff c;
+    int t;
+    struct stat buf;
+    /* argv[0] contains the fd number to read all the child info */
+    int r, fdin, fdout;
+
+    r = sscanf (argv[argc-1], "%d:%d", &fdin, &fdout);
+    if (r == 2 && fcntl(fdin, F_GETFD) != -1) {
+        fstat(fdin, &buf);
+        if (!S_ISFIFO(buf.st_mode))
+            shutItDown();
+    } else {
+        shutItDown();
+    }
+    initChildStuff (fdin, fdout, &c);
+
+    childProcess (&c);
+    return 0; /* NOT REACHED */
+}

test/java/lang/ProcessBuilder/Basic.java

@@ -26,9 +26,10 @@
  * @bug 4199068 4738465 4937983 4930681 4926230 4931433 4932663 4986689
  *      5026830 5023243 5070673 4052517 4811767 6192449 6397034 6413313
  *      6464154 6523983 6206031 4960438 6631352 6631966 6850957 6850958
- *      4947220 7018606 7034570 4244896
+ *      4947220 7018606 7034570 4244896 5049299
  * @summary Basic tests for Process and Environment Variable code
  * @run main/othervm/timeout=300 Basic
+ * @run main/othervm/timeout=300 -Djdk.lang.Process.launchMechanism=fork Basic
  * @author Martin Buchholz
  */