8078470: [Linux] Replace syscall use in os::fork_and_exec with glibc fork() and execve()

Reviewed-by: stuefe, dsamersoff, dcubed

src/os/linux/vm/os_linux.cpp

@@ -5929,14 +5929,6 @@ void Parker::unpark() {
 
 extern char** environ;
 
-#ifndef __NR_fork
-#define __NR_fork IA32_ONLY(2) IA64_ONLY(not defined) AMD64_ONLY(57)
-#endif
-
-#ifndef __NR_execve
-#define __NR_execve IA32_ONLY(11) IA64_ONLY(1033) AMD64_ONLY(59)
-#endif
-
 // Run the specified command in a separate process. Return its exit value,
 // or -1 on failure (e.g. can't fork a new process).
 // Unlike system(), this function can be called from signal handler. It
@@ -5944,13 +5936,7 @@ extern char** environ;
 int os::fork_and_exec(char* cmd) {
   const char * argv[4] = {"sh", "-c", cmd, NULL};
 
-  // fork() in LinuxThreads/NPTL is not async-safe. It needs to run
-  // pthread_atfork handlers and reset pthread library. All we need is a
-  // separate process to execve. Make a direct syscall to fork process.
-  // On IA64 there's no fork syscall, we have to use fork() and hope for
-  // the best...
-  pid_t pid = NOT_IA64(syscall(__NR_fork);)
-              IA64_ONLY(fork();)
+  pid_t pid = fork();
 
   if (pid < 0) {
     // fork failed
@@ -5959,15 +5945,7 @@ int os::fork_and_exec(char* cmd) {
   } else if (pid == 0) {
     // child process
 
-    // execve() in LinuxThreads will call pthread_kill_other_threads_np()
-    // first to kill every thread on the thread list. Because this list is
-    // not reset by fork() (see notes above), execve() will instead kill
-    // every thread in the parent process. We know this is the only thread
-    // in the new process, so make a system call directly.
-    // IA64 should use normal execve() from glibc to match the glibc fork()
-    // above.
-    NOT_IA64(syscall(__NR_execve, "/bin/sh", argv, environ);)
-    IA64_ONLY(execve("/bin/sh", (char* const*)argv, environ);)
+    execve("/bin/sh", (char* const*)argv, environ);
 
     // execve failed
     _exit(-1);

src/share/vm/utilities/vmError.cpp

 /*
- * Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1013,7 +1013,9 @@ void VMError::report_and_die() {
       out.print_raw   (cmd);
       out.print_raw_cr("\" ...");
 
-      os::fork_and_exec(cmd);
+      if (os::fork_and_exec(cmd) < 0) {
+        out.print_cr("os::fork_and_exec failed: %s (%d)", strerror(errno), errno);
+      }
     }
 
     // done with OnError
@@ -1098,7 +1100,9 @@ void VM_ReportJavaOutOfMemory::doit() {
 #endif
     tty->print_cr("\"%s\"...", cmd);
 
-    os::fork_and_exec(cmd);
+    if (os::fork_and_exec(cmd) < 0) {
+      tty->print_cr("os::fork_and_exec failed: %s (%d)", strerror(errno), errno);
+    }
   }
 }
 

test/runtime/ErrorHandling/TestOnError.java

+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestOnError
+ * @summary Test using -XX:OnError=<cmd>
+ * @library /testlibrary
+ * @build TestOnError com.oracle.java.testlibrary.*
+ * @run main TestOnError
+ * @bug 8078470
+ */
+
+import com.oracle.java.testlibrary.*;
+
+public class TestOnError {
+
+    public static void main(String[] args) throws Exception {
+        if (!Platform.isDebugBuild()) {
+            System.out.println("Test requires a non-product build - skipping");
+            return;
+        }
+
+        String msg = "Test Succeeded";
+
+        // Execute the VM so that a
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+           "-XX:-TransmitErrorReport",
+           "-XX:ErrorHandlerTest=12", // trigger potential SEGV
+           "-XX:OnError=echo " + msg,
+           TestOnError.class.getName());
+
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+        /* Actual output will include:
+           #
+           # -XX:OnError="echo Test Succeeded"
+           #   Executing /bin/sh -c "echo Test Succeeded"...
+           Test Succeeded
+
+           So we don't want to match on the "# Executing ..." line, and they
+           both get written to stdout.
+        */
+        output.stdoutShouldMatch("^" + msg); // match start of line only
+        System.out.println("PASSED");
+    }
+}

test/runtime/ErrorHandling/TestOnOutOfMemoryError.java

+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestOnOutOfMemoryError
+ * @summary Test using -XX:OnOutOfMemoryError=<cmd>
+ * @library /testlibrary
+ * @build TestOnOutOfMemoryError com.oracle.java.testlibrary.*
+ * @run main TestOnOutOfMemoryError
+ * @bug 8078470
+ */
+
+import com.oracle.java.testlibrary.*;
+
+public class TestOnOutOfMemoryError {
+
+    public static void main(String[] args) throws Exception {
+        if (args.length == 1) {
+            // This should guarantee to throw:
+            //  java.lang.OutOfMemoryError: Requested array size exceeds VM limit
+            Object[] oa = new Object[Integer.MAX_VALUE];
+            return;
+        }
+
+        // else this is the main test
+        String msg = "Test Succeeded";
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+           "-XX:OnOutOfMemoryError=echo " + msg,
+           TestOnOutOfMemoryError.class.getName(),
+           "throwOOME");
+
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+        /* Actual output should look like this:
+           #
+           # java.lang.OutOfMemoryError: Requested array size exceeds VM limit
+           # -XX:OnOutOfMemoryError="echo Test Succeeded"
+           #   Executing /bin/sh -c "echo Test Succeeded"...
+           Test Succeeded
+           Exception in thread "main" java.lang.OutOfMemoryError: Requested array size exceeds VM limit
+           at OOME.main(OOME.java:3)
+
+           So we don't want to match on the "# Executing ..." line, and they
+           both get written to stdout.
+        */
+        output.shouldContain("Requested array size exceeds VM limit");
+        output.stdoutShouldMatch("^" + msg); // match start of line only
+        System.out.println("PASSED");
+    }
+}