8066185: VM crashed with SIGSEGV VirtualMemoryTracker::add_reserved_region

Reviewed-by: coleenp, dholmes

src/share/bin/java.c

@@ -650,9 +650,26 @@ static void
 SetJvmEnvironment(int argc, char **argv) {
 
     static const char*  NMT_Env_Name    = "NMT_LEVEL_";
-
     int i;
     for (i = 0; i < argc; i++) {
+        char *arg = argv[i];
+        /*
+         * Since this must be a VM flag we stop processing once we see
+         * an argument the launcher would not have processed beyond (such
+         * as -version or -h), or an argument that indicates the following
+         * arguments are for the application (i.e. the main class name, or
+         * the -jar argument).
+         */
+        if ((i > 0 && *arg != '-')
+                || JLI_StrCmp(arg, "-version") == 0
+                || JLI_StrCmp(arg, "-fullversion") == 0
+                || JLI_StrCmp(arg, "-help") == 0
+                || JLI_StrCmp(arg, "-?") == 0
+                || JLI_StrCmp(arg, "-jar") == 0
+                || JLI_StrCmp(arg, "-X") == 0
+                ) {
+            return;
+        }
         /*
          * The following case checks for "-XX:NativeMemoryTracking=value".
          * If value is non null, an environmental variable set to this value
@@ -660,7 +677,6 @@ SetJvmEnvironment(int argc, char **argv) {
          * The argument is passed to the JVM, which will check validity.
          * The JVM is responsible for removing the env variable.
          */
-        char *arg = argv[i];
         if (JLI_StrCCmp(arg, "-XX:NativeMemoryTracking=") == 0) {
             int retval;
             // get what follows this parameter, include "="

test/tools/launcher/TestSpecialArgs.java

 /*
- * Copyright (c) 2012, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,14 @@
 
 /*
  * @test
- * @bug 7124089 7131021 8042469
- * @summary Checks for MacOSX specific flags are accepted or rejected, and
- *          MacOSX platforms specific environment is consistent.
+ * @bug 7124089 7131021 8042469 8066185
+ * @summary Checks for Launcher special flags, such as MacOSX specific flags,
+ *          and JVM NativeMemoryTracking flags.
  * @compile -XDignore.symbol.file TestSpecialArgs.java EnvironmentVariables.java
  * @run main TestSpecialArgs
  */
+import java.io.File;
+import java.io.FileNotFoundException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -36,10 +38,14 @@ import java.util.Set;
 
 public class TestSpecialArgs extends TestHelper {
 
-    public static void main(String... args) {
+    public static void main(String... args) throws Exception {
+        new TestSpecialArgs().run(args);
+    }
+
+    @Test
+    void testDocking() {
         final Map<String, String> envMap = new HashMap<>();
         envMap.put("_JAVA_LAUNCHER_DEBUG", "true");
-
         TestResult tr = doExec(envMap, javaCmd, "-XstartOnFirstThread", "-version");
         if (isMacOSX) {
             if (!tr.contains("In same thread")) {
@@ -69,7 +75,45 @@ public class TestSpecialArgs extends TestHelper {
                 throw new RuntimeException("Error: argument was accepted ????");
             }
         }
+        // MacOSX specific tests ensue......
+        if (!isMacOSX) {
+            return;
+        }
+        Set<String> envToRemove = new HashSet<>();
+        Map<String, String> map = System.getenv();
+        for (String s : map.keySet()) {
+            if (s.startsWith("JAVA_MAIN_CLASS_")
+                    || s.startsWith("APP_NAME_")
+                    || s.startsWith("APP_ICON_")) {
+                envToRemove.add(s);
+            }
+        }
+        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
+                "EnvironmentVariables", "JAVA_MAIN_CLASS_*",
+                "EnvironmentVariables");
+
+        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
+                "-Xdock:name=TestAppName", "EnvironmentVariables",
+                "APP_NAME_*", "TestAppName");
+
+        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
+                "-Xdock:icon=TestAppIcon", "EnvironmentVariables",
+                "APP_ICON_*", "TestAppIcon");
+    }
+
+    void runTest(Set<String> envToRemove, String... args) {
+        TestResult tr = doExec(null, envToRemove, args);
+        if (!tr.isOK()) {
+            System.err.println(tr.toString());
+            throw new RuntimeException("Test Fails");
+        }
+    }
 
+    @Test
+    void testNativeMemoryTracking() {
+        final Map<String, String> envMap = new HashMap<>();
+        envMap.put("_JAVA_LAUNCHER_DEBUG", "true");
+        TestResult tr;
         /*
          * test argument : -XX:NativeMemoryTracking=value
          * A JVM flag, comsumed by the JVM, but requiring launcher
@@ -86,147 +130,155 @@ public class TestSpecialArgs extends TestHelper {
          *       Code to create env variable not executed.
          * 4) give and invalid value and check to make sure JVM commented
          */
-        { // NativeMemoryTracking
-            String launcherPidString = "launcher.pid=";
-            String envVarPidString = "TRACER_MARKER: NativeMemoryTracking: env var is NMT_LEVEL_";
-            String NMT_Option_Value = "off";
-            String myClassName = "helloworld";
-            boolean haveLauncherPid = false;
+        String launcherPidString = "launcher.pid=";
+        String envVarPidString = "TRACER_MARKER: NativeMemoryTracking: env var is NMT_LEVEL_";
+        String NMT_Option_Value = "off";
+        String myClassName = "helloworld";
+        boolean haveLauncherPid = false;
 
-            // === Run the tests ===
+        // === Run the tests ===
+        // ---Test 1a
+        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=" + NMT_Option_Value,
+                "-version");
 
-            // ---Test 1a
-            tr = doExec(envMap,javaCmd, "-XX:NativeMemoryTracking=" + NMT_Option_Value,
-                        "-version");
+        // get the PID from the env var we set for the JVM
+        String envVarPid = null;
+        for (String line : tr.testOutput) {
+            if (line.contains(envVarPidString)) {
+                int sindex = envVarPidString.length();
+                envVarPid = line.substring(sindex);
+                break;
+            }
+        }
+        // did we find envVarPid?
+        if (envVarPid == null) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: failed to find env Var Pid in tracking info");
+        }
+        // we think we found the pid string.  min test, not "".
+        if (envVarPid.length() < 1) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: env Var Pid in tracking info is empty string");
+        }
 
-            // get the PID from the env var we set for the JVM
-            String envVarPid = null;
+        /*
+         * On Linux, Launcher Tracking will print the PID.  Use this info
+         * to validate what we got as the PID in the Launcher itself.
+         * Linux is the only one that prints this, and trying to get it
+         * here for win is awful.  So let the linux test make sure we get
+         * the valid pid, and for non-linux, just make sure pid string is
+         * non-zero.
+         */
+        if (isLinux) {
+            // get what the test says is the launcher pid
+            String launcherPid = null;
             for (String line : tr.testOutput) {
-                if (line.contains(envVarPidString)) {
-                    int sindex = envVarPidString.length();
-                    envVarPid = line.substring(sindex);
+                int index = line.indexOf(launcherPidString);
+                if (index >= 0) {
+                    int sindex = index + launcherPidString.length();
+                    int tindex = sindex + line.substring(sindex).indexOf("'");
+                    System.out.println("DEBUG INFO: sindex = " + sindex);
+                    System.out.println("DEBUG INFO: searching substring: " + line.substring(sindex));
+                    System.out.println("DEBUG INFO: tindex = " + tindex);
+                    // DEBUG INFO
+                    System.out.println(tr);
+                    launcherPid = line.substring(sindex, tindex);
                     break;
                 }
             }
-            // did we find envVarPid?
-            if (envVarPid == null) {
+            if (launcherPid == null) {
                 System.out.println(tr);
-                throw new RuntimeException("Error: failed to find env Var Pid in tracking info");
+                throw new RuntimeException("Error: failed to find launcher Pid in launcher tracking info");
             }
-            // we think we found the pid string.  min test, not "".
-            if (envVarPid.length() < 1) {
+
+            // did we create the env var with the correct pid?
+            if (!launcherPid.equals(envVarPid)) {
                 System.out.println(tr);
-                throw new RuntimeException("Error: env Var Pid in tracking info is empty string");
+                System.out.println("Error: wrong pid in creating env var");
+                System.out.println("Error Info: launcherPid = " + launcherPid);
+                System.out.println("Error Info: envVarPid   = " + envVarPid);
+                throw new RuntimeException("Error: wrong pid in creating env var");
             }
+        }
 
-            /*
-             * On Linux, Launcher Tracking will print the PID.  Use this info
-             * to validate what we got as the PID in the Launcher itself.
-             * Linux is the only one that prints this, and trying to get it
-             * here for win is awful.  So let the linux test make sure we get
-             * the valid pid, and for non-linux, just make sure pid string is
-             * non-zero.
-             */
-            if (isLinux) {
-                // get what the test says is the launcher pid
-                String launcherPid = null;
-                for (String line : tr.testOutput) {
-                    int index = line.indexOf(launcherPidString);
-                    if (index >= 0) {
-                        int sindex = index + launcherPidString.length();
-                        int tindex = sindex + line.substring(sindex).indexOf("'");
-                        System.out.println("DEBUG INFO: sindex = " + sindex);
-                        System.out.println("DEBUG INFO: searching substring: " + line.substring(sindex));
-                        System.out.println("DEBUG INFO: tindex = " + tindex);
-                        // DEBUG INFO
-                        System.out.println(tr);
-                        launcherPid = line.substring(sindex, tindex);
-                        break;
-                    }
-                }
-                if (launcherPid == null) {
-                    System.out.println(tr);
-                    throw new RuntimeException("Error: failed to find launcher Pid in launcher tracking info");
-                }
+        // --- Test 1b
+        if (!tr.contains("NativeMemoryTracking: got value " + NMT_Option_Value)) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: Valid param failed to set env variable");
+        }
 
-                // did we create the env var with the correct pid?
-                if (!launcherPid.equals(envVarPid)) {
-                    System.out.println(tr);
-                    System.out.println("Error: wrong pid in creating env var");
-                    System.out.println("Error Info: launcherPid = " + launcherPid);
-                    System.out.println("Error Info: envVarPid   = " + envVarPid);
-                    throw new RuntimeException("Error: wrong pid in creating env var");
-                }
-            }
+        // --- Test 2
+        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=",
+                "-version");
+        if (tr.contains("NativeMemoryTracking:")) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
+        }
+        if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: invalid param not checked by JVM");
+        }
 
+        // --- Test 3
+        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking",
+                "-version");
+        if (tr.contains("NativeMemoryTracking:")) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
+        }
+        if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: invalid param not checked by JVM");
+        }
+        // --- Test 4
+        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=BADVALUE",
+                "-version");
+        if (!tr.contains("expecting -XX:NativeMemoryTracking")) {
+            System.out.println(tr);
+            throw new RuntimeException("Error: invalid param did not get JVM Syntax error message");
+        }
+    }
 
-            // --- Test 1b
-            if (!tr.contains("NativeMemoryTracking: got value " + NMT_Option_Value)) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: Valid param failed to set env variable");
-            }
+    @Test
+    void testNMArgumentProcessing() throws FileNotFoundException {
+        TestResult tr = null;
+        // the direct invokers of the VM
+        String options[] = {
+            "-version", "-fullversion", "-help", "-?", "-X"
+        };
+        for (String option : options) {
+            tr = doExec(javaCmd, option, "-XX:NativeMemoryTracking=summary");
+            checkTestResult(tr);
+        }
 
-            // --- Test 2
-            tr = doExec(envMap,javaCmd, "-XX:NativeMemoryTracking=",
-                        "-version");
-            if (tr.contains("NativeMemoryTracking:")) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
-            }
-            if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: invalid param not checked by JVM");
-            }
+        // create a test jar
+        File jarFile = new File("test.jar");
+        createJar(jarFile, "public static void main(String... args){}");
 
-            // --- Test 3
-            tr = doExec(envMap,javaCmd, "-XX:NativeMemoryTracking",
-                        "-version");
-            if (tr.contains("NativeMemoryTracking:")) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
-            }
-            if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: invalid param not checked by JVM");
-            }
-            // --- Test 4
-            tr = doExec(envMap,javaCmd, "-XX:NativeMemoryTracking=BADVALUE",
-                        "-version");
-            if (!tr.contains("expecting -XX:NativeMemoryTracking")) {
-                System.out.println(tr);
-                throw new RuntimeException("Error: invalid param did not get JVM Syntax error message");
-            }
+        // ones that involve main-class of some sort
+        tr = doExec(javaCmd, "-jar", jarFile.getName(),
+                "-XX:NativeMemoryTracking=summary");
+        checkTestResult(tr);
 
-        } // NativeMemoryTracking
+        tr = doExec(javaCmd, "-cp", jarFile.getName(), "Foo",
+                "-XX:NativeMemoryTracking=summary");
+        checkTestResult(tr);
 
+        final Map<String, String> envMap = new HashMap<>();
+        // checkwith CLASSPATH set ie. no -cp or -classpath
+        envMap.put("CLASSPATH", ".");
+        tr = doExec(envMap, javaCmd, "Foo", "-XX:NativeMemoryTracking=summary");
+        checkTestResult(tr);
 
-        // MacOSX specific tests ensue......
-        if (!isMacOSX)
-            return;
-        Set<String> envToRemove = new HashSet<>();
-        Map<String, String> map = System.getenv();
-        for (String s : map.keySet()) {
-            if (s.startsWith("JAVA_MAIN_CLASS_")
-                    || s.startsWith("APP_NAME_")
-                    || s.startsWith("APP_ICON_")) {
-                envToRemove.add(s);
-            }
+        // make sure a missing class is handled correctly, because the class
+        // resolution is performed by the JVM.
+        tr = doExec(javaCmd, "AbsentClass", "-XX:NativeMemoryTracking=summary");
+        if (!tr.contains("Error: Could not find or load main class AbsentClass")) {
+            throw new RuntimeException("Test Fails");
         }
-        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
-                "EnvironmentVariables", "JAVA_MAIN_CLASS_*",
-                "EnvironmentVariables");
-
-        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
-                "-Xdock:name=TestAppName", "EnvironmentVariables",
-                "APP_NAME_*", "TestAppName");
-
-        runTest(envToRemove, javaCmd, "-cp", TEST_CLASSES_DIR.getAbsolutePath(),
-                "-Xdock:icon=TestAppIcon", "EnvironmentVariables",
-                "APP_ICON_*", "TestAppIcon");
     }
 
-    static void runTest(Set<String> envToRemove, String... args) {
-        TestResult tr = doExec(null, envToRemove, args);
+    void checkTestResult(TestResult tr) {
         if (!tr.isOK()) {
             System.err.println(tr.toString());
             throw new RuntimeException("Test Fails");