8064667: Add -XX:+CheckEndorsedAndExtDirs flag to JDK 8

Reviewed-by: coleenp, ccheung

src/share/vm/runtime/arguments.cpp

@@ -2962,6 +2962,23 @@ jint Arguments::parse_each_vm_init_arg(const JavaVMInitArgs* args,
 #endif
     // -D
     } else if (match_option(option, "-D", &tail)) {
+      if (CheckEndorsedAndExtDirs) {
+        if (match_option(option, "-Djava.endorsed.dirs=", &tail)) {
+          // abort if -Djava.endorsed.dirs is set
+          jio_fprintf(defaultStream::output_stream(),
+            "-Djava.endorsed.dirs will not be supported in a future release.\n"
+            "Refer to JEP 220 for details (http://openjdk.java.net/jeps/220).\n");
+          return JNI_EINVAL;
+        }
+        if (match_option(option, "-Djava.ext.dirs=", &tail)) {
+          // abort if -Djava.ext.dirs is set
+          jio_fprintf(defaultStream::output_stream(),
+            "-Djava.ext.dirs will not be supported in a future release.\n"
+            "Refer to JEP 220 for details (http://openjdk.java.net/jeps/220).\n");
+          return JNI_EINVAL;
+        }
+      }
+
       if (!add_property(tail)) {
         return JNI_ENOMEM;
       }
@@ -3395,6 +3412,146 @@ void Arguments::fix_appclasspath() {
   }
 }
 
+static bool has_jar_files(const char* directory) {
+  DIR* dir = os::opendir(directory);
+  if (dir == NULL) return false;
+
+  struct dirent *entry;
+  char *dbuf = NEW_C_HEAP_ARRAY(char, os::readdir_buf_size(directory), mtInternal);
+  bool hasJarFile = false;
+  while (!hasJarFile && (entry = os::readdir(dir, (dirent *) dbuf)) != NULL) {
+    const char* name = entry->d_name;
+    const char* ext = name + strlen(name) - 4;
+    hasJarFile = ext > name && (os::file_name_strcmp(ext, ".jar") == 0);
+  }
+  FREE_C_HEAP_ARRAY(char, dbuf, mtInternal);
+  os::closedir(dir);
+  return hasJarFile ;
+}
+
+// returns the number of directories in the given path containing JAR files
+// If the skip argument is not NULL, it will skip that directory
+static int check_non_empty_dirs(const char* path, const char* type, const char* skip) {
+  const char separator = *os::path_separator();
+  const char* const end = path + strlen(path);
+  int nonEmptyDirs = 0;
+  while (path < end) {
+    const char* tmp_end = strchr(path, separator);
+    if (tmp_end == NULL) {
+      if ((skip == NULL || strcmp(path, skip) != 0) && has_jar_files(path)) {
+        nonEmptyDirs++;
+        jio_fprintf(defaultStream::output_stream(),
+          "Non-empty %s directory: %s\n", type, path);
+      }
+      path = end;
+    } else {
+      char* dirpath = NEW_C_HEAP_ARRAY(char, tmp_end - path + 1, mtInternal);
+      memcpy(dirpath, path, tmp_end - path);
+      dirpath[tmp_end - path] = '\0';
+      if ((skip == NULL || strcmp(dirpath, skip) != 0) && has_jar_files(dirpath)) {
+        nonEmptyDirs++;
+        jio_fprintf(defaultStream::output_stream(),
+          "Non-empty %s directory: %s\n", type, dirpath);
+      }
+      FREE_C_HEAP_ARRAY(char, dirpath, mtInternal);
+      path = tmp_end + 1;
+    }
+  }
+  return nonEmptyDirs;
+}
+
+// Returns true if endorsed standards override mechanism and extension mechanism
+// are not used.
+static bool check_endorsed_and_ext_dirs() {
+  if (!CheckEndorsedAndExtDirs)
+    return true;
+
+  char endorsedDir[JVM_MAXPATHLEN];
+  char extDir[JVM_MAXPATHLEN];
+  const char* fileSep = os::file_separator();
+  jio_snprintf(endorsedDir, sizeof(endorsedDir), "%s%slib%sendorsed",
+               Arguments::get_java_home(), fileSep, fileSep);
+  jio_snprintf(extDir, sizeof(extDir), "%s%slib%sext",
+               Arguments::get_java_home(), fileSep, fileSep);
+
+  // check endorsed directory
+  int nonEmptyDirs = check_non_empty_dirs(Arguments::get_endorsed_dir(), "endorsed", NULL);
+
+  // check the extension directories but skip the default lib/ext directory
+  nonEmptyDirs += check_non_empty_dirs(Arguments::get_ext_dirs(), "extension", extDir);
+
+  // List of JAR files installed in the default lib/ext directory.
+  // -XX:+CheckEndorsedAndExtDirs checks if any non-JDK file installed
+  static const char* jdk_ext_jars[] = {
+      "access-bridge-32.jar",
+      "access-bridge-64.jar",
+      "access-bridge.jar",
+      "cldrdata.jar",
+      "dnsns.jar",
+      "jaccess.jar",
+      "jfxrt.jar",
+      "localedata.jar",
+      "nashorn.jar",
+      "sunec.jar",
+      "sunjce_provider.jar",
+      "sunmscapi.jar",
+      "sunpkcs11.jar",
+      "ucrypto.jar",
+      "zipfs.jar",
+      NULL
+  };
+
+  // check if the default lib/ext directory has any non-JDK jar files; if so, error
+  DIR* dir = os::opendir(extDir);
+  if (dir != NULL) {
+    int num_ext_jars = 0;
+    struct dirent *entry;
+    char *dbuf = NEW_C_HEAP_ARRAY(char, os::readdir_buf_size(extDir), mtInternal);
+    while ((entry = os::readdir(dir, (dirent *) dbuf)) != NULL) {
+      const char* name = entry->d_name;
+      const char* ext = name + strlen(name) - 4;
+      if (ext > name && (os::file_name_strcmp(ext, ".jar") == 0)) {
+        bool is_jdk_jar = false;
+        const char* jarfile = NULL;
+        for (int i=0; (jarfile = jdk_ext_jars[i]) != NULL; i++) {
+          if (os::file_name_strcmp(name, jarfile) == 0) {
+            is_jdk_jar = true;
+            break;
+          }
+        }
+        if (!is_jdk_jar) {
+          jio_fprintf(defaultStream::output_stream(),
+            "%s installed in <JAVA_HOME>/lib/ext\n", name);
+          num_ext_jars++;
+        }
+      }
+    }
+    FREE_C_HEAP_ARRAY(char, dbuf, mtInternal);
+    os::closedir(dir);
+    if (num_ext_jars > 0) {
+      nonEmptyDirs += 1;
+    }
+  }
+
+  // check if the default lib/endorsed directory exists; if so, error
+  dir = os::opendir(endorsedDir);
+  if (dir != NULL) {
+    jio_fprintf(defaultStream::output_stream(), "<JAVA_HOME>/lib/endorsed exists\n");
+    os::closedir(dir);
+    nonEmptyDirs += 1;
+  }
+
+  if (nonEmptyDirs > 0) {
+    jio_fprintf(defaultStream::output_stream(),
+      "Endorsed standards override mechanism and extension mechanism"
+      "will not be supported in a future release.\n"
+      "Refer to JEP 220 for details (http://openjdk.java.net/jeps/220).\n");
+    return false;
+  }
+
+  return true;
+}
+
 jint Arguments::finalize_vm_init_args(SysClassPath* scp_p, bool scp_assembly_required) {
   // This must be done after all -D arguments have been processed.
   scp_p->expand_endorsed();
@@ -3404,6 +3561,10 @@ jint Arguments::finalize_vm_init_args(SysClassPath* scp_p, bool scp_assembly_req
     Arguments::set_sysclasspath(scp_p->combined_path());
   }
 
+  if (!check_endorsed_and_ext_dirs()) {
+    return JNI_ERR;
+  }
+
   // This must be done after all arguments have been processed.
   // java_compiler() true means set to "NONE" or empty.
   if (java_compiler() && !xdebug_mode()) {

src/share/vm/runtime/globals.hpp

@@ -1210,6 +1210,9 @@ class CommandLineFlags {
   product(bool, CheckJNICalls, false,                                       \
           "Verify all arguments to JNI calls")                              \
                                                                             \
+  product(bool, CheckEndorsedAndExtDirs, false,                             \
+          "Verify the endorsed and extension directories are not used")     \
+                                                                            \
   product(bool, UseFastJNIAccessors, true,                                  \
           "Use optimized versions of Get<Primitive>Field")                  \
                                                                             \

test/runtime/CheckEndorsedAndExtDirs/EndorsedExtDirs.java

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8064667
+ * @summary Sanity test for -XX:+CheckEndorsedAndExtDirs
+ * @library /testlibrary
+ * @run main/othervm -XX:+CheckEndorsedAndExtDirs EndorsedExtDirs
+ */
+
+import com.oracle.java.testlibrary.*;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EndorsedExtDirs {
+    static final String cpath = System.getProperty("test.classes", ".");
+    public static void main(String arg[]) throws Exception {
+        fatalError("-XX:+CheckEndorsedAndExtDirs", "-Djava.endorsed.dirs=foo");
+        fatalError("-XX:+CheckEndorsedAndExtDirs", "-Djava.ext.dirs=bar");
+    }
+
+    static void fatalError(String... args) throws Exception {
+        List<String> commands = new ArrayList<>();
+        String java = System.getProperty("java.home") + "/bin/java";
+        commands.add(java);
+        for (String s : args) {
+            commands.add(s);
+        }
+        commands.add("-cp");
+        commands.add(cpath);
+        commands.add("EndorsedExtDirs");
+
+        System.out.println("Launching " + commands);
+        ProcessBuilder pb = new ProcessBuilder(commands);
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+        output.shouldContain("Could not create the Java Virtual Machine");
+        output.shouldHaveExitValue(1);
+    }
+}