8151486: Class.forName causes memory leak

Summary: Remove ClassLoader.domains
Reviewed-by: coleenp, dholmes, mchung, naoto

src/share/classes/java/lang/ClassLoader.java

@@ -255,9 +255,6 @@ public abstract class ClassLoader {
         new ProtectionDomain(new CodeSource(null, (Certificate[]) null),
                              null, this, null);
 
-    // The initiating protection domains for all classes loaded by this loader
-    private final Set<ProtectionDomain> domains;
-
     // Invoked by the VM to record every loaded class with this loader.
     void addClass(Class<?> c) {
         classes.addElement(c);
@@ -281,14 +278,11 @@ public abstract class ClassLoader {
         if (ParallelLoaders.isRegistered(this.getClass())) {
             parallelLockMap = new ConcurrentHashMap<>();
             package2certs = new ConcurrentHashMap<>();
-            domains =
-                Collections.synchronizedSet(new HashSet<ProtectionDomain>());
             assertionLock = new Object();
         } else {
             // no finer-grained lock; lock on the classloader instance
             parallelLockMap = null;
             package2certs = new Hashtable<>();
-            domains = new HashSet<>();
             assertionLock = this;
         }
     }
@@ -505,7 +499,6 @@ public abstract class ClassLoader {
                 }, new AccessControlContext(new ProtectionDomain[] {pd}));
             }
         }
-        domains.add(pd);
     }
 
     /**

test/java/lang/ClassLoader/forNameLeak/ClassForName.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.net.URLClassLoader;
+
+/*
+ * This class is loaded by the custom URLClassLoader, and then calls
+ * Class.forName();
+ */
+public class ClassForName implements Runnable {
+    static {
+        if (!(ClassForName.class.getClassLoader() instanceof URLClassLoader)) {
+            throw new RuntimeException("Supposed to be loaded by URLClassLoader");
+        }
+    }
+
+    public void run() {
+        try {
+            Class.forName(java.util.List.class.getName(), false,
+                          ClassLoader.getSystemClassLoader());
+        } catch (Throwable e) {
+            e.printStackTrace();
+        }
+    }
+}

test/java/lang/ClassLoader/forNameLeak/ClassForNameLeak.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8151486
+ * @summary Call Class.forName() on the system classloader from a class loaded
+ *          from a custom classloader.
+ * @library /lib/testlibrary
+ * @build jdk.testlibrary.Utils jdk.testlibrary.JarUtils
+ * @build ClassForName ClassForNameLeak
+ * @run main/othervm/policy=test.policy -Djava.security.manager ClassForNameLeak
+ */
+
+import java.lang.ref.PhantomReference;
+import java.lang.ref.Reference;
+import java.lang.ref.ReferenceQueue;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import jdk.testlibrary.Utils;
+import jdk.testlibrary.JarUtils;
+
+/*
+ * Create .jar, load ClassForName from .jar using a URLClassLoader
+ */
+public class ClassForNameLeak {
+    private static final long TIMEOUT = (long)(5000.0 * Utils.TIMEOUT_FACTOR);
+    private static final String TESTCLASSES = System.getProperty("test.classes", ".");
+    private static final String CLASSFILENAME = "ClassForName.class";
+    private static final int THREADS = 10;
+    private static final ReferenceQueue<ClassLoader> rq = new ReferenceQueue<>();
+
+    // Use a new classloader to load the ClassForName class, then run its
+    // Runnable.
+    public static PhantomReference<ClassLoader> loadAndRun(Path jarFilePath)
+            throws Exception {
+        ClassLoader classLoader = new URLClassLoader(
+                new URL[]{jarFilePath.toUri().toURL()}) {
+            @Override public String toString() { return "LeakedClassLoader"; }
+        };
+
+        Class<?> loadClass = Class.forName("ClassForName", true, classLoader);
+        ((Runnable) loadClass.newInstance()).run();
+
+        PhantomReference<ClassLoader> ref = new PhantomReference<>(classLoader, rq);
+        System.out.println("returning phantom ref: " + ref + " to " + classLoader);
+        return ref;
+    }
+
+    public static void main(final String[] args) throws Exception {
+        Path classFile = FileSystems.getDefault().getPath(TESTCLASSES,
+                                                          CLASSFILENAME);
+
+        // Create a temporary .jar file containing ClassForName.class
+        Path testClassesDir = Paths.get(TESTCLASSES);
+        Path jarFilePath = Files.createTempFile("cfn", ".jar");
+        JarUtils.createJarFile(jarFilePath, testClassesDir, classFile);
+        jarFilePath.toFile().deleteOnExit();
+
+        // Remove the ClassForName.class file that jtreg built, to make sure
+        // we're loading from the tmp .jar
+        Files.delete(classFile);
+
+        // Make simultaneous calls to the test method, to stress things a bit
+        ExecutorService es = Executors.newFixedThreadPool(THREADS);
+
+        List<Callable<PhantomReference<ClassLoader>>> callables =
+                Stream.generate(() -> {
+                    Callable<PhantomReference<ClassLoader>> cprcl = () -> {
+                        return loadAndRun(jarFilePath);
+                    };
+                    return cprcl;
+                }).limit(THREADS).collect(Collectors.toList());
+
+        List<Future<PhantomReference<ClassLoader>>> refs = es.invokeAll(callables);
+
+        // Give the GC a chance to enqueue the PhantomReferences
+        for (int i = 0; i < 10; i++) {
+            System.gc();
+        }
+        // Make sure all PhantomReferences to the leaked classloader are enqueued
+        for (int j = 0; j < THREADS; j++) {
+            Reference rmRef = rq.remove(TIMEOUT);
+            if (rmRef == null) {
+                throw new RuntimeException("ClassLoader was never enqueued!");
+            } else {
+                System.out.println("Enqueued " + rmRef);
+            }
+        }
+        System.out.println("All Classloaders successfully enqued");
+    }
+}

test/java/lang/ClassLoader/forNameLeak/test.policy

+grant {
+  permission java.io.FilePermission "<<ALL FILES>>", "read, write, delete";
+  permission java.lang.RuntimePermission "createClassLoader";
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.util.PropertyPermission "*", "read"; /* for Utils */
+};