8030221: Checking for anonymous class should check for NULL as well as potential nesting

Summary: Store the first non-anonymous class as the host when defining the anonymous class so don't need look for it later.
Reviewed-by: dholmes, lfoltan

hotspot/src/share/vm/prims/unsafe.cpp

@@ -861,6 +861,13 @@ Unsafe_DefineAnonymousClass_impl(JNIEnv *env,
   }
 
   const Klass* host_klass = java_lang_Class::as_Klass(JNIHandles::resolve_non_null(host_class));
+
+  // Make sure it's the real host class, not another anonymous class.
+  while (host_klass != NULL && host_klass->is_instance_klass() &&
+         InstanceKlass::cast(host_klass)->is_anonymous()) {
+    host_klass = InstanceKlass::cast(host_klass)->host_klass();
+  }
+
   // Primitive types have NULL Klass* fields in their java.lang.Class instances.
   if (host_klass == NULL) {
     THROW_0(vmSymbols::java_lang_IllegalArgumentException());

hotspot/src/share/vm/runtime/reflection.cpp

@@ -660,11 +660,13 @@ bool Reflection::verify_field_access(const Klass* current_class,
   }
 
   const Klass* host_class = current_class;
-  while (host_class->is_instance_klass() &&
-         InstanceKlass::cast(host_class)->is_anonymous()) {
-    const Klass* next_host_class = InstanceKlass::cast(host_class)->host_klass();
-    if (next_host_class == NULL)  break;
-    host_class = next_host_class;
+  if (host_class->is_instance_klass() &&
+      InstanceKlass::cast(host_class)->is_anonymous()) {
+    host_class = InstanceKlass::cast(host_class)->host_klass();
+    assert(host_class != NULL, "Anonymous class has null host class");
+    assert(!(host_class->is_instance_klass() &&
+           InstanceKlass::cast(host_class)->is_anonymous()),
+           "host_class should not be anonymous");
   }
   if (host_class == field_class) {
     return true;

hotspot/test/runtime/Unsafe/NestedUnsafe.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @summary Creates an anonymous class inside of an anonymous class.
+ * @library /testlibrary
+ * @modules java.base/jdk.internal.misc
+ *          java.compiler
+ *          java.management
+ * @run main NestedUnsafe
+ */
+
+import java.security.ProtectionDomain;
+import java.io.InputStream;
+import java.lang.*;
+import jdk.test.lib.*;
+import jdk.internal.misc.Unsafe;
+import static jdk.test.lib.Asserts.*;
+
+// package p;
+
+public class NestedUnsafe {
+    // The String concatenation should create the nested anonymous class.
+    static byte klassbuf[] = InMemoryJavaCompiler.compile("TestClass",
+        "public class TestClass { " +
+        "    public static void concat(String one, String two) throws Throwable { " +
+        "        System.out.println(one + two);" +
+        " } } ");
+
+    public static void main(String args[]) throws Exception {
+        Unsafe unsafe = Utils.getUnsafe();
+
+        Class klass = unsafe.defineAnonymousClass(NestedUnsafe.class, klassbuf, new Object[0]);
+        unsafe.ensureClassInitialized(klass);
+        Class[] cArgs = new Class[2];
+        cArgs[0] = String.class;
+        cArgs[1] = String.class;
+        try {
+            klass.getMethod("concat", cArgs).invoke(null, "AA", "BB");
+        } catch (Throwable ex) {
+            throw new RuntimeException("Exception: " + ex.toString());
+        }
+
+        // The anonymous class calls defineAnonymousClass creating a nested anonymous class.
+        byte klassbuf2[] = InMemoryJavaCompiler.compile("TestClass2",
+            "import jdk.internal.misc.Unsafe; " +
+            "public class TestClass2 { " +
+            "    public static void doit() throws Throwable { " +
+            "        Unsafe unsafe = jdk.internal.misc.Unsafe.getUnsafe(); " +
+            "        Class klass2 = unsafe.defineAnonymousClass(TestClass2.class, NestedUnsafe.klassbuf, new Object[0]); " +
+            "        unsafe.ensureClassInitialized(klass2); " +
+            "        Class[] dArgs = new Class[2]; " +
+            "        dArgs[0] = String.class; " +
+            "        dArgs[1] = String.class; " +
+            "        try { " +
+            "            klass2.getMethod(\"concat\", dArgs).invoke(null, \"CC\", \"DD\"); " +
+            "        } catch (Throwable ex) { " +
+            "            throw new RuntimeException(\"Exception: \" + ex.toString()); " +
+            "        } " +
+            "} } ",
+            "-XaddExports:java.base/jdk.internal.misc=ALL-UNNAMED");
+        Class klass2 = unsafe.defineAnonymousClass(NestedUnsafe.class, klassbuf2, new Object[0]);
+        try {
+            klass2.getMethod("doit").invoke(null);
+        } catch (Throwable ex) {
+            throw new RuntimeException("Exception: " + ex.toString());
+        }
+    }
+}