7194897: JSR 292: Cannot create more than 16 instances of an anonymous class

8027681: Lambda serialization fails once reflection proxy generation kicks in
Reviewed-by: ksrini, briangoetz, jfranck
Contributed-by: joel.franck@oracle.com, brian.goetz@oracle.com, robert.field@oracle.com

src/share/classes/sun/reflect/NativeConstructorAccessorImpl.java

 /*
- * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 package sun.reflect;
 
 import java.lang.reflect.*;
+import sun.reflect.misc.ReflectUtil;
 
 /** Used only for the first few invocations of a Constructor;
     afterward, switches to bytecode-based implementation */
@@ -44,7 +45,11 @@ class NativeConstructorAccessorImpl extends ConstructorAccessorImpl {
                IllegalArgumentException,
                InvocationTargetException
     {
-        if (++numInvocations > ReflectionFactory.inflationThreshold()) {
+        // We can't inflate a constructor belonging to a vm-anonymous class
+        // because that kind of class can't be referred to by name, hence can't
+        // be found from the generated bytecode.
+        if (++numInvocations > ReflectionFactory.inflationThreshold()
+                && !ReflectUtil.isVMAnonymousClass(c.getDeclaringClass())) {
             ConstructorAccessorImpl acc = (ConstructorAccessorImpl)
                 new MethodAccessorGenerator().
                     generateConstructor(c.getDeclaringClass(),

src/share/classes/sun/reflect/NativeMethodAccessorImpl.java

 /*
- * Copyright (c) 2001, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 package sun.reflect;
 
 import java.lang.reflect.*;
+import sun.reflect.misc.ReflectUtil;
 
 /** Used only for the first few invocations of a Method; afterward,
     switches to bytecode-based implementation */
@@ -42,7 +43,11 @@ class NativeMethodAccessorImpl extends MethodAccessorImpl {
     public Object invoke(Object obj, Object[] args)
         throws IllegalArgumentException, InvocationTargetException
     {
-        if (++numInvocations > ReflectionFactory.inflationThreshold()) {
+        // We can't inflate methods belonging to vm-anonymous classes because
+        // that kind of class can't be referred to by name, hence can't be
+        // found from the generated bytecode.
+        if (++numInvocations > ReflectionFactory.inflationThreshold()
+                && !ReflectUtil.isVMAnonymousClass(method.getDeclaringClass())) {
             MethodAccessorImpl acc = (MethodAccessorImpl)
                 new MethodAccessorGenerator().
                     generateMethod(method.getDeclaringClass(),

src/share/classes/sun/reflect/misc/ReflectUtil.java

 /*
- * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -298,4 +298,13 @@ public final class ReflectUtil {
         }
         return false;
     }
+
+    /**
+     * Checks if {@code Class cls} is a VM-anonymous class
+     * as defined by {@link sun.misc.Unsafe#defineAnonymousClass}
+     * (not to be confused with a Java Language anonymous inner class).
+     */
+    public static boolean isVMAnonymousClass(Class<?> cls) {
+        return cls.getSimpleName().contains("/");
+    }
 }

test/java/lang/invoke/lambda/RepetitiveLambdaSerialization.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8027681
+ * @summary Lambda serialization fails once reflection proxy generation kicks in
+ * @author  Robert Field
+ * @run main/othervm RepetitiveLambdaSerialization
+ */
+
+import java.io.*;
+
+public class RepetitiveLambdaSerialization {
+
+    static final int REPS = 20;
+
+    public static void main(String[] args) throws Exception {
+        LSI ls = z -> "[" + z + "]";
+        for (int i = 0; i < REPS; ++i) {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            ObjectOutput out = new ObjectOutputStream(baos);
+            out.writeObject(ls);
+            out.flush();
+            out.close();
+        }
+        System.out.println("Passed.");
+    }
+}
+
+interface LSI extends Serializable {
+    String convert(String x);
+}

test/java/util/stream/test/org/openjdk/tests/java/lang/invoke/SerializedLambdaTest.java

 /*
- * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -56,14 +56,18 @@ import static org.testng.Assert.fail;
  */
 @Test
 public class SerializedLambdaTest {
+    public static final int REPS = 50;
+
     @SuppressWarnings("unchecked")
     private<T> void assertSerial(T p, Consumer<T> asserter) throws IOException, ClassNotFoundException {
         asserter.accept(p);
 
-        byte[] bytes = serialize(p);
-        assertTrue(bytes.length > 0);
+        for (int i=0; i<REPS; i++) {
+            byte[] bytes = serialize(p);
+            assertTrue(bytes.length > 0);
 
-        asserter.accept((T) deserialize(bytes));
+            asserter.accept((T) deserialize(bytes));
+        }
     }
 
     private void assertNotSerial(Predicate<String> p, Consumer<Predicate<String>> asserter)

test/sun/reflect/AnonymousNewInstance/ManyNewInstanceAnonTest.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 7194897
+ * @summary JSR 292: Cannot create more than 16 instances of an anonymous class
+ * @author  Robert Field
+ * @library /lib/testlibrary
+ * @compile -XDignore.symbol.file ManyNewInstanceAnonTest.java
+ * @run main ClassFileInstaller ManyNewInstanceAnonTest
+ * @run main/othervm -Xbootclasspath/a:. -Xverify:all ManyNewInstanceAnonTest
+ */
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import sun.misc.Unsafe;
+
+public class ManyNewInstanceAnonTest {
+
+    static final int REPS = 20;
+    static final Class<?> klass = ManyNewInstanceAnonTest.class;
+
+    public static void main(String[] args) throws Exception {
+        Class<?> c = Unsafe.getUnsafe().defineAnonymousClass(klass, readClassFile(), null);
+        for (int i = 0; i < REPS; ++i) {
+            System.out.printf("%d: %s\n", i, c.newInstance());
+        }
+        System.out.println("Passed.");
+    }
+
+    private static byte[] readClassFile() throws Exception {
+        try (InputStream in = klass.getResourceAsStream(klass.getSimpleName() + ".class");
+             ByteArrayOutputStream out = new ByteArrayOutputStream())
+        {
+            int b;
+            while ((b = in.read()) != -1) {
+                out.write(b);
+            }
+            return out.toByteArray();
+        }
+    }
+}