5088429: varargs overloading problem

Summary: compiler implementation for overload resolution w/ varargs method does not match JLS
Reviewed-by: jjg

src/share/classes/com/sun/tools/javac/comp/Resolve.java

@@ -724,23 +724,11 @@ public class Resolve {
         switch (m2.kind) {
         case MTH:
             if (m1 == m2) return m1;
-            Type mt1 = types.memberType(site, m1);
-            noteWarner.unchecked = false;
-            boolean m1SignatureMoreSpecific =
-                (instantiate(env, site, m2, types.lowerBoundArgtypes(mt1), null,
-                             allowBoxing, false, noteWarner) != null ||
-                 useVarargs && instantiate(env, site, m2, types.lowerBoundArgtypes(mt1), null,
-                                           allowBoxing, true, noteWarner) != null) &&
-                !noteWarner.unchecked;
-            Type mt2 = types.memberType(site, m2);
-            noteWarner.unchecked = false;
-            boolean m2SignatureMoreSpecific =
-                (instantiate(env, site, m1, types.lowerBoundArgtypes(mt2), null,
-                             allowBoxing, false, noteWarner) != null ||
-                 useVarargs && instantiate(env, site, m1, types.lowerBoundArgtypes(mt2), null,
-                                           allowBoxing, true, noteWarner) != null) &&
-                !noteWarner.unchecked;
+            boolean m1SignatureMoreSpecific = signatureMoreSpecific(env, site, m1, m2, allowBoxing, useVarargs);
+            boolean m2SignatureMoreSpecific = signatureMoreSpecific(env, site, m2, m1, allowBoxing, useVarargs);
             if (m1SignatureMoreSpecific && m2SignatureMoreSpecific) {
+                Type mt1 = types.memberType(site, m1);
+                Type mt2 = types.memberType(site, m2);
                 if (!types.overrideEquivalent(mt1, mt2))
                     return new AmbiguityError(m1, m2);
                 // same signature; select (a) the non-bridge method, or
@@ -824,6 +812,50 @@ public class Resolve {
             throw new AssertionError();
         }
     }
+    //where
+    private boolean signatureMoreSpecific(Env<AttrContext> env, Type site, Symbol m1, Symbol m2, boolean allowBoxing, boolean useVarargs) {
+        Type mtype1 = types.memberType(site, adjustVarargs(m1, m2, useVarargs));
+        noteWarner.unchecked = false;
+        return (instantiate(env, site, adjustVarargs(m2, m1, useVarargs), types.lowerBoundArgtypes(mtype1), null,
+                             allowBoxing, false, noteWarner) != null ||
+                 useVarargs && instantiate(env, site, adjustVarargs(m2, m1, useVarargs), types.lowerBoundArgtypes(mtype1), null,
+                                           allowBoxing, true, noteWarner) != null) &&
+                !noteWarner.unchecked;
+    }
+    //where
+    private Symbol adjustVarargs(Symbol to, Symbol from, boolean useVarargs) {
+        List<Type> fromArgs = from.type.getParameterTypes();
+        List<Type> toArgs = to.type.getParameterTypes();
+        if (useVarargs &&
+                toArgs.length() < fromArgs.length() &&
+                (from.flags() & VARARGS) != 0 &&
+                (to.flags() & VARARGS) != 0) {
+            //if we are checking a varargs method 'from' against another varargs
+            //method 'to' (where arity of 'to' < arity of 'from') then expand signature
+            //of 'to' to 'fit' arity of 'from' (this means adding fake formals to 'to'
+            //until 'to' signature has the same arity as 'from')
+            ListBuffer<Type> args = ListBuffer.lb();
+            Type varargsTypeFrom = fromArgs.last();
+            Type varargsTypeTo = toArgs.last();
+            while (fromArgs.head != varargsTypeFrom) {
+                args.append(toArgs.head == varargsTypeTo ? types.elemtype(varargsTypeTo) : toArgs.head);
+                fromArgs = fromArgs.tail;
+                toArgs = toArgs.head == varargsTypeTo ?
+                    toArgs :
+                    toArgs.tail;
+            }
+            args.append(varargsTypeTo);
+            MethodSymbol msym = new MethodSymbol(to.flags_field,
+                                                 to.name,
+                                                 (Type)to.type.clone(), //see: 6990136
+                                                 to.owner);
+            MethodType mtype = msym.type.asMethodType();
+            mtype.argtypes = args.toList();
+            return msym;
+        } else {
+            return to;
+        }
+    }
 
     /** Find best qualified method matching given name, type and value
      *  arguments.

test/tools/javac/varargs/5088429/T5088429Neg01.java

+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 5088429
+ *
+ * @summary varargs overloading problem
+ * @author mcimadamore
+ * @compile/fail/ref=T5088429Neg01.out -XDrawDiagnostics T5088429Neg01.java
+ *
+ */
+
+class T5088429Neg01 {
+    interface A {}
+    interface B extends A {}
+
+    T5088429Neg01(A... args) {}
+    T5088429Neg01(A a, A... args) {}
+
+    void test(B b) {
+        new T5088429Neg01(b);
+    }
+}

test/tools/javac/varargs/5088429/T5088429Neg01.out

+T5088429Neg01.java:42:9: compiler.err.ref.ambiguous: T5088429Neg01, kindname.constructor, T5088429Neg01(T5088429Neg01.A...), T5088429Neg01, kindname.constructor, T5088429Neg01(T5088429Neg01.A,T5088429Neg01.A...), T5088429Neg01
+1 error

test/tools/javac/varargs/5088429/T5088429Neg02.java

+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 5088429
+ *
+ * @summary varargs overloading problem
+ * @author mcimadamore
+ * @compile/fail/ref=T5088429Neg02.out -XDrawDiagnostics T5088429Neg02.java
+ *
+ */
+
+class T5088429Neg02 {
+    interface A {}
+    interface B extends A {}
+
+    void m(A... args) {}
+    void m(A a, A... args) {}
+
+    void test(B b) {
+        m(b);
+    }
+}

test/tools/javac/varargs/5088429/T5088429Neg02.out

+T5088429Neg02.java:42:9: compiler.err.ref.ambiguous: m, kindname.method, m(T5088429Neg02.A...), T5088429Neg02, kindname.method, m(T5088429Neg02.A,T5088429Neg02.A...), T5088429Neg02
+1 error

test/tools/javac/varargs/5088429/T5088429Pos01.java

+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 5088429
+ *
+ * @summary varargs overloading problem
+ * @author mcimadamore
+ * @compile T5088429Pos01.java
+ *
+ */
+
+class T5088429Pos01 {
+    interface A {}
+    interface B extends A {}
+
+    T5088429Pos01(A... args) {}
+    T5088429Pos01(B b, A... args) {}
+
+    void test(B b) {
+        new T5088429Pos01(b);
+    }
+}

test/tools/javac/varargs/5088429/T5088429Pos02.java

+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 5088429
+ *
+ * @summary varargs overloading problem
+ * @author mcimadamore
+ * @compile T5088429Pos02.java
+ *
+ */
+
+class T5088429Pos02 {
+    interface A {}
+    interface B extends A {}
+
+    void m(A... args) {}
+    void m(B b, A... args) {}
+
+    void test(B b) {
+        m(b);
+    }
+}