8055514: Wrong, confusing error when non-static varargs referenced in static context

Summary: Improved heuristics in MethodResolutionPhase.mergeResults()
Reviewed-by: vromero

src/share/classes/com/sun/tools/javac/comp/Resolve.java

@@ -3045,7 +3045,7 @@ public class Resolve {
         /**
          * Should lookup stop at given phase with given result
          */
-        protected boolean shouldStop(Symbol sym, MethodResolutionPhase phase) {
+        final boolean shouldStop(Symbol sym, MethodResolutionPhase phase) {
             return phase.ordinal() > maxPhase.ordinal() ||
                     sym.kind < ERRONEOUS || sym.kind == AMBIGUOUS;
         }
@@ -4228,15 +4228,39 @@ public class Resolve {
         VARARITY(true, true) {
             @Override
             public Symbol mergeResults(Symbol bestSoFar, Symbol sym) {
-                switch (sym.kind) {
-                    case WRONG_MTH:
-                        return (bestSoFar.kind == WRONG_MTH || bestSoFar.kind == WRONG_MTHS) ?
-                            bestSoFar :
-                            sym;
-                    case ABSENT_MTH:
-                        return bestSoFar;
-                    default:
-                        return sym;
+                //Check invariants (see {@code LookupHelper.shouldStop})
+                Assert.check(bestSoFar.kind >= ERRONEOUS && bestSoFar.kind != AMBIGUOUS);
+                if (sym.kind < ERRONEOUS) {
+                    //varargs resolution successful
+                    return sym;
+                } else {
+                    //pick best error
+                    switch (bestSoFar.kind) {
+                        case WRONG_MTH:
+                        case WRONG_MTHS:
+                            //Override previous errors if they were caused by argument mismatch.
+                            //This generally means preferring current symbols - but we need to pay
+                            //attention to the fact that the varargs lookup returns 'less' candidates
+                            //than the previous rounds, and adjust that accordingly.
+                            switch (sym.kind) {
+                                case WRONG_MTH:
+                                    //if the previous round matched more than one method, return that
+                                    //result instead
+                                    return bestSoFar.kind == WRONG_MTHS ?
+                                            bestSoFar : sym;
+                                case ABSENT_MTH:
+                                    //do not override erroneous symbol if the arity lookup did not
+                                    //match any method
+                                    return bestSoFar;
+                                case WRONG_MTHS:
+                                default:
+                                    //safe to override
+                                    return sym;
+                            }
+                        default:
+                            //otherwise, return first error
+                            return bestSoFar;
+                    }
                 }
             }
         };

test/tools/javac/varargs/8055514/T8055514.java

+/*
+ * @test /nodynamiccopyright/
+ * @bug     8055514
+ * @summary  Wrong, confusing error when non-static varargs referenced in static context
+ * @compile/fail/ref=T8055514.out -Xlint:varargs -Werror -XDrawDiagnostics T8055514.java
+ */
+class T8055514 {
+    void m(int... args) { }
+
+    void m2(int... args) { }
+    static void m2(String s) { }
+
+    void m3(int... args) { }
+    static void m3(String s) { }
+    static void m3(Runnable r) { }
+
+    void m4(int... args) { }
+    void m4(int i1, int i2, int i3) { }
+
+    static void test() {
+        m(1,2,3); //only one candidate (varargs) - varargs error wins
+        m2(1,2,3); //two candidates - only one applicable (varargs) - varargs error wins
+        m3(1,2,3); //three candidates - only one applicable (varargs) - varargs error wins
+        m4(1,2,3); //two candidates - both applicable - basic error wins
+    }
+}

test/tools/javac/varargs/8055514/T8055514.out

+T8055514.java:21:9: compiler.err.non-static.cant.be.ref: kindname.method, m(int...)
+T8055514.java:22:9: compiler.err.non-static.cant.be.ref: kindname.method, m2(int...)
+T8055514.java:23:9: compiler.err.non-static.cant.be.ref: kindname.method, m3(int...)
+T8055514.java:24:9: compiler.err.non-static.cant.be.ref: kindname.method, m4(int,int,int)
+4 errors