8029467: Widening of booleans causes bad results

Reviewed-by: jlaskey, lagergren

src/jdk/nashorn/internal/codegen/Attr.java

@@ -766,7 +766,7 @@ final class Attr extends NodeOperatorVisitor<LexicalContext> {
                 symbol.setType(Type.OBJECT);
             }
 
-            returnType = Type.widest(returnTypes.pop(), symbol.getSymbolType());
+            returnType = widestReturnType(returnTypes.pop(), symbol.getSymbolType());
         } else {
             returnType = Type.OBJECT; //undefined
         }
@@ -1433,10 +1433,30 @@ final class Attr extends NodeOperatorVisitor<LexicalContext> {
         ensureTypeNotUnknown(trueExpr);
         ensureTypeNotUnknown(falseExpr);
 
-        final Type type = Type.widest(trueExpr.getType(), falseExpr.getType());
+        final Type type = widestReturnType(trueExpr.getType(), falseExpr.getType());
         return end(ensureSymbol(type, ternaryNode));
     }
 
+    /**
+     * When doing widening for return types of a function or a ternary operator, it is not valid to widen a boolean to
+     * anything other than Object. Also, widening a numeric type to an object type must widen to Object proper and not
+     * any more specific subclass (e.g. widest of int/long/double and String is Object).
+     * @param t1 type 1
+     * @param t2 type 2
+     * @return wider of t1 and t2, except if one is boolean and the other is neither boolean nor unknown, or if one is
+     * numeric and the other is neither numeric nor unknown in which case {@code Type.OBJECT} is returned.
+     */
+    private static Type widestReturnType(final Type t1, final Type t2) {
+        if (t1.isUnknown()) {
+            return t2;
+        } else if (t2.isUnknown()) {
+            return t1;
+        } else if (t1.isBoolean() != t2.isBoolean() || t1.isNumeric() != t2.isNumeric()) {
+            return Type.OBJECT;
+        }
+        return Type.widest(t1, t2);
+    }
+
     private void initCompileConstant(final CompilerConstants cc, final Block block, final int flags) {
         final Class<?> type = cc.type();
         // Must not call this method for constants with no explicit types; use the one with (..., Type) signature instead.

test/script/basic/JDK-8029467.js

+/*
+ * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * 
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ * 
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * JDK-8029467: Widening of booleans causes bad results
+ *
+ * @test
+ * @run
+ */
+
+print((function (x) { return x ? true : 0 })(true))
+print((function (x) { if(x) { return true } else { return 0 } })(true))
+print(typeof (function (x) { return x ? 1 : "123" })(true) === "number")
+print(typeof (function (x) { if(x) { return 1 } else { return "123" } })(true) === "number")

test/script/basic/JDK-8029467.js.EXPECTED

+true
+true
+true
+true