Merge

src/share/vm/opto/library_call.cpp

@@ -160,6 +160,7 @@ class LibraryCallKit : public GraphKit {
   bool inline_trans(vmIntrinsics::ID id);
   bool inline_abs(vmIntrinsics::ID id);
   bool inline_sqrt(vmIntrinsics::ID id);
+  void finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);
   bool inline_pow(vmIntrinsics::ID id);
   bool inline_exp(vmIntrinsics::ID id);
   bool inline_min_max(vmIntrinsics::ID id);
@@ -1535,39 +1536,78 @@ bool LibraryCallKit::inline_abs(vmIntrinsics::ID id) {
   return true;
 }
 
-//------------------------------inline_exp-------------------------------------
-// Inline exp instructions, if possible.  The Intel hardware only misses
-// really odd corner cases (+/- Infinity).  Just uncommon-trap them.
-bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) {
-  assert(id == vmIntrinsics::_dexp, "Not exp");
-
-  // If this inlining ever returned NaN in the past, we do not intrinsify it
-  // every again.  NaN results requires StrictMath.exp handling.
-  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
-
-  _sp += arg_size();        // restore stack pointer
-  Node *x = pop_math_arg();
-  Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x));
-
+void LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
   //-------------------
-  //result=(result.isNaN())? StrictMath::exp():result;
-  // Check: If isNaN() by checking result!=result? then go to Strict Math
+  //result=(result.isNaN())? funcAddr():result;
+  // Check: If isNaN() by checking result!=result? then either trap
+  // or go to runtime
   Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));
   // Build the boolean node
   Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );
 
-  { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
+  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
+    {
+      BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
       // End the current control-flow path
       push_pair(x);
-    // Math.exp intrinsic returned a NaN, which requires StrictMath.exp
-    // to handle.  Recompile without intrinsifying Math.exp
+      if (y != NULL) {
+        push_pair(y);
+      }
+      // The pow or exp intrinsic returned a NaN, which requires a call
+      // to the runtime.  Recompile with the runtime call.
       uncommon_trap(Deoptimization::Reason_intrinsic,
                     Deoptimization::Action_make_not_entrant);
     }
+    push_pair(result);
+  } else {
+    // If this inlining ever returned NaN in the past, we compile a call
+    // to the runtime to properly handle corner cases
 
-  C->set_has_split_ifs(true); // Has chance for split-if optimization
+    IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
+    Node* if_slow = _gvn.transform( new (C, 1) IfFalseNode(iff) );
+    Node* if_fast = _gvn.transform( new (C, 1) IfTrueNode(iff) );
+
+    if (!if_slow->is_top()) {
+      RegionNode* result_region = new(C, 3) RegionNode(3);
+      PhiNode*    result_val = new (C, 3) PhiNode(result_region, Type::DOUBLE);
 
+      result_region->init_req(1, if_fast);
+      result_val->init_req(1, result);
+
+      set_control(if_slow);
+
+      const TypePtr* no_memory_effects = NULL;
+      Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
+                                   no_memory_effects,
+                                   x, top(), y, y ? top() : NULL);
+      Node* value = _gvn.transform(new (C, 1) ProjNode(rt, TypeFunc::Parms+0));
+#ifdef ASSERT
+      Node* value_top = _gvn.transform(new (C, 1) ProjNode(rt, TypeFunc::Parms+1));
+      assert(value_top == top(), "second value must be top");
+#endif
+
+      result_region->init_req(2, control());
+      result_val->init_req(2, value);
+      push_result(result_region, result_val);
+    } else {
       push_pair(result);
+    }
+  }
+}
+
+//------------------------------inline_exp-------------------------------------
+// Inline exp instructions, if possible.  The Intel hardware only misses
+// really odd corner cases (+/- Infinity).  Just uncommon-trap them.
+bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) {
+  assert(id == vmIntrinsics::_dexp, "Not exp");
+
+  _sp += arg_size();        // restore stack pointer
+  Node *x = pop_math_arg();
+  Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x));
+
+  finish_pow_exp(result, x, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
+
+  C->set_has_split_ifs(true); // Has chance for split-if optimization
 
   return true;
 }
@@ -1577,17 +1617,12 @@ bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) {
 bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
   assert(id == vmIntrinsics::_dpow, "Not pow");
 
-  // If this inlining ever returned NaN in the past, we do not intrinsify it
-  // every again.  NaN results requires StrictMath.pow handling.
-  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
-
-  // Do not intrinsify on older platforms which lack cmove.
-  if (ConditionalMoveLimit == 0)  return false;
-
   // Pseudocode for pow
   // if (x <= 0.0) {
-  //   if ((double)((int)y)==y) { // if y is int
-  //     result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y)
+  //   long longy = (long)y;
+  //   if ((double)longy == y) { // if y is long
+  //     if (y + 1 == y) longy = 0; // huge number: even
+  //     result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);
   //   } else {
   //     result = NaN;
   //   }
@@ -1595,7 +1630,7 @@ bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
   //   result = DPow(x,y);
   // }
   // if (result != result)?  {
-  //   uncommon_trap();
+  //   result = uncommon_trap() or runtime_call();
   // }
   // return result;
 
@@ -1603,15 +1638,14 @@ bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
   Node* y = pop_math_arg();
   Node* x = pop_math_arg();
 
-  Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) );
+  Node* result = NULL;
 
-  // Short form: if not top-level (i.e., Math.pow but inlining Math.pow
-  // inside of something) then skip the fancy tests and just check for
-  // NaN result.
-  Node *result = NULL;
-  if( jvms()->depth() >= 1 ) {
-    result = fast_result;
+  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
+    // Short form: skip the fancy tests and just check for NaN result.
+    result = _gvn.transform( new (C, 3) PowDNode(0, x, y) );
   } else {
+    // If this inlining ever returned NaN in the past, include all
+    // checks + call to the runtime.
 
     // Set the merge point for If node with condition of (x <= 0.0)
     // There are four possible paths to region node and phi node
@@ -1627,55 +1661,95 @@ bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
     Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) );
     // Branch either way
     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
-    Node *opt_test = _gvn.transform(if1);
-    //assert( opt_test->is_If(), "Expect an IfNode");
-    IfNode *opt_if1 = (IfNode*)opt_test;
     // Fast path taken; set region slot 3
-    Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) );
+    Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(if1) );
     r->init_req(3,fast_taken); // Capture fast-control
 
     // Fast path not-taken, i.e. slow path
-    Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) );
+    Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(if1) );
 
     // Set fast path result
-    Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) );
+    Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) );
     phi->init_req(3, fast_result);
 
     // Complex path
-    // Build the second if node (if y is int)
-    // Node for (int)y
-    Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y));
-    // Node for (double)((int) y)
-    Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty));
-    // Check (double)((int) y) : y
-    Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y));
-    // Check if (y isn't int) then go to slow path
-
-    Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) );
+    // Build the second if node (if y is long)
+    // Node for (long)y
+    Node *longy = _gvn.transform( new (C, 2) ConvD2LNode(y));
+    // Node for (double)((long) y)
+    Node *doublelongy= _gvn.transform( new (C, 2) ConvL2DNode(longy));
+    // Check (double)((long) y) : y
+    Node *cmplongy= _gvn.transform(new (C, 3) CmpDNode(doublelongy, y));
+    // Check if (y isn't long) then go to slow path
+
+    Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmplongy, BoolTest::ne ) );
     // Branch either way
     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
-    Node *slow_path = opt_iff(r,if2); // Set region path 2
+    Node* ylong_path = _gvn.transform( new (C, 1) IfFalseNode(if2));
 
-    // Calculate DPow(abs(x), y)*(1 & (int)y)
+    Node *slow_path = _gvn.transform( new (C, 1) IfTrueNode(if2) );
+
+    // Calculate DPow(abs(x), y)*(1 & (long)y)
     // Node for constant 1
-    Node *conone = intcon(1);
-    // 1& (int)y
-    Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) );
+    Node *conone = longcon(1);
+    // 1& (long)y
+    Node *signnode= _gvn.transform( new (C, 3) AndLNode(conone, longy) );
+
+    // A huge number is always even. Detect a huge number by checking
+    // if y + 1 == y and set integer to be tested for parity to 0.
+    // Required for corner case:
+    // (long)9.223372036854776E18 = max_jlong
+    // (double)(long)9.223372036854776E18 = 9.223372036854776E18
+    // max_jlong is odd but 9.223372036854776E18 is even
+    Node* yplus1 = _gvn.transform( new (C, 3) AddDNode(y, makecon(TypeD::make(1))));
+    Node *cmpyplus1= _gvn.transform(new (C, 3) CmpDNode(yplus1, y));
+    Node *bolyplus1 = _gvn.transform( new (C, 2) BoolNode( cmpyplus1, BoolTest::eq ) );
+    Node* correctedsign = NULL;
+    if (ConditionalMoveLimit != 0) {
+      correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
+    } else {
+      IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
+      RegionNode *r = new (C, 3) RegionNode(3);
+      Node *phi = new (C, 3) PhiNode(r, TypeLong::LONG);
+      r->init_req(1, _gvn.transform( new (C, 1) IfFalseNode(ifyplus1)));
+      r->init_req(2, _gvn.transform( new (C, 1) IfTrueNode(ifyplus1)));
+      phi->init_req(1, signnode);
+      phi->init_req(2, longcon(0));
+      correctedsign = _gvn.transform(phi);
+      ylong_path = _gvn.transform(r);
+      record_for_igvn(r);
+    }
+
     // zero node
-    Node *conzero = intcon(0);
-    // Check (1&(int)y)==0?
-    Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero));
-    // Check if (1&(int)y)!=0?, if so the result is negative
+    Node *conzero = longcon(0);
+    // Check (1&(long)y)==0?
+    Node *cmpeq1 = _gvn.transform(new (C, 3) CmpLNode(correctedsign, conzero));
+    // Check if (1&(long)y)!=0?, if so the result is negative
     Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) );
     // abs(x)
     Node *absx=_gvn.transform( new (C, 2) AbsDNode(x));
     // abs(x)^y
-    Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) );
+    Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, absx, y) );
     // -abs(x)^y
     Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy));
-    // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
-    Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
+    // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
+    Node *signresult = NULL;
+    if (ConditionalMoveLimit != 0) {
+      signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
+    } else {
+      IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
+      RegionNode *r = new (C, 3) RegionNode(3);
+      Node *phi = new (C, 3) PhiNode(r, Type::DOUBLE);
+      r->init_req(1, _gvn.transform( new (C, 1) IfFalseNode(ifyeven)));
+      r->init_req(2, _gvn.transform( new (C, 1) IfTrueNode(ifyeven)));
+      phi->init_req(1, absxpowy);
+      phi->init_req(2, negabsxpowy);
+      signresult = _gvn.transform(phi);
+      ylong_path = _gvn.transform(r);
+      record_for_igvn(r);
+    }
     // Set complex path fast result
+    r->init_req(2, ylong_path);
     phi->init_req(2, signresult);
 
     static const jlong nan_bits = CONST64(0x7ff8000000000000);
@@ -1689,27 +1763,10 @@ bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
     result=_gvn.transform(phi);
   }
 
-  //-------------------
-  //result=(result.isNaN())? uncommon_trap():result;
-  // Check: If isNaN() by checking result!=result? then go to Strict Math
-  Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));
-  // Build the boolean node
-  Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );
-
-  { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
-    // End the current control-flow path
-    push_pair(x);
-    push_pair(y);
-    // Math.pow intrinsic returned a NaN, which requires StrictMath.pow
-    // to handle.  Recompile without intrinsifying Math.pow.
-    uncommon_trap(Deoptimization::Reason_intrinsic,
-                  Deoptimization::Action_make_not_entrant);
-  }
+  finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
 
   C->set_has_split_ifs(true); // Has chance for split-if optimization
 
-  push_pair(result);
-
   return true;
 }
 

src/share/vm/opto/parse2.cpp

@@ -1278,9 +1278,9 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest,
        // or the narrowOop equivalent.
        const Type* obj_type = _gvn.type(obj);
        const TypeOopPtr* tboth = obj_type->join(con_type)->isa_oopptr();
-       if (tboth != NULL && tboth != obj_type && tboth->higher_equal(obj_type)) {
+       if (tboth != NULL && tboth->klass_is_exact() && tboth != obj_type &&
+           tboth->higher_equal(obj_type)) {
           // obj has to be of the exact type Foo if the CmpP succeeds.
-          assert(tboth->klass_is_exact(), "klass should be exact");
           int obj_in_map = map()->find_edge(obj);
           JVMState* jvms = this->jvms();
           if (obj_in_map >= 0 &&

src/share/vm/opto/subnode.cpp

@@ -554,9 +554,7 @@ const Type *CmpUNode::sub( const Type *t1, const Type *t2 ) const {
       return TypeInt::CC_GE;
     } else if (hi0 <= lo1) {
       // Check for special case in Hashtable::get.  (See below.)
-      if ((jint)lo0 >= 0 && (jint)lo1 >= 0 &&
-          in(1)->Opcode() == Op_ModI &&
-          in(1)->in(2) == in(2) )
+      if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && is_index_range_check())
         return TypeInt::CC_LT;
       return TypeInt::CC_LE;
     }
@@ -567,13 +565,17 @@ const Type *CmpUNode::sub( const Type *t1, const Type *t2 ) const {
   // to be positive.
   // (This is a gross hack, since the sub method never
   // looks at the structure of the node in any other case.)
-  if ((jint)lo0 >= 0 && (jint)lo1 >= 0 &&
-      in(1)->Opcode() == Op_ModI &&
-      in(1)->in(2)->uncast() == in(2)->uncast())
+  if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && is_index_range_check())
     return TypeInt::CC_LT;
   return TypeInt::CC;                   // else use worst case results
 }
 
+bool CmpUNode::is_index_range_check() const {
+  // Check for the "(X ModI Y) CmpU Y" shape
+  return (in(1)->Opcode() == Op_ModI &&
+          in(1)->in(2)->eqv_uncast(in(2)));
+}
+
 //------------------------------Idealize---------------------------------------
 Node *CmpINode::Ideal( PhaseGVN *phase, bool can_reshape ) {
   if (phase->type(in(2))->higher_equal(TypeInt::ZERO)) {

src/share/vm/opto/subnode.hpp

@@ -158,6 +158,7 @@ public:
   CmpUNode( Node *in1, Node *in2 ) : CmpNode(in1,in2) {}
   virtual int Opcode() const;
   virtual const Type *sub( const Type *, const Type * ) const;
+  bool is_index_range_check() const;
 };
 
 //------------------------------CmpPNode---------------------------------------

test/compiler/7177917/Test7177917.java

+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * Micro-benchmark for Math.pow() and Math.exp()
+ */
+
+import java.util.*;
+
+public class Test7177917 {
+
+  static double d;
+
+  static Random r = new Random(0);
+
+  static long  m_pow(double[][] values) {
+    double res = 0;
+    long start = System.nanoTime();
+    for (int i = 0; i < values.length; i++) {
+      res += Math.pow(values[i][0], values[i][1]);
+    }
+    long stop = System.nanoTime();
+    d = res;
+    return (stop - start) / 1000;
+  }
+
+  static long  m_exp(double[] values) {
+    double res = 0;
+    long start = System.nanoTime();
+    for (int i = 0; i < values.length; i++) {
+      res += Math.exp(values[i]);
+    }
+    long stop = System.nanoTime();
+    d = res;
+    return (stop - start) / 1000;
+  }
+
+  static double[][] pow_values(int nb) {
+    double[][] res = new double[nb][2];
+    for (int i = 0; i < nb; i++) {
+      double ylogx = (1 + (r.nextDouble() * 2045)) - 1023; // 2045 rather than 2046 as a safety margin
+      double x = Math.abs(Double.longBitsToDouble(r.nextLong()));
+      while (x != x) {
+        x = Math.abs(Double.longBitsToDouble(r.nextLong()));
+      }
+      double logx = Math.log(x) / Math.log(2);
+      double y = ylogx / logx;
+
+      res[i][0] = x;
+      res[i][1] = y;
+    }
+    return res;
+  }
+
+  static double[] exp_values(int nb) {
+    double[] res = new double[nb];
+    for (int i = 0; i < nb; i++) {
+      double ylogx = (1 + (r.nextDouble() * 2045)) - 1023; // 2045 rather than 2046 as a safety margin
+      double x = Math.E;
+      double logx = Math.log(x) / Math.log(2);
+      double y = ylogx / logx;
+      res[i] = y;
+    }
+    return res;
+  }
+
+  static public void main(String[] args) {
+    {
+      // warmup
+      double[][] warmup_values = pow_values(10);
+      m_pow(warmup_values);
+
+      for (int i = 0; i < 20000; i++) {
+        m_pow(warmup_values);
+      }
+      // test pow perf
+      double[][] values = pow_values(1000000);
+      System.out.println("==> POW " + m_pow(values));
+
+      // force uncommon trap
+      double[][] nan_values = new double[1][2];
+      nan_values[0][0] = Double.NaN;
+      nan_values[0][1] = Double.NaN;
+      m_pow(nan_values);
+
+      // force recompilation
+      for (int i = 0; i < 20000; i++) {
+        m_pow(warmup_values);
+      }
+
+      // test pow perf again
+      System.out.println("==> POW " + m_pow(values));
+    }
+    {
+      // warmup
+      double[] warmup_values = exp_values(10);
+      m_exp(warmup_values);
+
+      for (int i = 0; i < 20000; i++) {
+        m_exp(warmup_values);
+      }
+
+      // test pow perf
+      double[] values = exp_values(1000000);
+      System.out.println("==> EXP " + m_exp(values));
+
+      // force uncommon trap
+      double[] nan_values = new double[1];
+      nan_values[0] = Double.NaN;
+      m_exp(nan_values);
+
+      // force recompilation
+      for (int i = 0; i < 20000; i++) {
+        m_exp(warmup_values);
+      }
+
+      // test pow perf again
+      System.out.println("==> EXP " + m_exp(values));
+    }
+  }
+}