7008866: Missing loop predicate for loop with multiple entries

Summary: Add predicates when loop head bytecode is parsed instead of when back branch bytecode is parsed.
Reviewed-by: never

src/share/vm/opto/c2_globals.hpp

@@ -180,6 +180,9 @@
   develop(bool, TraceLoopPredicate, false,                                  \
           "Trace generation of loop predicates")                            \
                                                                             \
+  develop(bool, TraceLoopOpts, false,                                       \
+          "Trace executed loop optimizations")                              \
+                                                                            \
   product(bool, OptimizeFill, false,                                        \
           "convert fill/copy loops into intrinsic")                         \
                                                                             \

src/share/vm/opto/graphKit.cpp

@@ -3338,6 +3338,49 @@ InitializeNode* AllocateNode::initialization() {
   return NULL;
 }
 
+//----------------------------- loop predicates ---------------------------
+
+//------------------------------add_predicate_impl----------------------------
+void GraphKit::add_predicate_impl(Deoptimization::DeoptReason reason, int nargs) {
+  // Too many traps seen?
+  if (too_many_traps(reason)) {
+#ifdef ASSERT
+    if (TraceLoopPredicate) {
+      int tc = C->trap_count(reason);
+      tty->print("too many traps=%s tcount=%d in ",
+                    Deoptimization::trap_reason_name(reason), tc);
+      method()->print(); // which method has too many predicate traps
+      tty->cr();
+    }
+#endif
+    // We cannot afford to take more traps here,
+    // do not generate predicate.
+    return;
+  }
+
+  Node *cont    = _gvn.intcon(1);
+  Node* opq     = _gvn.transform(new (C, 2) Opaque1Node(C, cont));
+  Node *bol     = _gvn.transform(new (C, 2) Conv2BNode(opq));
+  IfNode* iff   = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
+  Node* iffalse = _gvn.transform(new (C, 1) IfFalseNode(iff));
+  C->add_predicate_opaq(opq);
+  {
+    PreserveJVMState pjvms(this);
+    set_control(iffalse);
+    _sp += nargs;
+    uncommon_trap(reason, Deoptimization::Action_maybe_recompile);
+  }
+  Node* iftrue = _gvn.transform(new (C, 1) IfTrueNode(iff));
+  set_control(iftrue);
+}
+
+//------------------------------add_predicate---------------------------------
+void GraphKit::add_predicate(int nargs) {
+  if (UseLoopPredicate) {
+    add_predicate_impl(Deoptimization::Reason_predicate, nargs);
+  }
+}
+
 //----------------------------- store barriers ----------------------------
 #define __ ideal.
 

src/share/vm/opto/graphKit.hpp

@@ -793,6 +793,10 @@ class GraphKit : public Phase {
     if (!tst->is_Con())  record_for_igvn(iff);     // Range-check and Null-check removal is later
     return iff;
   }
+
+  // Insert a loop predicate into the graph
+  void add_predicate(int nargs = 0);
+  void add_predicate_impl(Deoptimization::DeoptReason reason, int nargs);
 };
 
 // Helper class to support building of control flow branches. Upon

src/share/vm/opto/idealKit.cpp

@@ -154,8 +154,18 @@ void IdealKit::end_if() {
 //
 // Pushes the loop top cvstate first, then the else (loop exit) cvstate
 // onto the stack.
-void IdealKit::loop(IdealVariable& iv, Node* init, BoolTest::mask relop, Node* limit, float prob, float cnt) {
+void IdealKit::loop(GraphKit* gkit, int nargs, IdealVariable& iv, Node* init, BoolTest::mask relop, Node* limit, float prob, float cnt) {
   assert((state() & (BlockS|LoopS|IfThenS|ElseS)), "bad state for new loop");
+
+  // Sync IdealKit and graphKit.
+  gkit->set_all_memory(this->merged_memory());
+  gkit->set_control(this->ctrl());
+  // Add loop predicate.
+  gkit->add_predicate(nargs);
+  // Update IdealKit memory.
+  this->set_all_memory(gkit->merged_memory());
+  this->set_ctrl(gkit->control());
+
   set(iv, init);
   Node* head = make_label(1);
   bind(head);

src/share/vm/opto/idealKit.hpp

@@ -29,6 +29,7 @@
 #include "opto/cfgnode.hpp"
 #include "opto/connode.hpp"
 #include "opto/divnode.hpp"
+#include "opto/graphKit.hpp"
 #include "opto/mulnode.hpp"
 #include "opto/phaseX.hpp"
 #include "opto/subnode.hpp"
@@ -160,7 +161,7 @@ class IdealKit: public StackObj {
                bool push_new_state = true);
   void else_();
   void end_if();
-  void loop(IdealVariable& iv, Node* init, BoolTest::mask cmp, Node* limit,
+  void loop(GraphKit* gkit, int nargs, IdealVariable& iv, Node* init, BoolTest::mask cmp, Node* limit,
             float prob = PROB_LIKELY(0.9), float cnt = COUNT_UNKNOWN);
   void end_loop();
   Node* make_label(int goto_ct);

src/share/vm/opto/library_call.cpp

@@ -1101,6 +1101,8 @@ Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_ar
   float likely   = PROB_LIKELY(0.9);
   float unlikely = PROB_UNLIKELY(0.9);
 
+  const int nargs = 2; // number of arguments to push back for uncommon trap in predicate
+
   const int value_offset  = java_lang_String::value_offset_in_bytes();
   const int count_offset  = java_lang_String::count_offset_in_bytes();
   const int offset_offset = java_lang_String::offset_offset_in_bytes();
@@ -1138,12 +1140,12 @@ Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_ar
   Node* return_    = __ make_label(1);
 
   __ set(rtn,__ ConI(-1));
-  __ loop(i, sourceOffset, BoolTest::lt, sourceEnd); {
+  __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
        Node* i2  = __ AddI(__ value(i), targetCountLess1);
        // pin to prohibit loading of "next iteration" value which may SEGV (rare)
        Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
        __ if_then(src, BoolTest::eq, lastChar, unlikely); {
-         __ loop(j, zero, BoolTest::lt, targetCountLess1); {
+         __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
               Node* tpj = __ AddI(targetOffset, __ value(j));
               Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
               Node* ipj  = __ AddI(__ value(i), __ value(j));

src/share/vm/opto/loopUnswitch.cpp

@@ -110,6 +110,13 @@ void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) {
   IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
   assert(unswitch_iff != NULL, "should be at least one");
 
+#ifndef PRODUCT
+  if (TraceLoopOpts) {
+    tty->print("Unswitch   %d ", head->unswitch_count()+1);
+    loop->dump_head();
+  }
+#endif
+
   // Need to revert back to normal loop
   if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) {
     head->as_CountedLoop()->set_normal_loop();

src/share/vm/opto/loopnode.hpp

@@ -93,6 +93,7 @@ public:
       in(1) != NULL && phase->type(in(1)) != Type::TOP &&
       in(2) != NULL && phase->type(in(2)) != Type::TOP;
   }
+  bool is_valid_counted_loop() const;
 #ifndef PRODUCT
   virtual void dump_spec(outputStream *st) const;
 #endif
@@ -101,9 +102,8 @@ public:
 //------------------------------Counted Loops----------------------------------
 // Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
 // path (and maybe some other exit paths).  The trip-counter exit is always
-// last in the loop.  The trip-counter does not have to stride by a constant,
-// but it does have to stride by a loop-invariant amount; the exit value is
-// also loop invariant.
+// last in the loop.  The trip-counter have to stride by a constant;
+// the exit value is also loop invariant.
 
 // CountedLoopNodes and CountedLoopEndNodes come in matched pairs.  The
 // CountedLoopNode has the incoming loop control and the loop-back-control
@@ -112,7 +112,7 @@ public:
 // CountedLoopNode if there is control flow in the loop), the post-increment
 // trip-counter value, and the limit.  The trip-counter value is always of
 // the form (Op old-trip-counter stride).  The old-trip-counter is produced
-// by a Phi connected to the CountedLoopNode.  The stride is loop invariant.
+// by a Phi connected to the CountedLoopNode.  The stride is constant.
 // The Op is any commutable opcode, including Add, Mul, Xor.  The
 // CountedLoopEndNode also takes in the loop-invariant limit value.
 
@@ -696,6 +696,9 @@ private:
   // Is safept not required by an outer loop?
   bool is_deleteable_safept(Node* sfpt);
 
+  // Replace parallel induction variable (parallel to trip counter)
+  void replace_parallel_iv(IdealLoopTree *loop);
+
   // Perform verification that the graph is valid.
   PhaseIdealLoop( PhaseIterGVN &igvn) :
     PhaseTransform(Ideal_Loop),
@@ -751,7 +754,7 @@ public:
   // Per-Node transform
   virtual Node *transform( Node *a_node ) { return 0; }
 
-  Node *is_counted_loop( Node *x, IdealLoopTree *loop );
+  bool is_counted_loop( Node *x, IdealLoopTree *loop );
 
   // Return a post-walked LoopNode
   IdealLoopTree *get_loop( Node *n ) const {
@@ -815,16 +818,22 @@ public:
   bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
 
   // Return true if proj is for "proj->[region->..]call_uct"
-  bool is_uncommon_trap_proj(ProjNode* proj, bool must_reason_predicate = false);
+  // Return true if proj is for "proj->[region->..]call_uct"
+  static bool is_uncommon_trap_proj(ProjNode* proj, Deoptimization::DeoptReason reason);
   // Return true for    "if(test)-> proj -> ...
   //                          |
   //                          V
   //                      other_proj->[region->..]call_uct"
-  bool is_uncommon_trap_if_pattern(ProjNode* proj, bool must_reason_predicate = false);
+  static bool is_uncommon_trap_if_pattern(ProjNode* proj, Deoptimization::DeoptReason reason);
   // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
-  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj);
-  // Find a good location to insert a predicate
-  ProjNode* find_predicate_insertion_point(Node* start_c);
+  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
+                                        Deoptimization::DeoptReason reason);
+  void register_control(Node* n, IdealLoopTree *loop, Node* pred);
+
+   // Find a good location to insert a predicate
+  static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
+  // Find a predicate
+  static Node* find_predicate(Node* entry);
   // Construct a range check for a predicate if
   BoolNode* rc_predicate(Node* ctrl,
                          int scale, Node* offset,
@@ -936,7 +945,7 @@ public:
   Node *has_local_phi_input( Node *n );
   // Mark an IfNode as being dominated by a prior test,
   // without actually altering the CFG (and hence IDOM info).
-  void dominated_by( Node *prevdom, Node *iff );
+  void dominated_by( Node *prevdom, Node *iff, bool flip = false );
 
   // Split Node 'n' through merge point
   Node *split_thru_region( Node *n, Node *region );

src/share/vm/opto/loopopts.cpp

@@ -42,13 +42,13 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
     return NULL;
   }
   int wins = 0;
-  assert( !n->is_CFG(), "" );
-  assert( region->is_Region(), "" );
+  assert(!n->is_CFG(), "");
+  assert(region->is_Region(), "");
 
   const Type* type = n->bottom_type();
   const TypeOopPtr *t_oop = _igvn.type(n)->isa_oopptr();
   Node *phi;
-  if( t_oop != NULL && t_oop->is_known_instance_field() ) {
+  if (t_oop != NULL && t_oop->is_known_instance_field()) {
     int iid    = t_oop->instance_id();
     int index  = C->get_alias_index(t_oop);
     int offset = t_oop->offset();
@@ -57,20 +57,20 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
     phi = PhiNode::make_blank(region, n);
   }
   uint old_unique = C->unique();
-  for( uint i = 1; i < region->req(); i++ ) {
+  for (uint i = 1; i < region->req(); i++) {
     Node *x;
     Node* the_clone = NULL;
-    if( region->in(i) == C->top() ) {
+    if (region->in(i) == C->top()) {
       x = C->top();             // Dead path?  Use a dead data op
     } else {
       x = n->clone();           // Else clone up the data op
       the_clone = x;            // Remember for possible deletion.
       // Alter data node to use pre-phi inputs
-      if( n->in(0) == region )
+      if (n->in(0) == region)
         x->set_req( 0, region->in(i) );
-      for( uint j = 1; j < n->req(); j++ ) {
+      for (uint j = 1; j < n->req(); j++) {
         Node *in = n->in(j);
-        if( in->is_Phi() && in->in(0) == region )
+        if (in->is_Phi() && in->in(0) == region)
           x->set_req( j, in->in(i) ); // Use pre-Phi input for the clone
       }
     }
@@ -85,7 +85,7 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
     // happen if the singleton occurs on loop entry, as the elimination of
     // the PhiNode may cause the resulting node to migrate back to a previous
     // loop iteration.
-    if( singleton && t == Type::TOP ) {
+    if (singleton && t == Type::TOP) {
       // Is_Loop() == false does not confirm the absence of a loop (e.g., an
       // irreducible loop may not be indicated by an affirmative is_Loop());
       // therefore, the only top we can split thru a phi is on a backedge of
@@ -93,7 +93,7 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
       singleton &= region->is_Loop() && (i != LoopNode::EntryControl);
     }
 
-    if( singleton ) {
+    if (singleton) {
       wins++;
       x = ((PhaseGVN&)_igvn).makecon(t);
     } else {
@@ -108,12 +108,12 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
       // igvn->type(x) is set to x->Value() already.
       x->raise_bottom_type(t);
       Node *y = x->Identity(&_igvn);
-      if( y != x ) {
+      if (y != x) {
         wins++;
         x = y;
       } else {
         y = _igvn.hash_find(x);
-        if( y ) {
+        if (y) {
           wins++;
           x = y;
         } else {
@@ -129,7 +129,7 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
     phi->set_req( i, x );
   }
   // Too few wins?
-  if( wins <= policy ) {
+  if (wins <= policy) {
     _igvn.remove_dead_node(phi);
     return NULL;
   }
@@ -137,7 +137,7 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
   // Record Phi
   register_new_node( phi, region );
 
-  for( uint i2 = 1; i2 < phi->req(); i2++ ) {
+  for (uint i2 = 1; i2 < phi->req(); i2++) {
     Node *x = phi->in(i2);
     // If we commoned up the cloned 'x' with another existing Node,
     // the existing Node picks up a new use.  We need to make the
@@ -145,24 +145,44 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
     Node *old_ctrl;
     IdealLoopTree *old_loop;
 
+    if (x->is_Con()) {
+      // Constant's control is always root.
+      set_ctrl(x, C->root());
+      continue;
+    }
     // The occasional new node
-    if( x->_idx >= old_unique ) {   // Found a new, unplaced node?
-      old_ctrl = x->is_Con() ? C->root() : NULL;
-      old_loop = NULL;              // Not in any prior loop
+    if (x->_idx >= old_unique) {     // Found a new, unplaced node?
+      old_ctrl = NULL;
+      old_loop = NULL;               // Not in any prior loop
     } else {
-      old_ctrl = x->is_Con() ? C->root() : get_ctrl(x);
+      old_ctrl = get_ctrl(x);
       old_loop = get_loop(old_ctrl); // Get prior loop
     }
     // New late point must dominate new use
-    Node *new_ctrl = dom_lca( old_ctrl, region->in(i2) );
+    Node *new_ctrl = dom_lca(old_ctrl, region->in(i2));
+    if (new_ctrl == old_ctrl) // Nothing is changed
+      continue;
+
+    IdealLoopTree *new_loop = get_loop(new_ctrl);
+
+    // Don't move x into a loop if its uses are
+    // outside of loop. Otherwise x will be cloned
+    // for each use outside of this loop.
+    IdealLoopTree *use_loop = get_loop(region);
+    if (!new_loop->is_member(use_loop) &&
+        (old_loop == NULL || !new_loop->is_member(old_loop))) {
+      // Take early control, later control will be recalculated
+      // during next iteration of loop optimizations.
+      new_ctrl = get_early_ctrl(x);
+      new_loop = get_loop(new_ctrl);
+    }
     // Set new location
     set_ctrl(x, new_ctrl);
-    IdealLoopTree *new_loop = get_loop( new_ctrl );
     // If changing loop bodies, see if we need to collect into new body
-    if( old_loop != new_loop ) {
-      if( old_loop && !old_loop->_child )
+    if (old_loop != new_loop) {
+      if (old_loop && !old_loop->_child)
         old_loop->_body.yank(x);
-      if( !new_loop->_child )
+      if (!new_loop->_child)
         new_loop->_body.push(x);  // Collect body info
     }
   }
@@ -174,9 +194,9 @@ Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
 // Replace the dominated test with an obvious true or false.  Place it on the
 // IGVN worklist for later cleanup.  Move control-dependent data Nodes on the
 // live path up to the dominating control.
-void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff ) {
+void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip ) {
 #ifndef PRODUCT
-  if( VerifyLoopOptimizations && PrintOpto ) tty->print_cr("dominating test");
+  if (VerifyLoopOptimizations && PrintOpto) tty->print_cr("dominating test");
 #endif
 
 
@@ -185,6 +205,12 @@ void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff ) {
   assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
   int pop = prevdom->Opcode();
   assert( pop == Op_IfFalse || pop == Op_IfTrue, "" );
+  if (flip) {
+    if (pop == Op_IfTrue)
+      pop = Op_IfFalse;
+    else
+      pop = Op_IfTrue;
+  }
   // 'con' is set to true or false to kill the dominated test.
   Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO);
   set_ctrl(con, C->root()); // Constant gets a new use
@@ -197,7 +223,7 @@ void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff ) {
   // I can assume this path reaches an infinite loop.  In this case it's not
   // important to optimize the data Nodes - either the whole compilation will
   // be tossed or this path (and all data Nodes) will go dead.
-  if( iff->outcnt() != 2 ) return;
+  if (iff->outcnt() != 2) return;
 
   // Make control-dependent data Nodes on the live path (path that will remain
   // once the dominated IF is removed) become control-dependent on the
@@ -207,16 +233,16 @@ void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff ) {
 
   for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
     Node* cd = dp->fast_out(i); // Control-dependent node
-    if( cd->depends_only_on_test() ) {
-      assert( cd->in(0) == dp, "" );
-      _igvn.hash_delete( cd );
+    if (cd->depends_only_on_test()) {
+      assert(cd->in(0) == dp, "");
+      _igvn.hash_delete(cd);
       cd->set_req(0, prevdom);
-      set_early_ctrl( cd );
+      set_early_ctrl(cd);
       _igvn._worklist.push(cd);
       IdealLoopTree *new_loop = get_loop(get_ctrl(cd));
-      if( old_loop != new_loop ) {
-        if( !old_loop->_child ) old_loop->_body.yank(cd);
-        if( !new_loop->_child ) new_loop->_body.push(cd);
+      if (old_loop != new_loop) {
+        if (!old_loop->_child) old_loop->_body.yank(cd);
+        if (!new_loop->_child) new_loop->_body.push(cd);
       }
       --i;
       --imax;
@@ -2338,6 +2364,11 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
   }
 
 #if !defined(PRODUCT)
+  if (TraceLoopOpts) {
+    tty->print("PartialPeel  ");
+    loop->dump_head();
+  }
+
   if (TracePartialPeeling) {
     tty->print_cr("before partial peel one iteration");
     Node_List wl;
@@ -2481,6 +2512,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
   // Create new loop head for new phis and to hang
   // the nodes being moved (sinked) from the peel region.
   LoopNode* new_head = new (C, 3) LoopNode(last_peel, last_peel);
+  new_head->set_unswitch_count(head->unswitch_count()); // Preserve
   _igvn.register_new_node_with_optimizer(new_head);
   assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled");
   first_not_peeled->set_req(0, new_head);
@@ -2651,24 +2683,23 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
 // prevent loop-fallout uses of the pre-incremented trip counter (which are
 // then alive with the post-incremented trip counter forcing an extra
 // register move)
-void PhaseIdealLoop::reorg_offsets( IdealLoopTree *loop ) {
+void PhaseIdealLoop::reorg_offsets(IdealLoopTree *loop) {
+  // Perform it only for canonical counted loops.
+  // Loop's shape could be messed up by iteration_split_impl.
+  if (!loop->_head->is_CountedLoop())
+    return;
+  if (!loop->_head->as_Loop()->is_valid_counted_loop())
+    return;
 
   CountedLoopNode *cl = loop->_head->as_CountedLoop();
   CountedLoopEndNode *cle = cl->loopexit();
-  if( !cle ) return;            // The occasional dead loop
-  // Find loop exit control
   Node *exit = cle->proj_out(false);
-  assert( exit->Opcode() == Op_IfFalse, "" );
+  Node *phi = cl->phi();
 
   // Check for the special case of folks using the pre-incremented
   // trip-counter on the fall-out path (forces the pre-incremented
   // and post-incremented trip counter to be live at the same time).
   // Fix this by adjusting to use the post-increment trip counter.
-  Node *phi = cl->phi();
-  if( !phi ) return;            // Dead infinite loop
-
-  // Shape messed up, probably by iteration_split_impl
-  if (phi->in(LoopNode::LoopBackControl) != cl->incr()) return;
 
   bool progress = true;
   while (progress) {
@@ -2677,21 +2708,19 @@ void PhaseIdealLoop::reorg_offsets( IdealLoopTree *loop ) {
       Node* use = phi->fast_out(i);   // User of trip-counter
       if (!has_ctrl(use))  continue;
       Node *u_ctrl = get_ctrl(use);
-      if( use->is_Phi() ) {
+      if (use->is_Phi()) {
         u_ctrl = NULL;
-        for( uint j = 1; j < use->req(); j++ )
-          if( use->in(j) == phi )
-            u_ctrl = dom_lca( u_ctrl, use->in(0)->in(j) );
+        for (uint j = 1; j < use->req(); j++)
+          if (use->in(j) == phi)
+            u_ctrl = dom_lca(u_ctrl, use->in(0)->in(j));
       }
       IdealLoopTree *u_loop = get_loop(u_ctrl);
       // Look for loop-invariant use
-      if( u_loop == loop ) continue;
-      if( loop->is_member( u_loop ) ) continue;
+      if (u_loop == loop) continue;
+      if (loop->is_member(u_loop)) continue;
       // Check that use is live out the bottom.  Assuming the trip-counter
       // update is right at the bottom, uses of of the loop middle are ok.
-      if( dom_lca( exit, u_ctrl ) != exit ) continue;
-      // protect against stride not being a constant
-      if( !cle->stride_is_con() ) continue;
+      if (dom_lca(exit, u_ctrl) != exit) continue;
       // Hit!  Refactor use to use the post-incremented tripcounter.
       // Compute a post-increment tripcounter.
       Node *opaq = new (C, 2) Opaque2Node( C, cle->incr() );
@@ -2702,9 +2731,10 @@ void PhaseIdealLoop::reorg_offsets( IdealLoopTree *loop ) {
       register_new_node( post, u_ctrl );
       _igvn.hash_delete(use);
       _igvn._worklist.push(use);
-      for( uint j = 1; j < use->req(); j++ )
-        if( use->in(j) == phi )
+      for (uint j = 1; j < use->req(); j++) {
+        if (use->in(j) == phi)
           use->set_req(j, post);
+      }
       // Since DU info changed, rerun loop
       progress = true;
       break;

src/share/vm/opto/parse.hpp

@@ -136,6 +136,7 @@ class Parse : public GraphKit {
     uint               _count;          // how many times executed?  Currently only set by _goto's
     bool               _is_parsed;      // has this block been parsed yet?
     bool               _is_handler;     // is this block an exception handler?
+    bool               _has_merged_backedge; // does this block have merged backedge?
     SafePointNode*     _start_map;      // all values flowing into this block
     MethodLivenessResult _live_locals;  // lazily initialized liveness bitmap
 
@@ -168,6 +169,18 @@ class Parse : public GraphKit {
     // True after any predecessor flows control into this block
     bool is_merged() const                 { return _start_map != NULL; }
 
+#ifdef ASSERT
+    // True after backedge predecessor flows control into this block
+    bool has_merged_backedge() const       { return _has_merged_backedge; }
+    void mark_merged_backedge(Block* pred) {
+      assert(is_SEL_head(), "should be loop head");
+      if (pred != NULL && is_SEL_backedge(pred)) {
+        assert(is_parsed(), "block should be parsed before merging backedges");
+        _has_merged_backedge = true;
+      }
+    }
+#endif
+
     // True when all non-exception predecessors have been parsed.
     bool is_ready() const                  { return preds_parsed() == pred_count(); }
 
@@ -441,11 +454,6 @@ class Parse : public GraphKit {
     }
   }
 
-  // Return true if the parser should add a loop predicate
-  bool should_add_predicate(int target_bci);
-  // Insert a loop predicate into the graph
-  void add_predicate();
-
   // Note:  Intrinsic generation routines may be found in library_call.cpp.
 
   // Helper function to setup Ideal Call nodes

src/share/vm/opto/parse1.cpp

@@ -637,6 +637,25 @@ void Parse::do_all_blocks() {
         // (Note that dead locals do not get phis built, ever.)
         ensure_phis_everywhere();
 
+        if (block->is_SEL_head() &&
+            UseLoopPredicate) {
+          // Add predicate to single entry (not irreducible) loop head.
+          assert(!block->has_merged_backedge(), "only entry paths should be merged for now");
+          // Need correct bci for predicate.
+          // It is fine to set it here since do_one_block() will set it anyway.
+          set_parse_bci(block->start());
+          add_predicate();
+          // Add new region for back branches.
+          int edges = block->pred_count() - block->preds_parsed() + 1; // +1 for original region
+          RegionNode *r = new (C, edges+1) RegionNode(edges+1);
+          _gvn.set_type(r, Type::CONTROL);
+          record_for_igvn(r);
+          r->init_req(edges, control());
+          set_control(r);
+          // Add new phis.
+          ensure_phis_everywhere();
+        }
+
         // Leave behind an undisturbed copy of the map, for future merges.
         set_map(clone_map());
       }
@@ -1113,7 +1132,7 @@ void Parse::Block::init_node(Parse* outer, int rpo) {
   _preds_parsed = 0;
   _count = 0;
   assert(pred_count() == 0 && preds_parsed() == 0, "sanity");
-  assert(!(is_merged() || is_parsed() || is_handler()), "sanity");
+  assert(!(is_merged() || is_parsed() || is_handler() || has_merged_backedge()), "sanity");
   assert(_live_locals.size() == 0, "sanity");
 
   // entry point has additional predecessor
@@ -1350,10 +1369,6 @@ void Parse::do_one_block() {
     set_parse_bci(iter().cur_bci());
 
     if (bci() == block()->limit()) {
-      // insert a predicate if it falls through to a loop head block
-      if (should_add_predicate(bci())){
-        add_predicate();
-      }
       // Do not walk into the next block until directed by do_all_blocks.
       merge(bci());
       break;
@@ -1498,17 +1513,29 @@ void Parse::merge_common(Parse::Block* target, int pnum) {
         || target->is_handler()       // These have unpredictable inputs.
         || target->is_loop_head()     // Known multiple inputs
         || control()->is_Region()) {  // We must hide this guy.
+
+      int current_bci = bci();
+      set_parse_bci(target->start()); // Set target bci
+      if (target->is_SEL_head()) {
+        DEBUG_ONLY( target->mark_merged_backedge(block()); )
+        if (target->start() == 0) {
+          // Add loop predicate for the special case when
+          // there are backbranches to the method entry.
+          add_predicate();
+        }
+      }
       // Add a Region to start the new basic block.  Phis will be added
       // later lazily.
       int edges = target->pred_count();
       if (edges < pnum)  edges = pnum;  // might be a new path!
-      Node *r = new (C, edges+1) RegionNode(edges+1);
+      RegionNode *r = new (C, edges+1) RegionNode(edges+1);
       gvn().set_type(r, Type::CONTROL);
       record_for_igvn(r);
       // zap all inputs to NULL for debugging (done in Node(uint) constructor)
       // for (int j = 1; j < edges+1; j++) { r->init_req(j, NULL); }
       r->init_req(pnum, control());
       set_control(r);
+      set_parse_bci(current_bci); // Restore bci
     }
 
     // Convert the existing Parser mapping into a mapping at this bci.
@@ -1517,7 +1544,11 @@ void Parse::merge_common(Parse::Block* target, int pnum) {
 
   } else {                      // Prior mapping at this bci
     if (TraceOptoParse) {  tty->print(" with previous state"); }
-
+#ifdef ASSERT
+    if (target->is_SEL_head()) {
+      target->mark_merged_backedge(block());
+    }
+#endif
     // We must not manufacture more phis if the target is already parsed.
     bool nophi = target->is_parsed();
 
@@ -2054,37 +2085,6 @@ void Parse::add_safepoint() {
   }
 }
 
-//------------------------------should_add_predicate--------------------------
-bool Parse::should_add_predicate(int target_bci) {
-  if (!UseLoopPredicate) return false;
-  Block* target = successor_for_bci(target_bci);
-  if (target != NULL          &&
-      target->is_loop_head()  &&
-      block()->rpo() < target->rpo()) {
-    return true;
-  }
-  return false;
-}
-
-//------------------------------add_predicate---------------------------------
-void Parse::add_predicate() {
-  assert(UseLoopPredicate,"use only for loop predicate");
-  Node *cont    = _gvn.intcon(1);
-  Node* opq     = _gvn.transform(new (C, 2) Opaque1Node(C, cont));
-  Node *bol     = _gvn.transform(new (C, 2) Conv2BNode(opq));
-  IfNode* iff   = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
-  Node* iffalse = _gvn.transform(new (C, 1) IfFalseNode(iff));
-  C->add_predicate_opaq(opq);
-  {
-    PreserveJVMState pjvms(this);
-    set_control(iffalse);
-    uncommon_trap(Deoptimization::Reason_predicate,
-                  Deoptimization::Action_maybe_recompile);
-  }
-  Node* iftrue = _gvn.transform(new (C, 1) IfTrueNode(iff));
-  set_control(iftrue);
-}
-
 #ifndef PRODUCT
 //------------------------show_parse_info--------------------------------------
 void Parse::show_parse_info() {

src/share/vm/opto/parse2.cpp

@@ -293,11 +293,6 @@ void Parse::do_tableswitch() {
   if (len < 1) {
     // If this is a backward branch, add safepoint
     maybe_add_safepoint(default_dest);
-    if (should_add_predicate(default_dest)){
-      _sp += 1; // set original stack for use by uncommon_trap
-      add_predicate();
-      _sp -= 1;
-    }
     merge(default_dest);
     return;
   }
@@ -344,11 +339,6 @@ void Parse::do_lookupswitch() {
 
   if (len < 1) {    // If this is a backward branch, add safepoint
     maybe_add_safepoint(default_dest);
-    if (should_add_predicate(default_dest)){
-      _sp += 1; // set original stack for use by uncommon_trap
-      add_predicate();
-      _sp -= 1;
-    }
     merge(default_dest);
     return;
   }
@@ -756,9 +746,6 @@ void Parse::do_jsr() {
   push(_gvn.makecon(ret_addr));
 
   // Flow to the jsr.
-  if (should_add_predicate(jsr_bci)){
-    add_predicate();
-  }
   merge(jsr_bci);
 }
 
@@ -1040,11 +1027,6 @@ void Parse::do_ifnull(BoolTest::mask btest, Node *c) {
       profile_taken_branch(target_bci);
       adjust_map_after_if(btest, c, prob, branch_block, next_block);
       if (!stopped()) {
-        if (should_add_predicate(target_bci)){ // add a predicate if it branches to a loop
-          int nargs = repush_if_args(); // set original stack for uncommon_trap
-          add_predicate();
-          _sp -= nargs;
-        }
         merge(target_bci);
       }
     }
@@ -1168,11 +1150,6 @@ void Parse::do_if(BoolTest::mask btest, Node* c) {
       profile_taken_branch(target_bci);
       adjust_map_after_if(taken_btest, c, prob, branch_block, next_block);
       if (!stopped()) {
-        if (should_add_predicate(target_bci)){ // add a predicate if it branches to a loop
-          int nargs = repush_if_args(); // set original stack for the uncommon_trap
-          add_predicate();
-          _sp -= nargs;
-        }
         merge(target_bci);
       }
     }
@@ -2166,10 +2143,6 @@ void Parse::do_one_bytecode() {
     // Update method data
     profile_taken_branch(target_bci);
 
-    // Add loop predicate if it goes to a loop
-    if (should_add_predicate(target_bci)){
-      add_predicate();
-    }
     // Merge the current control into the target basic block
     merge(target_bci);
 

src/share/vm/opto/stringopts.cpp

@@ -969,6 +969,10 @@ Node* PhaseStringOpts::int_stringSize(GraphKit& kit, Node* arg) {
     // for (int i=0; ; i++)
     //   if (x <= sizeTable[i])
     //     return i+1;
+
+    // Add loop predicate first.
+    kit.add_predicate();
+
     RegionNode *loop = new (C, 3) RegionNode(3);
     loop->init_req(1, kit.control());
     kit.gvn().set_type(loop, Type::CONTROL);
@@ -1086,6 +1090,9 @@ void PhaseStringOpts::int_getChars(GraphKit& kit, Node* arg, Node* char_array, N
   // }
 
   {
+    // Add loop predicate first.
+    kit.add_predicate();
+
     RegionNode *head = new (C, 3) RegionNode(3);
     head->init_req(1, kit.control());
     kit.gvn().set_type(head, Type::CONTROL);