6894779: Loop Predication for Loop Optimizer in C2

Summary: Loop predication implementation
Reviewed-by: never, kvn

src/share/vm/includeDB_compiler2

@@ -601,6 +601,7 @@ locknode.hpp                            subnode.hpp
 
 loopTransform.cpp                       addnode.hpp
 loopTransform.cpp                       allocation.inline.hpp
+loopTransform.cpp                       callnode.hpp
 loopTransform.cpp                       connode.hpp
 loopTransform.cpp                       compileLog.hpp
 loopTransform.cpp                       divnode.hpp

src/share/vm/opto/c2_globals.hpp

@@ -154,6 +154,12 @@
   notproduct(bool, TraceProfileTripCount, false,                            \
           "Trace profile loop trip count information")                      \
                                                                             \
+  product(bool, UseLoopPredicate, true,                                     \
+          "Generate a predicate to select fast/slow loop versions")         \
+                                                                            \
+  develop(bool, TraceLoopPredicate, false,                                  \
+          "Trace generation of loop predicates")                            \
+                                                                            \
   develop(bool, OptoCoalesce, true,                                         \
           "Use Conservative Copy Coalescing in the Register Allocator")     \
                                                                             \

src/share/vm/opto/compile.cpp

@@ -932,6 +932,7 @@ void Compile::Init(int aliaslevel) {
 
   _intrinsics = NULL;
   _macro_nodes = new GrowableArray<Node*>(comp_arena(), 8,  0, NULL);
+  _predicate_opaqs = new GrowableArray<Node*>(comp_arena(), 8,  0, NULL);
   register_library_intrinsics();
 }
 
@@ -1553,6 +1554,19 @@ void Compile::Finish_Warm() {
   }
 }
 
+//---------------------cleanup_loop_predicates-----------------------
+// Remove the opaque nodes that protect the predicates so that all unused
+// checks and uncommon_traps will be eliminated from the ideal graph
+void Compile::cleanup_loop_predicates(PhaseIterGVN &igvn) {
+  if (predicate_count()==0) return;
+  for (int i = predicate_count(); i > 0; i--) {
+    Node * n = predicate_opaque1_node(i-1);
+    assert(n->Opcode() == Op_Opaque1, "must be");
+    igvn.replace_node(n, n->in(1));
+  }
+  assert(predicate_count()==0, "should be clean!");
+  igvn.optimize();
+}
 
 //------------------------------Optimize---------------------------------------
 // Given a graph, optimize it.
@@ -1594,7 +1608,7 @@ void Compile::Optimize() {
   if((loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
     {
       TracePhase t2("idealLoop", &_t_idealLoop, true);
-      PhaseIdealLoop ideal_loop( igvn, true );
+      PhaseIdealLoop ideal_loop( igvn, true, UseLoopPredicate);
       loop_opts_cnt--;
       if (major_progress()) print_method("PhaseIdealLoop 1", 2);
       if (failing())  return;
@@ -1602,7 +1616,7 @@ void Compile::Optimize() {
     // Loop opts pass if partial peeling occurred in previous pass
     if(PartialPeelLoop && major_progress() && (loop_opts_cnt > 0)) {
       TracePhase t3("idealLoop", &_t_idealLoop, true);
-      PhaseIdealLoop ideal_loop( igvn, false );
+      PhaseIdealLoop ideal_loop( igvn, false, UseLoopPredicate);
       loop_opts_cnt--;
       if (major_progress()) print_method("PhaseIdealLoop 2", 2);
       if (failing())  return;
@@ -1610,7 +1624,7 @@ void Compile::Optimize() {
     // Loop opts pass for loop-unrolling before CCP
     if(major_progress() && (loop_opts_cnt > 0)) {
       TracePhase t4("idealLoop", &_t_idealLoop, true);
-      PhaseIdealLoop ideal_loop( igvn, false );
+      PhaseIdealLoop ideal_loop( igvn, false, UseLoopPredicate);
       loop_opts_cnt--;
       if (major_progress()) print_method("PhaseIdealLoop 3", 2);
     }
@@ -1648,13 +1662,21 @@ void Compile::Optimize() {
   // peeling, unrolling, etc.
   if(loop_opts_cnt > 0) {
     debug_only( int cnt = 0; );
+    bool loop_predication = UseLoopPredicate;
     while(major_progress() && (loop_opts_cnt > 0)) {
       TracePhase t2("idealLoop", &_t_idealLoop, true);
       assert( cnt++ < 40, "infinite cycle in loop optimization" );
-      PhaseIdealLoop ideal_loop( igvn, true );
+      PhaseIdealLoop ideal_loop( igvn, true, loop_predication);
       loop_opts_cnt--;
       if (major_progress()) print_method("PhaseIdealLoop iterations", 2);
       if (failing())  return;
+      // Perform loop predication optimization during first iteration after CCP.
+      // After that switch it off and cleanup unused loop predicates.
+      if (loop_predication) {
+        loop_predication = false;
+        cleanup_loop_predicates(igvn);
+        if (failing())  return;
+      }
     }
   }
 

src/share/vm/opto/compile.hpp

@@ -38,6 +38,7 @@ class Node_Notes;
 class OptoReg;
 class PhaseCFG;
 class PhaseGVN;
+class PhaseIterGVN;
 class PhaseRegAlloc;
 class PhaseCCP;
 class PhaseCCP_DCE;
@@ -172,6 +173,7 @@ class Compile : public Phase {
   const char*           _failure_reason;        // for record_failure/failing pattern
   GrowableArray<CallGenerator*>* _intrinsics;   // List of intrinsics.
   GrowableArray<Node*>* _macro_nodes;           // List of nodes which need to be expanded before matching.
+  GrowableArray<Node*>* _predicate_opaqs;       // List of Opaque1 nodes for the loop predicates.
   ConnectionGraph*      _congraph;
 #ifndef PRODUCT
   IdealGraphPrinter*    _printer;
@@ -351,7 +353,9 @@ class Compile : public Phase {
   }
 
   int           macro_count()                   { return _macro_nodes->length(); }
+  int           predicate_count()               { return _predicate_opaqs->length();}
   Node*         macro_node(int idx)             { return _macro_nodes->at(idx); }
+  Node*         predicate_opaque1_node(int idx) { return _predicate_opaqs->at(idx);}
   ConnectionGraph* congraph()                   { return _congraph;}
   void add_macro_node(Node * n) {
     //assert(n->is_macro(), "must be a macro node");
@@ -363,7 +367,19 @@ class Compile : public Phase {
     // that the node is in the array before attempting to remove it
     if (_macro_nodes->contains(n))
       _macro_nodes->remove(n);
+    // remove from _predicate_opaqs list also if it is there
+    if (predicate_count() > 0 && _predicate_opaqs->contains(n)){
+      _predicate_opaqs->remove(n);
     }
+  }
+  void add_predicate_opaq(Node * n) {
+    assert(!_predicate_opaqs->contains(n), " duplicate entry in predicate opaque1");
+    assert(_macro_nodes->contains(n), "should have already been in macro list");
+    _predicate_opaqs->append(n);
+  }
+  // remove the opaque nodes that protect the predicates so that the unused checks and
+  // uncommon traps will be eliminated from the graph.
+  void cleanup_loop_predicates(PhaseIterGVN &igvn);
 
   // Compilation environment.
   Arena*            comp_arena()                { return &_comp_arena; }

src/share/vm/opto/loopnode.cpp

@@ -1420,11 +1420,57 @@ static void log_loop_tree(IdealLoopTree* root, IdealLoopTree* loop, CompileLog*
   }
 }
 
+//---------------------collect_potentially_useful_predicates-----------------------
+// Helper function to collect potentially useful predicates to prevent them from
+// being eliminated by PhaseIdealLoop::eliminate_useless_predicates
+void PhaseIdealLoop::collect_potentially_useful_predicates(
+                         IdealLoopTree * loop, Unique_Node_List &useful_predicates) {
+  if (loop->_child) { // child
+    collect_potentially_useful_predicates(loop->_child, useful_predicates);
+  }
+
+  // self (only loops that we can apply loop predication may use their predicates)
+  if (loop->_head->is_Loop()     &&
+      !loop->_irreducible        &&
+      !loop->tail()->is_top()) {
+    LoopNode *lpn  = loop->_head->as_Loop();
+    Node* entry = lpn->in(LoopNode::EntryControl);
+    ProjNode *predicate_proj = find_predicate_insertion_point(entry);
+    if (predicate_proj != NULL ) { // right pattern that can be used by loop predication
+      assert(entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
+      useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
+    }
+  }
+
+  if ( loop->_next ) { // sibling
+    collect_potentially_useful_predicates(loop->_next, useful_predicates);
+  }
+}
+
+//------------------------eliminate_useless_predicates-----------------------------
+// Eliminate all inserted predicates if they could not be used by loop predication.
+void PhaseIdealLoop::eliminate_useless_predicates() {
+  if (C->predicate_count() == 0) return; // no predicate left
+
+  Unique_Node_List useful_predicates; // to store useful predicates
+  if (C->has_loops()) {
+    collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates);
+  }
+
+  for (int i = C->predicate_count(); i > 0; i--) {
+     Node * n = C->predicate_opaque1_node(i-1);
+     assert(n->Opcode() == Op_Opaque1, "must be");
+     if (!useful_predicates.member(n)) { // not in the useful list
+       _igvn.replace_node(n, n->in(1));
+     }
+  }
+}
+
 //=============================================================================
 //----------------------------build_and_optimize-------------------------------
 // Create a PhaseLoop.  Build the ideal Loop tree.  Map each Ideal Node to
 // its corresponding LoopNode.  If 'optimize' is true, do some loop cleanups.
-void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) {
+void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool do_loop_pred) {
   int old_progress = C->major_progress();
 
   // Reset major-progress flag for the driver's heuristics
@@ -1577,6 +1623,12 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) {
     return;
   }
 
+  // some parser-inserted loop predicates could never be used by loop
+  // predication. Eliminate them before loop optimization
+  if (UseLoopPredicate) {
+    eliminate_useless_predicates();
+  }
+
   // clear out the dead code
   while(_deadlist.size()) {
     _igvn.remove_globally_dead_node(_deadlist.pop());
@@ -1603,7 +1655,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) {
       // Because RCE opportunities can be masked by split_thru_phi,
       // look for RCE candidates and inhibit split_thru_phi
       // on just their loop-phi's for this pass of loop opts
-      if( SplitIfBlocks && do_split_ifs ) {
+      if (SplitIfBlocks && do_split_ifs) {
         if (lpt->policy_range_check(this)) {
           lpt->_rce_candidate = 1; // = true
         }
@@ -1619,12 +1671,17 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) {
     NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
   }
 
+  // Perform loop predication before iteration splitting
+  if (do_loop_pred && C->has_loops() && !C->major_progress()) {
+    _ltree_root->_child->loop_predication(this);
+  }
+
   // Perform iteration-splitting on inner loops.  Split iterations to avoid
   // range checks or one-shot null checks.
 
   // If split-if's didn't hack the graph too bad (no CFG changes)
   // then do loop opts.
-  if( C->has_loops() && !C->major_progress() ) {
+  if (C->has_loops() && !C->major_progress()) {
     memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) );
     _ltree_root->_child->iteration_split( this, worklist );
     // No verify after peeling!  GCM has hoisted code out of the loop.
@@ -1636,7 +1693,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) {
   // Do verify graph edges in any case
   NOT_PRODUCT( C->verify_graph_edges(); );
 
-  if( !do_split_ifs ) {
+  if (!do_split_ifs) {
     // We saw major progress in Split-If to get here.  We forced a
     // pass with unrolling and not split-if, however more split-if's
     // might make progress.  If the unrolling didn't make progress
@@ -2763,6 +2820,22 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) {
   Node *legal = LCA;            // Walk 'legal' up the IDOM chain
   Node *least = legal;          // Best legal position so far
   while( early != legal ) {     // While not at earliest legal
+#ifdef ASSERT
+    if (legal->is_Start() && !early->is_Root()) {
+      // Bad graph. Print idom path and fail.
+      tty->print_cr( "Bad graph detected in build_loop_late");
+      tty->print("n: ");n->dump(); tty->cr();
+      tty->print("early: ");early->dump(); tty->cr();
+      int ct = 0;
+      Node *dbg_legal = LCA;
+      while(!dbg_legal->is_Start() && ct < 100) {
+        tty->print("idom[%d] ",ct); dbg_legal->dump(); tty->cr();
+        ct++;
+        dbg_legal = idom(dbg_legal);
+      }
+      assert(false, "Bad graph detected in build_loop_late");
+    }
+#endif
     // Find least loop nesting depth
     legal = idom(legal);        // Bump up the IDOM tree
     // Check for lower nesting depth

src/share/vm/opto/loopnode.hpp

@@ -30,6 +30,7 @@ class LoopNode;
 class Node;
 class PhaseIdealLoop;
 class VectorSet;
+class Invariance;
 struct small_cache;
 
 //
@@ -325,6 +326,10 @@ public:
   // Returns TRUE if loop tree is structurally changed.
   bool beautify_loops( PhaseIdealLoop *phase );
 
+  // Perform optimization to use the loop predicates for null checks and range checks.
+  // Applies to any loop level (not just the innermost one)
+  bool loop_predication( PhaseIdealLoop *phase);
+
   // Perform iteration-splitting on inner loops.  Split iterations to
   // avoid range checks or one-shot null checks.  Returns false if the
   // current round of loop opts should stop.
@@ -395,6 +400,9 @@ public:
   // into longer memory ops, we may want to increase alignment.
   bool policy_align( PhaseIdealLoop *phase ) const;
 
+  // Return TRUE if "iff" is a range check.
+  bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const;
+
   // Compute loop trip count from profile data
   void compute_profile_trip_cnt( PhaseIdealLoop *phase );
 
@@ -521,9 +529,6 @@ class PhaseIdealLoop : public PhaseTransform {
   }
   Node *dom_lca_for_get_late_ctrl_internal( Node *lca, Node *n, Node *tag );
 
-  // true if CFG node d dominates CFG node n
-  bool is_dominator(Node *d, Node *n);
-
   // Helper function for directing control inputs away from CFG split
   // points.
   Node *find_non_split_ctrl( Node *ctrl ) const {
@@ -572,6 +577,17 @@ public:
     assert(n == find_non_split_ctrl(n), "must return legal ctrl" );
     return n;
   }
+  // true if CFG node d dominates CFG node n
+  bool is_dominator(Node *d, Node *n);
+  // return get_ctrl for a data node and self(n) for a CFG node
+  Node* ctrl_or_self(Node* n) {
+    if (has_ctrl(n))
+      return get_ctrl(n);
+    else {
+      assert (n->is_CFG(), "must be a CFG node");
+      return n;
+    }
+  }
 
 private:
   Node *get_ctrl_no_update( Node *i ) const {
@@ -600,7 +616,7 @@ private:
   // Lazy-dazy update of 'get_ctrl' and 'idom_at' mechanisms.  Replace
   // the 'old_node' with 'new_node'.  Kill old-node.  Add a reference
   // from old_node to new_node to support the lazy update.  Reference
-  // replaces loop reference, since that is not neede for dead node.
+  // replaces loop reference, since that is not needed for dead node.
 public:
   void lazy_update( Node *old_node, Node *new_node ) {
     assert( old_node != new_node, "no cycles please" );
@@ -679,11 +695,11 @@ private:
     _dom_lca_tags(C->comp_arena()),
     _verify_me(NULL),
     _verify_only(true) {
-    build_and_optimize(false);
+    build_and_optimize(false, false);
   }
 
   // build the loop tree and perform any requested optimizations
-  void build_and_optimize(bool do_split_if);
+  void build_and_optimize(bool do_split_if, bool do_loop_pred);
 
 public:
   // Dominators for the sea of nodes
@@ -694,13 +710,13 @@ public:
   Node *dom_lca_internal( Node *n1, Node *n2 ) const;
 
   // Compute the Ideal Node to Loop mapping
-  PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs) :
+  PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool do_loop_pred) :
     PhaseTransform(Ideal_Loop),
     _igvn(igvn),
     _dom_lca_tags(C->comp_arena()),
     _verify_me(NULL),
     _verify_only(false) {
-    build_and_optimize(do_split_ifs);
+    build_and_optimize(do_split_ifs, do_loop_pred);
   }
 
   // Verify that verify_me made the same decisions as a fresh run.
@@ -710,7 +726,7 @@ public:
     _dom_lca_tags(C->comp_arena()),
     _verify_me(verify_me),
     _verify_only(false) {
-    build_and_optimize(false);
+    build_and_optimize(false, false);
   }
 
   // Build and verify the loop tree without modifying the graph.  This
@@ -790,6 +806,30 @@ public:
   // Return true if exp is a scaled induction var plus (or minus) constant
   bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
 
+  // Return true if proj is for "proj->[region->..]call_uct"
+  bool is_uncommon_trap_proj(ProjNode* proj, bool must_reason_predicate = false);
+  // Return true for    "if(test)-> proj -> ...
+  //                          |
+  //                          V
+  //                      other_proj->[region->..]call_uct"
+  bool is_uncommon_trap_if_pattern(ProjNode* proj, bool must_reason_predicate = false);
+  // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
+  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj);
+  // Find a good location to insert a predicate
+  ProjNode* find_predicate_insertion_point(Node* start_c);
+  // Construct a range check for a predicate if
+  BoolNode* rc_predicate(Node* ctrl,
+                         int scale, Node* offset,
+                         Node* init, Node* limit, Node* stride,
+                         Node* range);
+
+  // Implementation of the loop predication to promote checks outside the loop
+  bool loop_predication_impl(IdealLoopTree *loop);
+
+  // Helper function to collect predicate for eliminating the useless ones
+  void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
+  void eliminate_useless_predicates();
+
   // Eliminate range-checks and other trip-counter vs loop-invariant tests.
   void do_range_check( IdealLoopTree *loop, Node_List &old_new );
 
@@ -906,7 +946,6 @@ private:
   const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl);
 
   // Helper functions
-  void register_new_node( Node *n, Node *blk );
   Node *spinup( Node *iff, Node *new_false, Node *new_true, Node *region, Node *phi, small_cache *cache );
   Node *find_use_block( Node *use, Node *def, Node *old_false, Node *new_false, Node *old_true, Node *new_true );
   void handle_use( Node *use, Node *def, small_cache *cache, Node *region_dom, Node *new_false, Node *new_true, Node *old_false, Node *old_true );
@@ -918,6 +957,7 @@ private:
 public:
   void set_created_loop_node() { _created_loop_node = true; }
   bool created_loop_node()     { return _created_loop_node; }
+  void register_new_node( Node *n, Node *blk );
 
 #ifndef PRODUCT
   void dump( ) const;

src/share/vm/opto/parse.hpp

@@ -430,6 +430,11 @@ class Parse : public GraphKit {
     }
   }
 
+  // Return true if the parser should add a loop predicate
+  bool should_add_predicate(int target_bci);
+  // Insert a loop predicate into the graph
+  void add_predicate();
+
   // Note:  Intrinsic generation routines may be found in library_call.cpp.
 
   // Helper function to setup Ideal Call nodes
@@ -491,7 +496,7 @@ class Parse : public GraphKit {
 
   void    do_ifnull(BoolTest::mask btest, Node* c);
   void    do_if(BoolTest::mask btest, Node* c);
-  void    repush_if_args();
+  int     repush_if_args();
   void    adjust_map_after_if(BoolTest::mask btest, Node* c, float prob,
                               Block* path, Block* other_path);
   IfNode* jump_if_fork_int(Node* a, Node* b, BoolTest::mask mask);

src/share/vm/opto/parse1.cpp

@@ -1383,6 +1383,10 @@ void Parse::do_one_block() {
     set_parse_bci(iter().cur_bci());
 
     if (bci() == block()->limit()) {
+      // insert a predicate if it falls through to a loop head block
+      if (should_add_predicate(bci())){
+        add_predicate();
+      }
       // Do not walk into the next block until directed by do_all_blocks.
       merge(bci());
       break;
@@ -2083,6 +2087,37 @@ void Parse::add_safepoint() {
   }
 }
 
+//------------------------------should_add_predicate--------------------------
+bool Parse::should_add_predicate(int target_bci) {
+  if (!UseLoopPredicate) return false;
+  Block* target = successor_for_bci(target_bci);
+  if (target != NULL          &&
+      target->is_loop_head()  &&
+      block()->rpo() < target->rpo()) {
+    return true;
+  }
+  return false;
+}
+
+//------------------------------add_predicate---------------------------------
+void Parse::add_predicate() {
+  assert(UseLoopPredicate,"use only for loop predicate");
+  Node *cont    = _gvn.intcon(1);
+  Node* opq     = _gvn.transform(new (C, 2) Opaque1Node(C, cont));
+  Node *bol     = _gvn.transform(new (C, 2) Conv2BNode(opq));
+  IfNode* iff   = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
+  Node* iffalse = _gvn.transform(new (C, 1) IfFalseNode(iff));
+  C->add_predicate_opaq(opq);
+  {
+    PreserveJVMState pjvms(this);
+    set_control(iffalse);
+    uncommon_trap(Deoptimization::Reason_predicate,
+                  Deoptimization::Action_maybe_recompile);
+  }
+  Node* iftrue = _gvn.transform(new (C, 1) IfTrueNode(iff));
+  set_control(iftrue);
+}
+
 #ifndef PRODUCT
 //------------------------show_parse_info--------------------------------------
 void Parse::show_parse_info() {

src/share/vm/opto/parse2.cpp

@@ -278,6 +278,11 @@ void Parse::do_tableswitch() {
   if (len < 1) {
     // If this is a backward branch, add safepoint
     maybe_add_safepoint(default_dest);
+    if (should_add_predicate(default_dest)){
+      _sp += 1; // set original stack for use by uncommon_trap
+      add_predicate();
+      _sp -= 1;
+    }
     merge(default_dest);
     return;
   }
@@ -324,6 +329,11 @@ void Parse::do_lookupswitch() {
 
   if (len < 1) {    // If this is a backward branch, add safepoint
     maybe_add_safepoint(default_dest);
+    if (should_add_predicate(default_dest)){
+      _sp += 1; // set original stack for use by uncommon_trap
+      add_predicate();
+      _sp -= 1;
+    }
     merge(default_dest);
     return;
   }
@@ -731,6 +741,9 @@ void Parse::do_jsr() {
   push(_gvn.makecon(ret_addr));
 
   // Flow to the jsr.
+  if (should_add_predicate(jsr_bci)){
+    add_predicate();
+  }
   merge(jsr_bci);
 }
 
@@ -881,7 +894,7 @@ bool Parse::seems_never_taken(float prob) {
 
 //-------------------------------repush_if_args--------------------------------
 // Push arguments of an "if" bytecode back onto the stack by adjusting _sp.
-inline void Parse::repush_if_args() {
+inline int Parse::repush_if_args() {
 #ifndef PRODUCT
   if (PrintOpto && WizardMode) {
     tty->print("defending against excessive implicit null exceptions on %s @%d in ",
@@ -895,6 +908,7 @@ inline void Parse::repush_if_args() {
   assert(argument(0) != NULL, "must exist");
   assert(bc_depth == 1 || argument(1) != NULL, "two must exist");
   _sp += bc_depth;
+  return bc_depth;
 }
 
 //----------------------------------do_ifnull----------------------------------
@@ -954,10 +968,16 @@ void Parse::do_ifnull(BoolTest::mask btest, Node *c) {
       // Update method data
       profile_taken_branch(target_bci);
       adjust_map_after_if(btest, c, prob, branch_block, next_block);
-      if (!stopped())
+      if (!stopped()) {
+        if (should_add_predicate(target_bci)){ // add a predicate if it branches to a loop
+          int nargs = repush_if_args(); // set original stack for uncommon_trap
+          add_predicate();
+          _sp -= nargs;
+        }
         merge(target_bci);
       }
     }
+  }
 
   // False branch
   Node* iffalse = _gvn.transform( new (C, 1) IfFalseNode(iff) );
@@ -1076,10 +1096,16 @@ void Parse::do_if(BoolTest::mask btest, Node* c) {
       // Update method data
       profile_taken_branch(target_bci);
       adjust_map_after_if(taken_btest, c, prob, branch_block, next_block);
-      if (!stopped())
+      if (!stopped()) {
+        if (should_add_predicate(target_bci)){ // add a predicate if it branches to a loop
+          int nargs = repush_if_args(); // set original stack for the uncommon_trap
+          add_predicate();
+          _sp -= nargs;
+        }
         merge(target_bci);
       }
     }
+  }
 
   untaken_branch = _gvn.transform(untaken_branch);
   set_control(untaken_branch);
@@ -2080,6 +2106,10 @@ void Parse::do_one_bytecode() {
     // Update method data
     profile_taken_branch(target_bci);
 
+    // Add loop predicate if it goes to a loop
+    if (should_add_predicate(target_bci)){
+      add_predicate();
+    }
     // Merge the current control into the target basic block
     merge(target_bci);
 

src/share/vm/opto/split_if.cpp

@@ -219,6 +219,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
 
 //------------------------------register_new_node------------------------------
 void PhaseIdealLoop::register_new_node( Node *n, Node *blk ) {
+  assert(!n->is_CFG(), "must be data node");
   _igvn.register_new_node_with_optimizer(n);
   set_ctrl(n, blk);
   IdealLoopTree *loop = get_loop(blk);

src/share/vm/runtime/deoptimization.cpp

@@ -1672,7 +1672,8 @@ const char* Deoptimization::_trap_reason_name[Reason_LIMIT] = {
   "unhandled",
   "constraint",
   "div0_check",
-  "age"
+  "age",
+  "predicate"
 };
 const char* Deoptimization::_trap_action_name[Action_LIMIT] = {
   // Note:  Keep this in sync. with enum DeoptAction.

src/share/vm/runtime/deoptimization.hpp

@@ -46,6 +46,7 @@ class Deoptimization : AllStatic {
     Reason_constraint,            // arbitrary runtime constraint violated
     Reason_div0_check,            // a null_check due to division by zero
     Reason_age,                   // nmethod too old; tier threshold reached
+    Reason_predicate,             // compiler generated predicate failed
     Reason_LIMIT,
     // Note:  Keep this enum in sync. with _trap_reason_name.
     Reason_RECORDED_LIMIT = Reason_unloaded   // some are not recorded per bc