6736417: Fastdebug C2 crashes in StoreBNode::Ideal

Summary: The result of step_through_mergemem() and remove_dead_region() is not checked in some cases.
Reviewed-by: never

src/share/vm/opto/callnode.cpp

@@ -829,9 +829,7 @@ SafePointNode* SafePointNode::next_exception() const {
 //------------------------------Ideal------------------------------------------
 // Skip over any collapsed Regions
 Node *SafePointNode::Ideal(PhaseGVN *phase, bool can_reshape) {
-  if (remove_dead_region(phase, can_reshape))  return this;
-
-  return NULL;
+  return remove_dead_region(phase, can_reshape) ? this : NULL;
 }
 
 //------------------------------Identity---------------------------------------

src/share/vm/opto/connode.cpp

@@ -101,6 +101,8 @@ matter ever).
 // Move constants to the right.
 Node *CMoveNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if( in(0) && remove_dead_region(phase, can_reshape) ) return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
   assert( !phase->eqv(in(Condition), this) &&
           !phase->eqv(in(IfFalse), this) &&
           !phase->eqv(in(IfTrue), this), "dead loop in CMoveNode::Ideal" );

src/share/vm/opto/divnode.cpp

@@ -402,6 +402,8 @@ Node *DivINode::Identity( PhaseTransform *phase ) {
 // Divides can be changed to multiplies and/or shifts
 Node *DivINode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (in(0) && remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   const Type *t = phase->type( in(2) );
   if( t == TypeInt::ONE )       // Identity?
@@ -499,6 +501,8 @@ Node *DivLNode::Identity( PhaseTransform *phase ) {
 // Dividing by a power of 2 is a shift.
 Node *DivLNode::Ideal( PhaseGVN *phase, bool can_reshape) {
   if (in(0) && remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   const Type *t = phase->type( in(2) );
   if( t == TypeLong::ONE )      // Identity?
@@ -640,6 +644,8 @@ Node *DivFNode::Identity( PhaseTransform *phase ) {
 //------------------------------Idealize---------------------------------------
 Node *DivFNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (in(0) && remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   const Type *t2 = phase->type( in(2) );
   if( t2 == TypeF::ONE )         // Identity?
@@ -725,6 +731,8 @@ Node *DivDNode::Identity( PhaseTransform *phase ) {
 //------------------------------Idealize---------------------------------------
 Node *DivDNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (in(0) && remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   const Type *t2 = phase->type( in(2) );
   if( t2 == TypeD::ONE )         // Identity?
@@ -760,7 +768,9 @@ Node *DivDNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 //------------------------------Idealize---------------------------------------
 Node *ModINode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // Check for dead control input
-  if( remove_dead_region(phase, can_reshape) )  return this;
+  if( in(0) && remove_dead_region(phase, can_reshape) )  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   // Get the modulus
   const Type *t = phase->type( in(2) );
@@ -929,7 +939,9 @@ const Type *ModINode::Value( PhaseTransform *phase ) const {
 //------------------------------Idealize---------------------------------------
 Node *ModLNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // Check for dead control input
-  if( remove_dead_region(phase, can_reshape) )  return this;
+  if( in(0) && remove_dead_region(phase, can_reshape) )  return this;
+  // Don't bother trying to transform a dead node
+  if( in(0) && in(0)->is_top() )  return NULL;
 
   // Get the modulus
   const Type *t = phase->type( in(2) );

src/share/vm/opto/memnode.cpp

@@ -214,6 +214,9 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) {
   Node *ctl = in(MemNode::Control);
   if (ctl && remove_dead_region(phase, can_reshape))
     return this;
+  ctl = in(MemNode::Control);
+  // Don't bother trying to transform a dead node
+  if( ctl && ctl->is_top() )  return NodeSentinel;
 
   // Ignore if memory is dead, or self-loop
   Node *mem = in(MemNode::Memory);
@@ -244,6 +247,7 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) {
 
   if (mem != old_mem) {
     set_req(MemNode::Memory, mem);
+    if (phase->type( mem ) == Type::TOP) return NodeSentinel;
     return this;
   }
 
@@ -1316,6 +1320,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, phase);
     if (opt_mem != mem) {
       set_req(MemNode::Memory, opt_mem);
+      if (phase->type( opt_mem ) == Type::TOP) return NULL;
       return this;
     }
     const TypeOopPtr *t_oop = addr_t->isa_oopptr();
@@ -2447,8 +2452,7 @@ MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
 // Return a node which is more "ideal" than the current node.  Strip out
 // control copies
 Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
-  if (remove_dead_region(phase, can_reshape))  return this;
-  return NULL;
+  return remove_dead_region(phase, can_reshape) ? this : NULL;
 }
 
 //------------------------------Value------------------------------------------

src/share/vm/opto/node.cpp

@@ -1166,16 +1166,15 @@ bool Node::dominates(Node* sub, Node_List &nlist) {
 // using it dead as well.  This will happen normally via the usual IterGVN
 // worklist but this call is more efficient.  Do not update use-def info
 // inside the dead region, just at the borders.
-static bool kill_dead_code( Node *dead, PhaseIterGVN *igvn ) {
+static void kill_dead_code( Node *dead, PhaseIterGVN *igvn ) {
   // Con's are a popular node to re-hit in the hash table again.
-  if( dead->is_Con() ) return false;
+  if( dead->is_Con() ) return;
 
   // Can't put ResourceMark here since igvn->_worklist uses the same arena
   // for verify pass with +VerifyOpto and we add/remove elements in it here.
   Node_List  nstack(Thread::current()->resource_area());
 
   Node *top = igvn->C->top();
-  bool progress = false;
   nstack.push(dead);
 
   while (nstack.size() > 0) {
@@ -1214,7 +1213,6 @@ static bool kill_dead_code( Node *dead, PhaseIterGVN *igvn ) {
       for (uint i=0; i < dead->req(); i++) {
         Node *n = dead->in(i);      // Get input to dead guy
         if (n != NULL && !n->is_top()) { // Input is valid?
-          progress = true;
           dead->set_req(i, top);    // Smash input away
           if (n->outcnt() == 0) {   // Input also goes dead?
             if (!n->is_Con())
@@ -1233,7 +1231,7 @@ static bool kill_dead_code( Node *dead, PhaseIterGVN *igvn ) {
       }
     } // (dead->outcnt() == 0)
   }   // while (nstack.size() > 0) for outputs
-  return progress;
+  return;
 }
 
 //------------------------------remove_dead_region-----------------------------
@@ -1243,7 +1241,8 @@ bool Node::remove_dead_region(PhaseGVN *phase, bool can_reshape) {
   // Lost control into this guy?  I.e., it became unreachable?
   // Aggressively kill all unreachable code.
   if (can_reshape && n->is_top()) {
-    return kill_dead_code(this, phase->is_IterGVN());
+    kill_dead_code(this, phase->is_IterGVN());
+    return false; // Node is dead.
   }
 
   if( n->is_Region() && n->as_Region()->is_copy() ) {

src/share/vm/opto/phaseX.cpp

@@ -986,7 +986,9 @@ Node *PhaseIterGVN::transform_old( Node *n ) {
   // Apply the Ideal call in a loop until it no longer applies
   Node *k = n;
   DEBUG_ONLY(dead_loop_check(k);)
+  DEBUG_ONLY(bool is_new = (k->outcnt() == 0);)
   Node *i = k->Ideal(this, /*can_reshape=*/true);
+  assert(i != k || is_new || i->outcnt() > 0, "don't return dead nodes");
 #ifndef PRODUCT
   if( VerifyIterativeGVN )
     verify_step(k);
@@ -1024,7 +1026,9 @@ Node *PhaseIterGVN::transform_old( Node *n ) {
     }
     DEBUG_ONLY(dead_loop_check(k);)
     // Try idealizing again
+    DEBUG_ONLY(is_new = (k->outcnt() == 0);)
     i = k->Ideal(this, /*can_reshape=*/true);
+    assert(i != k || is_new || i->outcnt() > 0, "don't return dead nodes");
 #ifndef PRODUCT
     if( VerifyIterativeGVN )
       verify_step(k);