6916062: assert(_inserts <= _insert_limit,"hash table overflow") in NodeHash::hash_insert

Summary: Missing check for not empty worklist when puting memory node back on worklist and expecting address type update.
Reviewed-by: never

src/share/vm/opto/memnode.cpp

@@ -256,7 +256,8 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) {
   if( t_adr == Type::TOP )              return NodeSentinel; // caller will return NULL
 
   if( can_reshape && igvn != NULL &&
-      (igvn->_worklist.member(address) || phase->type(address) != adr_type()) ) {
+      (igvn->_worklist.member(address) ||
+       igvn->_worklist.size() > 0 && (phase->type(address) != adr_type())) ) {
     // The address's base and type may change when the address is processed.
     // Delay this mem node transformation until the address is processed.
     phase->is_IterGVN()->_worklist.push(this);

src/share/vm/opto/phaseX.cpp

@@ -844,10 +844,33 @@ void PhaseIterGVN::optimize() {
   }
 #endif
 
+#ifdef ASSERT
+  Node* prev = NULL;
+  uint rep_cnt = 0;
+#endif
+  uint loop_count = 0;
+
   // Pull from worklist; transform node;
   // If node has changed: update edge info and put uses on worklist.
   while( _worklist.size() ) {
     Node *n  = _worklist.pop();
+    if (++loop_count >= K * C->unique()) {
+      debug_only(n->dump(4);)
+      assert(false, "infinite loop in PhaseIterGVN::optimize");
+      C->record_method_not_compilable("infinite loop in PhaseIterGVN::optimize");
+      return;
+    }
+#ifdef ASSERT
+    if (n == prev) {
+      if (++rep_cnt > 3) {
+        n->dump(4);
+        assert(false, "loop in Ideal transformation");
+      }
+    } else {
+      rep_cnt = 0;
+    }
+    prev = n;
+#endif
     if (TraceIterativeGVN && Verbose) {
       tty->print("  Pop ");
       NOT_PRODUCT( n->dump(); )