8056154: JVM crash with EXCEPTION_ACCESS_VIOLATION when there are many threads running

Summary: Don't make compiled MH intrinsics not entrant when redefining classes
Reviewed-by: kvn, vlivanov

src/share/vm/classfile/systemDictionary.cpp

@@ -2291,6 +2291,9 @@ methodHandle SystemDictionary::find_method_handle_intrinsic(vmIntrinsics::ID iid
   }
 
   assert(spe != NULL && spe->method() != NULL, "");
+  assert(!UseCompiler || (spe->method()->has_compiled_code() &&
+         spe->method()->code()->entry_point() == spe->method()->from_compiled_entry()),
+         "MH intrinsic invariant");
   return spe->method();
 }
 

src/share/vm/code/codeCache.cpp

@@ -712,7 +712,9 @@ int CodeCache::mark_for_evol_deoptimization(instanceKlassHandle dependee) {
 void CodeCache::mark_all_nmethods_for_deoptimization() {
   MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
   FOR_ALL_ALIVE_NMETHODS(nm) {
-    nm->mark_for_deoptimization();
+    if (!nm->method()->is_method_handle_intrinsic()) {
+      nm->mark_for_deoptimization();
+    }
   }
 }
 

src/share/vm/code/compiledIC.cpp

@@ -595,6 +595,7 @@ void CompiledStaticCall::compute_entry(methodHandle m, StaticCallInfo& info) {
   } else {
     // Callee is interpreted code.  In any case entering the interpreter
     // puts a converter-frame on the stack to save arguments.
+    assert(!m->is_method_handle_intrinsic(), "Compiled code should never call interpreter MH intrinsics");
     info._to_interpreter = true;
     info._entry      = m()->get_c2i_entry();
   }

src/share/vm/code/nmethod.hpp

@@ -448,7 +448,10 @@ class nmethod : public CodeBlob {
   // alive.  It is used when an uncommon trap happens.  Returns true
   // if this thread changed the state of the nmethod or false if
   // another thread performed the transition.
-  bool  make_not_entrant() { return make_not_entrant_or_zombie(not_entrant); }
+  bool  make_not_entrant() {
+    assert(!method()->is_method_handle_intrinsic(), "Cannot make MH intrinsic not entrant");
+    return make_not_entrant_or_zombie(not_entrant);
+  }
   bool  make_zombie()      { return make_not_entrant_or_zombie(zombie); }
 
   // used by jvmti to track if the unload event has been reported