6988439: Parallel Class Loading test deadlock involving MethodData_lock and Pending List Lock

Summary: Don't acquire methodData_lock while holding pending list lock
Reviewed-by: kvn, never, ysr

src/share/vm/oops/instanceRefKlass.cpp

@@ -457,6 +457,11 @@ void instanceRefKlass::oop_verify_on(oop obj, outputStream* st) {
   }
 }
 
+bool instanceRefKlass::owns_pending_list_lock(JavaThread* thread) {
+  Handle h_lock(thread, java_lang_ref_Reference::pending_list_lock());
+  return ObjectSynchronizer::current_thread_holds_lock(thread, h_lock);
+}
+
 void instanceRefKlass::acquire_pending_list_lock(BasicLock *pending_list_basic_lock) {
   // we may enter this with pending exception set
   PRESERVE_EXCEPTION_MARK;  // exceptions are never thrown, needed for TRAPS argument

src/share/vm/oops/instanceRefKlass.hpp

@@ -89,6 +89,7 @@ class instanceRefKlass: public instanceKlass {
 
   static void release_and_notify_pending_list_lock(BasicLock *pending_list_basic_lock);
   static void acquire_pending_list_lock(BasicLock *pending_list_basic_lock);
+  static bool owns_pending_list_lock(JavaThread* thread);
 
   // Update non-static oop maps so 'referent', 'nextPending' and
   // 'discovered' will look like non-oops

src/share/vm/oops/methodOop.cpp

@@ -309,6 +309,12 @@ void methodOopDesc::print_invocation_count() {
 // Build a methodDataOop object to hold information about this method
 // collected in the interpreter.
 void methodOopDesc::build_interpreter_method_data(methodHandle method, TRAPS) {
+  // Do not profile method if current thread holds the pending list lock,
+  // which avoids deadlock for acquiring the MethodData_lock.
+  if (instanceRefKlass::owns_pending_list_lock((JavaThread*)THREAD)) {
+    return;
+  }
+
   // Grab a lock here to prevent multiple
   // methodDataOops from being created.
   MutexLocker ml(MethodData_lock, THREAD);