7158682: G1: Handle leak when running nsk.sysdict tests

Summary: Place HandleMarks in the code that allocates handles for the pending list lock so that the handles are freed and multiple, unsuccessful, attempts to schedule a GC do not cause an OOM.
Reviewed-by: brutisso

src/share/vm/oops/instanceRefKlass.cpp

@@ -515,6 +515,12 @@ bool instanceRefKlass::owns_pending_list_lock(JavaThread* thread) {
 void instanceRefKlass::acquire_pending_list_lock(BasicLock *pending_list_basic_lock) {
   // we may enter this with pending exception set
   PRESERVE_EXCEPTION_MARK;  // exceptions are never thrown, needed for TRAPS argument
+
+  // Create a HandleMark in case we retry a GC multiple times.
+  // Each time we attempt the GC, we allocate the handle below
+  // to hold the pending list lock. We want to free this handle.
+  HandleMark hm;
+
   Handle h_lock(THREAD, java_lang_ref_Reference::pending_list_lock());
   ObjectSynchronizer::fast_enter(h_lock, pending_list_basic_lock, false, THREAD);
   assert(ObjectSynchronizer::current_thread_holds_lock(
@@ -527,7 +533,12 @@ void instanceRefKlass::release_and_notify_pending_list_lock(
   BasicLock *pending_list_basic_lock) {
   // we may enter this with pending exception set
   PRESERVE_EXCEPTION_MARK;  // exceptions are never thrown, needed for TRAPS argument
-  //
+
+  // Create a HandleMark in case we retry a GC multiple times.
+  // Each time we attempt the GC, we allocate the handle below
+  // to hold the pending list lock. We want to free this handle.
+  HandleMark hm;
+
   Handle h_lock(THREAD, java_lang_ref_Reference::pending_list_lock());
   assert(ObjectSynchronizer::current_thread_holds_lock(
            JavaThread::current(), h_lock),