6983311: G1: LoopTest hangs when run with -XX:+ExplicitInvokesConcurrent

Summary: Clear the concurrent marking "in progress" flag while the FullGCCount_lock is held. This avoids a race that can cause back to back System.gc() calls, when ExplicitGCInvokesConcurrent is enabled, to fail to initiate a marking cycle causing the requesting thread to hang.
Reviewed-by: tonyp, ysr

src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp

@@ -303,9 +303,10 @@ void ConcurrentMarkThread::print_on(outputStream* st) const {
 }
 
 void ConcurrentMarkThread::sleepBeforeNextCycle() {
-  clear_in_progress();
   // We join here because we don't want to do the "shouldConcurrentMark()"
   // below while the world is otherwise stopped.
+  assert(!in_progress(), "should have been cleared");
+
   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
   while (!started()) {
     CGC_lock->wait(Mutex::_no_safepoint_check_flag);

src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp

@@ -69,12 +69,12 @@ class ConcurrentMarkThread: public ConcurrentGCThread {
 
   ConcurrentMark* cm()     { return _cm; }
 
-  void set_started()       { _started = true;  }
-  void clear_started()     { _started = false; }
+  void set_started()       { assert(!_in_progress, "cycle in progress"); _started = true;  }
+  void clear_started()     { assert(_in_progress, "must be starting a cycle"); _started = false; }
   bool started()           { return _started;  }
 
-  void set_in_progress()   { _in_progress = true;  }
-  void clear_in_progress() { _in_progress = false; }
+  void set_in_progress()   { assert(_started, "must be starting a cycle"); _in_progress = true;  }
+  void clear_in_progress() { assert(!_started, "must not be starting a new cycle"); _in_progress = false; }
   bool in_progress()       { return _in_progress;  }
 
   // This flag returns true from the moment a marking cycle is

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -1785,6 +1785,14 @@ void G1CollectedHeap::increment_full_collections_completed(bool outer) {
 
   _full_collections_completed += 1;
 
+  // We need to clear the "in_progress" flag in the CM thread before
+  // we wake up any waiters (especially when ExplicitInvokesConcurrent
+  // is set) so that if a waiter requests another System.gc() it doesn't
+  // incorrectly see that a marking cyle is still in progress.
+  if (outer) {
+    _cmThread->clear_in_progress();
+  }
+
   // This notify_all() will ensure that a thread that called
   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)
   // and it's waiting for a full GC to finish will be woken up. It is