8037112: gc/g1/TestHumongousAllocInitialMark.java caused SIGSEGV

Reviewed-by: brutisso, mgerdin

src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp

@@ -89,6 +89,10 @@ void ConcurrentMarkThread::run() {
   while (!_should_terminate) {
     // wait until started is set.
     sleepBeforeNextCycle();
+    if (_should_terminate) {
+      break;
+    }
+
     {
       ResourceMark rm;
       HandleMark   hm;
@@ -303,11 +307,21 @@ void ConcurrentMarkThread::yield() {
 }
 
 void ConcurrentMarkThread::stop() {
-  // it is ok to take late safepoints here, if needed
-  MutexLockerEx mu(Terminator_lock);
-  _should_terminate = true;
-  while (!_has_terminated) {
-    Terminator_lock->wait();
+  {
+    MutexLockerEx ml(Terminator_lock);
+    _should_terminate = true;
+  }
+
+  {
+    MutexLockerEx ml(CGC_lock, Mutex::_no_safepoint_check_flag);
+    CGC_lock->notify_all();
+  }
+
+  {
+    MutexLockerEx ml(Terminator_lock);
+    while (!_has_terminated) {
+      Terminator_lock->wait();
+    }
   }
 }
 
@@ -327,11 +341,14 @@ void ConcurrentMarkThread::sleepBeforeNextCycle() {
   assert(!in_progress(), "should have been cleared");
 
   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
-  while (!started()) {
+  while (!started() && !_should_terminate) {
     CGC_lock->wait(Mutex::_no_safepoint_check_flag);
   }
-  set_in_progress();
-  clear_started();
+
+  if (started()) {
+    set_in_progress();
+    clear_started();
+  }
 }
 
 // Note: As is the case with CMS - this method, although exported

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -435,6 +435,9 @@ HeapRegion* G1CollectedHeap::pop_dirty_cards_region()
 void G1CollectedHeap::stop_conc_gc_threads() {
   _cg1r->stop();
   _cmThread->stop();
+  if (G1StringDedup::is_enabled()) {
+    G1StringDedup::stop();
+  }
 }
 
 #ifdef ASSERT
@@ -2182,6 +2185,16 @@ jint G1CollectedHeap::initialize() {
   return JNI_OK;
 }
 
+void G1CollectedHeap::stop() {
+  // Abort any ongoing concurrent root region scanning and stop all
+  // concurrent threads. We do this to make sure these threads do
+  // not continue to execute and access resources (e.g. gclog_or_tty)
+  // that are destroyed during shutdown.
+  _cm->root_regions()->abort();
+  _cm->root_regions()->wait_until_scan_finished();
+  stop_conc_gc_threads();
+}
+
 size_t G1CollectedHeap::conservative_max_heap_alignment() {
   return HeapRegion::max_region_size();
 }

src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

@@ -1082,6 +1082,8 @@ public:
   // specified by the policy object.
   jint initialize();
 
+  virtual void stop();
+
   // Return the (conservative) maximum heap alignment for any G1 heap
   static size_t conservative_max_heap_alignment();
 

src/share/vm/gc_implementation/g1/g1StringDedup.cpp

@@ -44,6 +44,11 @@ void G1StringDedup::initialize() {
   }
 }
 
+void G1StringDedup::stop() {
+  assert(is_enabled(), "String deduplication not enabled");
+  G1StringDedupThread::stop();
+}
+
 bool G1StringDedup::is_candidate_from_mark(oop obj) {
   if (java_lang_String::is_instance(obj)) {
     bool from_young = G1CollectedHeap::heap()->heap_region_containing_raw(obj)->is_young();

src/share/vm/gc_implementation/g1/g1StringDedup.hpp

@@ -110,8 +110,12 @@ public:
     return _enabled;
   }
 
+  // Initialize string deduplication.
   static void initialize();
 
+  // Stop the deduplication thread.
+  static void stop();
+
   // Immediately deduplicates the given String object, bypassing the
   // the deduplication queue.
   static void deduplicate(oop java_string);

src/share/vm/gc_implementation/g1/g1StringDedupQueue.cpp

@@ -35,6 +35,7 @@ const size_t        G1StringDedupQueue::_max_cache_size = 0; // Max cache size p
 
 G1StringDedupQueue::G1StringDedupQueue() :
   _cursor(0),
+  _cancel(false),
   _empty(true),
   _dropped(0) {
   _nqueues = MAX2(ParallelGCThreads, (size_t)1);
@@ -55,11 +56,17 @@ void G1StringDedupQueue::create() {
 
 void G1StringDedupQueue::wait() {
   MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag);
-  while (_queue->_empty) {
+  while (_queue->_empty && !_queue->_cancel) {
     ml.wait(Mutex::_no_safepoint_check_flag);
   }
 }
 
+void G1StringDedupQueue::cancel_wait() {
+  MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag);
+  _queue->_cancel = true;
+  ml.notify();
+}
+
 void G1StringDedupQueue::push(uint worker_id, oop java_string) {
   assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
   assert(worker_id < _queue->_nqueues, "Invalid queue");

src/share/vm/gc_implementation/g1/g1StringDedupQueue.hpp

@@ -65,6 +65,7 @@ private:
   G1StringDedupWorkerQueue*  _queues;
   size_t                     _nqueues;
   size_t                     _cursor;
+  bool                       _cancel;
   volatile bool              _empty;
 
   // Statistics counter, only used for logging.
@@ -81,6 +82,9 @@ public:
   // Blocks and waits for the queue to become non-empty.
   static void wait();
 
+  // Wakes up any thread blocked waiting for the queue to become non-empty.
+  static void cancel_wait();
+
   // Pushes a deduplication candidate onto a specific GC worker queue.
   static void push(uint worker_id, oop java_string);
 

src/share/vm/gc_implementation/g1/g1StringDedupThread.cpp

@@ -73,6 +73,9 @@ void G1StringDedupThread::run() {
 
     // Wait for the queue to become non-empty
     G1StringDedupQueue::wait();
+    if (_should_terminate) {
+      break;
+    }
 
     // Include this thread in safepoints
     stsJoin();
@@ -108,7 +111,23 @@ void G1StringDedupThread::run() {
     stsLeave();
   }
 
-  ShouldNotReachHere();
+  terminate();
+}
+
+void G1StringDedupThread::stop() {
+  {
+    MonitorLockerEx ml(Terminator_lock);
+    _thread->_should_terminate = true;
+  }
+
+  G1StringDedupQueue::cancel_wait();
+
+  {
+    MonitorLockerEx ml(Terminator_lock);
+    while (!_thread->_has_terminated) {
+      ml.wait();
+    }
+  }
 }
 
 void G1StringDedupThread::print(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat) {

src/share/vm/gc_implementation/g1/g1StringDedupThread.hpp

@@ -47,6 +47,8 @@ private:
 
 public:
   static void create();
+  static void stop();
+
   static G1StringDedupThread* thread();
 
   virtual void run();

src/share/vm/gc_interface/collectedHeap.hpp

@@ -208,6 +208,9 @@ class CollectedHeap : public CHeapObj<mtInternal> {
   // This is the correct place to place such initialization methods.
   virtual void post_initialize() = 0;
 
+  // Stop any onging concurrent work and prepare for exit.
+  virtual void stop() {}
+
   MemRegion reserved_region() const { return _reserved; }
   address base() const { return (address)reserved_region().start(); }
 

src/share/vm/runtime/java.cpp

@@ -497,6 +497,9 @@ void before_exit(JavaThread * thread) {
     os::infinite_sleep();
   }
 
+  // Stop any ongoing concurrent GC work
+  Universe::heap()->stop();
+
   // Terminate watcher thread - must before disenrolling any periodic task
   if (PeriodicTask::num_tasks() > 0)
     WatcherThread::stop();