6820167: GCALotAtAllSafepoints + FullGCALot(ScavengeALot) options crash JVM

Summary: Short-circuit gc-a-lot attempts by non-JavaThreads; SkipGCALot c'tor to elide re-entrant gc-a-lot attempts.
Reviewed-by: apetrusenko, jcoomes, jmasa, kamg

src/share/vm/memory/gcLocker.hpp

@@ -242,6 +242,31 @@ class Pause_No_Safepoint_Verifier : public Pause_No_GC_Verifier {
 #endif
 };
 
+// A SkipGCALot object is used to elide the usual effect of gc-a-lot
+// over a section of execution by a thread. Currently, it's used only to
+// prevent re-entrant calls to GC.
+class SkipGCALot : public StackObj {
+  private:
+   bool _saved;
+   Thread* _t;
+
+  public:
+#ifdef ASSERT
+    SkipGCALot(Thread* t) : _t(t) {
+      _saved = _t->skip_gcalot();
+      _t->set_skip_gcalot(true);
+    }
+
+    ~SkipGCALot() {
+      assert(_t->skip_gcalot(), "Save-restore protocol invariant");
+      _t->set_skip_gcalot(_saved);
+    }
+#else
+    SkipGCALot(Thread* t) { }
+    ~SkipGCALot() { }
+#endif
+};
+
 // JRT_LEAF currently can be called from either _thread_in_Java or
 // _thread_in_native mode. In _thread_in_native, it is ok
 // for another thread to trigger GC. The rest of the JRT_LEAF

src/share/vm/runtime/interfaceSupport.cpp

@@ -66,11 +66,14 @@ void InterfaceSupport::trace(const char* result_type, const char* header) {
 
 void InterfaceSupport::gc_alot() {
   Thread *thread = Thread::current();
-  if (thread->is_VM_thread()) return; // Avoid concurrent calls
+  if (!thread->is_Java_thread()) return; // Avoid concurrent calls
   // Check for new, not quite initialized thread. A thread in new mode cannot initiate a GC.
   JavaThread *current_thread = (JavaThread *)thread;
   if (current_thread->active_handles() == NULL) return;
 
+  // Short-circuit any possible re-entrant gc-a-lot attempt
+  if (thread->skip_gcalot()) return;
+
   if (is_init_completed()) {
 
     if (++_fullgc_alot_invocation < FullGCALotStart) {

src/share/vm/runtime/thread.cpp

@@ -127,6 +127,7 @@ Thread::Thread() {
   debug_only(_owned_locks = NULL;)
   debug_only(_allow_allocation_count = 0;)
   NOT_PRODUCT(_allow_safepoint_count = 0;)
+  NOT_PRODUCT(_skip_gcalot = false;)
   CHECK_UNHANDLED_OOPS_ONLY(_gc_locked_out_count = 0;)
   _jvmti_env_iteration_count = 0;
   _vm_operation_started_count = 0;
@@ -784,7 +785,6 @@ void Thread::check_for_valid_safepoint_state(bool potential_vm_operation) {
       // We could enter a safepoint here and thus have a gc
       InterfaceSupport::check_gc_alot();
     }
-
 #endif
 }
 #endif

src/share/vm/runtime/thread.hpp

@@ -191,6 +191,9 @@ class Thread: public ThreadShadow {
   NOT_PRODUCT(int _allow_safepoint_count;)       // If 0, thread allow a safepoint to happen
   debug_only (int _allow_allocation_count;)      // If 0, the thread is allowed to allocate oops.
 
+  // Used by SkipGCALot class.
+  NOT_PRODUCT(bool _skip_gcalot;)                // Should we elide gc-a-lot?
+
   // Record when GC is locked out via the GC_locker mechanism
   CHECK_UNHANDLED_OOPS_ONLY(int _gc_locked_out_count;)
 
@@ -308,6 +311,11 @@ class Thread: public ThreadShadow {
   bool is_gc_locked_out() { return _gc_locked_out_count > 0; }
 #endif // CHECK_UNHANDLED_OOPS
 
+#ifndef PRODUCT
+  bool skip_gcalot()           { return _skip_gcalot; }
+  void set_skip_gcalot(bool v) { _skip_gcalot = v;    }
+#endif
+
  public:
   // Installs a pending exception to be inserted later
   static void send_async_exception(oop thread_oop, oop java_throwable);

src/share/vm/runtime/vmThread.cpp

@@ -531,6 +531,7 @@ void VMThread::execute(VM_Operation* op) {
   Thread* t = Thread::current();
 
   if (!t->is_VM_thread()) {
+    SkipGCALot sgcalot(t);    // avoid re-entrant attempts to gc-a-lot
     // JavaThread or WatcherThread
     t->check_for_valid_safepoint_state(true);