7011379: G1: overly long concurrent marking cycles

Summary: This changeset introduces filtering of SATB buffers at the point when they are about to be enqueued. If this filtering clears enough entries on each buffer, the buffer can then be re-used and not enqueued. This cuts down the number of SATB buffers that need to be processed by the concurrent marking threads.
Reviewed-by: johnc, ysr

src/share/vm/gc_implementation/g1/concurrentMark.cpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1171,12 +1171,12 @@ void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
     if (G1TraceMarkStackOverflow)
       gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
   } else {
+    SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
     // We're done with marking.
     // This is the end of  the marking cycle, we're expected all
     // threads to have SATB queues with active set to true.
-    JavaThread::satb_mark_queue_set().set_active_all_threads(
-                                                  false, /* new active value */
-                                                  true /* expected_active */);
+    satb_mq_set.set_active_all_threads(false, /* new active value */
+                                       true /* expected_active */);
 
     if (VerifyDuringGC) {
       HandleMark hm;  // handle scope

src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp

@@ -28,7 +28,7 @@
 #include "gc_implementation/g1/concurrentMark.hpp"
 #include "gc_implementation/g1/g1CollectedHeap.hpp"
 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
-#include "gc_implementation/g1/heapRegionSeq.hpp"
+#include "gc_implementation/g1/heapRegionSeq.inline.hpp"
 #include "utilities/taskqueue.hpp"
 
 // Inline functions for G1CollectedHeap

src/share/vm/gc_implementation/g1/g1_globals.hpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -99,6 +99,13 @@
   develop(intx, G1SATBProcessCompletedThreshold, 20,                        \
           "Number of completed buffers that triggers log processing.")      \
                                                                             \
+  product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60,                \
+          "Before enqueueing them, each mutator thread tries to do some "   \
+          "filtering on the SATB buffers it generates. If post-filtering "  \
+          "the percentage of retained entries is over this threshold "      \
+          "the buffer will be enqueued for processing. A value of 0 "       \
+          "specifies that mutator threads should not do such filtering.")   \
+                                                                            \
   develop(intx, G1ExtraRegionSurvRate, 33,                                  \
           "If the young survival rate is S, and there's room left in "      \
           "to-space, we will allow regions whose survival rate is up to "   \

src/share/vm/gc_implementation/g1/heapRegionSeq.hpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -106,7 +106,7 @@ class HeapRegionSeq: public CHeapObj {
 
   // If "addr" falls within a region in the sequence, return that region,
   // or else NULL.
-  HeapRegion* addr_to_region(const void* addr);
+  inline HeapRegion* addr_to_region(const void* addr);
 
   void print();
 

src/share/vm/gc_implementation/g1/ptrQueue.cpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,8 +38,8 @@
 # include "thread_windows.inline.hpp"
 #endif
 
-PtrQueue::PtrQueue(PtrQueueSet* qset_, bool perm, bool active) :
-  _qset(qset_), _buf(NULL), _index(0), _active(active),
+PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) :
+  _qset(qset), _buf(NULL), _index(0), _active(active),
   _perm(perm), _lock(NULL)
 {}
 
@@ -153,10 +153,16 @@ void PtrQueueSet::reduce_free_list() {
 }
 
 void PtrQueue::handle_zero_index() {
-  assert(0 == _index, "Precondition.");
+  assert(_index == 0, "Precondition.");
+
   // This thread records the full buffer and allocates a new one (while
   // holding the lock if there is one).
   if (_buf != NULL) {
+    if (!should_enqueue_buffer()) {
+      assert(_index > 0, "the buffer can only be re-used if it's not full");
+      return;
+    }
+
     if (_lock) {
       assert(_lock->owned_by_self(), "Required.");
 

src/share/vm/gc_implementation/g1/ptrQueue.hpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -68,7 +68,7 @@ protected:
 public:
   // Initialize this queue to contain a null buffer, and be part of the
   // given PtrQueueSet.
-  PtrQueue(PtrQueueSet*, bool perm = false, bool active = false);
+  PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false);
   // Release any contained resources.
   void flush();
   // Calls flush() when destroyed.
@@ -85,6 +85,14 @@ public:
     else enqueue_known_active(ptr);
   }
 
+  // This method is called when we're doing the zero index handling
+  // and gives a chance to the queues to do any pre-enqueueing
+  // processing they might want to do on the buffer. It should return
+  // true if the buffer should be enqueued, or false if enough
+  // entries were cleared from it so that it can be re-used. It should
+  // not return false if the buffer is still full (otherwise we can
+  // get into an infinite loop).
+  virtual bool should_enqueue_buffer() { return true; }
   void handle_zero_index();
   void locking_enqueue_completed_buffer(void** buf);
 

src/share/vm/gc_implementation/g1/satbQueue.cpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,98 @@
  */
 
 #include "precompiled.hpp"
+#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
 #include "gc_implementation/g1/satbQueue.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/sharedHeap.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/thread.hpp"
 
+// This method removes entries from an SATB buffer that will not be
+// useful to the concurrent marking threads. An entry is removed if it
+// satisfies one of the following conditions:
+//
+// * it points to an object outside the G1 heap (G1's concurrent
+//     marking only visits objects inside the G1 heap),
+// * it points to an object that has been allocated since marking
+//     started (according to SATB those objects do not need to be
+//     visited during marking), or
+// * it points to an object that has already been marked (no need to
+//     process it again).
+//
+// The rest of the entries will be retained and are compacted towards
+// the top of the buffer. If with this filtering we clear a large
+// enough chunk of the buffer we can re-use it (instead of enqueueing
+// it) and we can just allow the mutator to carry on executing.
+
+bool ObjPtrQueue::should_enqueue_buffer() {
+  assert(_lock == NULL || _lock->owned_by_self(),
+         "we should have taken the lock before calling this");
+
+  // A value of 0 means "don't filter SATB buffers".
+  if (G1SATBBufferEnqueueingThresholdPercent == 0) {
+    return true;
+  }
+
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+  // This method should only be called if there is a non-NULL buffer
+  // that is full.
+  assert(_index == 0, "pre-condition");
+  assert(_buf != NULL, "pre-condition");
+
+  void** buf = _buf;
+  size_t sz = _sz;
+
+  // Used for sanity checking at the end of the loop.
+  debug_only(size_t entries = 0; size_t retained = 0;)
+
+  size_t i = sz;
+  size_t new_index = sz;
+
+  // Given that we are expecting _index == 0, we could have changed
+  // the loop condition to (i > 0). But we are using _index for
+  // generality.
+  while (i > _index) {
+    assert(i > 0, "we should have at least one more entry to process");
+    i -= oopSize;
+    debug_only(entries += 1;)
+    oop* p = (oop*) &buf[byte_index_to_index((int) i)];
+    oop obj = *p;
+    // NULL the entry so that unused parts of the buffer contain NULLs
+    // at the end. If we are going to retain it we will copy it to its
+    // final place. If we have retained all entries we have visited so
+    // far, we'll just end up copying it to the same place.
+    *p = NULL;
+
+    bool retain = g1h->is_obj_ill(obj);
+    if (retain) {
+      assert(new_index > 0, "we should not have already filled up the buffer");
+      new_index -= oopSize;
+      assert(new_index >= i,
+             "new_index should never be below i, as we alwaysr compact 'up'");
+      oop* new_p = (oop*) &buf[byte_index_to_index((int) new_index)];
+      assert(new_p >= p, "the destination location should never be below "
+             "the source as we always compact 'up'");
+      assert(*new_p == NULL,
+             "we should have already cleared the destination location");
+      *new_p = obj;
+      debug_only(retained += 1;)
+    }
+  }
+  size_t entries_calc = (sz - _index) / oopSize;
+  assert(entries == entries_calc, "the number of entries we counted "
+         "should match the number of entries we calculated");
+  size_t retained_calc = (sz - new_index) / oopSize;
+  assert(retained == retained_calc, "the number of retained entries we counted "
+         "should match the number of retained entries we calculated");
+  size_t perc = retained_calc * 100 / entries_calc;
+  bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent;
+  _index = new_index;
+
+  return should_enqueue;
+}
+
 void ObjPtrQueue::apply_closure(ObjectClosure* cl) {
   if (_buf != NULL) {
     apply_closure_to_buffer(cl, _buf, _index, _sz);

src/share/vm/gc_implementation/g1/satbQueue.hpp

 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,13 +33,18 @@ class JavaThread;
 // A ptrQueue whose elements are "oops", pointers to object heads.
 class ObjPtrQueue: public PtrQueue {
 public:
-  ObjPtrQueue(PtrQueueSet* qset_, bool perm = false) :
+  ObjPtrQueue(PtrQueueSet* qset, bool perm = false) :
     // SATB queues are only active during marking cycles. We create
     // them with their active field set to false. If a thread is
     // created during a cycle and its SATB queue needs to be activated
     // before the thread starts running, we'll need to set its active
     // field to true. This is done in JavaThread::initialize_queues().
-    PtrQueue(qset_, perm, false /* active */) { }
+    PtrQueue(qset, perm, false /* active */) { }
+
+  // Overrides PtrQueue::should_enqueue_buffer(). See the method's
+  // definition for more information.
+  virtual bool should_enqueue_buffer();
+
   // Apply the closure to all elements, and reset the index to make the
   // buffer empty.
   void apply_closure(ObjectClosure* cl);