6841313: G1: dirty cards of survivor regions in parallel

Reviewed-by: tonyp, iveresov

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -2739,8 +2739,6 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
         _in_cset_fast_test = NULL;
         _in_cset_fast_test_base = NULL;
 
-        release_gc_alloc_regions(false /* totally */);
-
         cleanup_surviving_young_words();
 
         if (g1_policy()->in_young_gc_mode()) {
@@ -4132,6 +4130,7 @@ void G1CollectedHeap::evacuate_collection_set() {
     G1KeepAliveClosure keep_alive(this);
     JNIHandles::weak_oops_do(&is_alive, &keep_alive);
   }
+  release_gc_alloc_regions(false /* totally */);
   g1_rem_set()->cleanup_after_oops_into_collection_set_do();
 
   concurrent_g1_refine()->clear_hot_cache();
@@ -4265,12 +4264,18 @@ void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRe
 class G1ParCleanupCTTask : public AbstractGangTask {
   CardTableModRefBS* _ct_bs;
   G1CollectedHeap* _g1h;
+  HeapRegion* volatile _so_head;
+  HeapRegion* volatile _su_head;
 public:
   G1ParCleanupCTTask(CardTableModRefBS* ct_bs,
-                     G1CollectedHeap* g1h) :
+                     G1CollectedHeap* g1h,
+                     HeapRegion* scan_only_list,
+                     HeapRegion* survivor_list) :
     AbstractGangTask("G1 Par Cleanup CT Task"),
     _ct_bs(ct_bs),
-    _g1h(g1h)
+    _g1h(g1h),
+    _so_head(scan_only_list),
+    _su_head(survivor_list)
   { }
 
   void work(int i) {
@@ -4278,22 +4283,64 @@ public:
     while (r = _g1h->pop_dirty_cards_region()) {
       clear_cards(r);
     }
+    // Redirty the cards of the scan-only and survivor regions.
+    dirty_list(&this->_so_head);
+    dirty_list(&this->_su_head);
   }
+
   void clear_cards(HeapRegion* r) {
     // Cards for Survivor and Scan-Only regions will be dirtied later.
     if (!r->is_scan_only() && !r->is_survivor()) {
       _ct_bs->clear(MemRegion(r->bottom(), r->end()));
     }
   }
+
+  void dirty_list(HeapRegion* volatile * head_ptr) {
+    HeapRegion* head;
+    do {
+      // Pop region off the list.
+      head = *head_ptr;
+      if (head != NULL) {
+        HeapRegion* r = (HeapRegion*)
+          Atomic::cmpxchg_ptr(head->get_next_young_region(), head_ptr, head);
+        if (r == head) {
+          assert(!r->isHumongous(), "Humongous regions shouldn't be on survivor list");
+          _ct_bs->dirty(MemRegion(r->bottom(), r->end()));
+        }
+      }
+    } while (*head_ptr != NULL);
+  }
 };
 
 
+#ifndef PRODUCT
+class G1VerifyCardTableCleanup: public HeapRegionClosure {
+  CardTableModRefBS* _ct_bs;
+public:
+  G1VerifyCardTableCleanup(CardTableModRefBS* ct_bs)
+    : _ct_bs(ct_bs)
+  { }
+  virtual bool doHeapRegion(HeapRegion* r)
+  {
+    MemRegion mr(r->bottom(), r->end());
+    if (r->is_scan_only() || r->is_survivor()) {
+      _ct_bs->verify_dirty_region(mr);
+    } else {
+      _ct_bs->verify_clean_region(mr);
+    }
+    return false;
+  }
+};
+#endif
+
 void G1CollectedHeap::cleanUpCardTable() {
   CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set());
   double start = os::elapsedTime();
 
   // Iterate over the dirty cards region list.
-  G1ParCleanupCTTask cleanup_task(ct_bs, this);
+  G1ParCleanupCTTask cleanup_task(ct_bs, this,
+                                  _young_list->first_scan_only_region(),
+                                  _young_list->first_survivor_region());
   if (ParallelGCThreads > 0) {
     set_par_threads(workers()->total_workers());
     workers()->run_task(&cleanup_task);
@@ -4309,18 +4356,22 @@ void G1CollectedHeap::cleanUpCardTable() {
       }
       r->set_next_dirty_cards_region(NULL);
     }
-  }
     // now, redirty the cards of the scan-only and survivor regions
     // (it seemed faster to do it this way, instead of iterating over
     // all regions and then clearing / dirtying as appropriate)
     dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region());
     dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region());
-
+  }
   double elapsed = os::elapsedTime() - start;
   g1_policy()->record_clear_ct_time( elapsed * 1000.0);
+#ifndef PRODUCT
+  if (G1VerifyCTCleanup || VerifyAfterGC) {
+    G1VerifyCardTableCleanup cleanup_verifier(ct_bs);
+    heap_region_iterate(&cleanup_verifier);
+  }
+#endif
 }
 
-
 void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) {
   if (g1_policy()->should_do_collection_pause(word_size)) {
     do_collection_pause();

src/share/vm/gc_implementation/g1/g1_globals.hpp

@@ -260,6 +260,9 @@
                                                                             \
   develop(intx, G1CardCountCacheExpandThreshold, 16,                        \
           "Expand the card count cache if the number of collisions for "    \
-          "a particular entry exceeds this value.")
+          "a particular entry exceeds this value.")                         \
+                                                                            \
+  develop(bool, G1VerifyCTCleanup, false,                                   \
+          "Verify card table cleanup.")
 
 G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG)

src/share/vm/memory/cardTableModRefBS.cpp

@@ -660,6 +660,29 @@ void CardTableModRefBS::verify_clean_region(MemRegion mr) {
   GuaranteeNotModClosure blk(this);
   non_clean_card_iterate_work(mr, &blk, false);
 }
+
+// To verify a MemRegion is entirely dirty this closure is passed to
+// dirty_card_iterate. If the region is dirty do_MemRegion will be
+// invoked only once with a MemRegion equal to the one being
+// verified.
+class GuaranteeDirtyClosure: public MemRegionClosure {
+  CardTableModRefBS* _ct;
+  MemRegion _mr;
+  bool _result;
+public:
+  GuaranteeDirtyClosure(CardTableModRefBS* ct, MemRegion mr)
+    : _ct(ct), _mr(mr), _result(false) {}
+  void do_MemRegion(MemRegion mr) {
+    _result = _mr.equals(mr);
+  }
+  bool result() const { return _result; }
+};
+
+void CardTableModRefBS::verify_dirty_region(MemRegion mr) {
+  GuaranteeDirtyClosure blk(this, mr);
+  dirty_card_iterate(mr, &blk);
+  guarantee(blk.result(), "Non-dirty cards in region that should be dirty");
+}
 #endif
 
 bool CardTableModRefBSForCTRS::card_will_be_scanned(jbyte cv) {

src/share/vm/memory/cardTableModRefBS.hpp

@@ -456,6 +456,7 @@ public:
   void verify_guard();
 
   void verify_clean_region(MemRegion mr) PRODUCT_RETURN;
+  void verify_dirty_region(MemRegion mr) PRODUCT_RETURN;
 
   static size_t par_chunk_heapword_alignment() {
     return CardsPerStrideChunk * card_size_in_words;