8048179: Early reclaim of large objects that are referenced by a few objects

Summary: Push the remembered sets of large objects with few referenced into the dirty card queue at the beginning of the evacuation so that they may end up with zero remembered set entries at the end of the collection, and are potentially reclaimed. Also improve timing measurements of the early reclaim mechanism, and shorten flag names.
Reviewed-by: brutisso, jmasa, dfazunen

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -2142,7 +2142,7 @@ void G1CollectedHeap::stop() {
 }
 
 void G1CollectedHeap::clear_humongous_is_live_table() {
-  guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Should only be called if true");
+  guarantee(G1EagerReclaimHumongousObjects, "Should only be called if true");
   _humongous_is_live.clear();
 }
 
@@ -3676,8 +3676,24 @@ class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
  private:
   size_t _total_humongous;
   size_t _candidate_humongous;
+
+  DirtyCardQueue _dcq;
+
+  bool humongous_region_is_candidate(uint index) {
+    HeapRegion* region = G1CollectedHeap::heap()->region_at(index);
+    assert(region->startsHumongous(), "Must start a humongous object");
+    HeapRegionRemSet* const rset = region->rem_set();
+    bool const allow_stale_refs = G1EagerReclaimHumongousObjectsWithStaleRefs;
+    return !oop(region->bottom())->is_objArray() &&
+           ((allow_stale_refs && rset->occupancy_less_or_equal_than(G1RSetSparseRegionEntries)) ||
+            (!allow_stale_refs && rset->is_empty()));
+  }
+
  public:
-  RegisterHumongousWithInCSetFastTestClosure() : _total_humongous(0), _candidate_humongous(0) {
+  RegisterHumongousWithInCSetFastTestClosure()
+  : _total_humongous(0),
+    _candidate_humongous(0),
+    _dcq(&JavaThread::dirty_card_queue_set()) {
   }
 
   virtual bool doHeapRegion(HeapRegion* r) {
@@ -3687,11 +3703,29 @@ class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
     G1CollectedHeap* g1h = G1CollectedHeap::heap();
 
     uint region_idx = r->hrm_index();
-    bool is_candidate = !g1h->humongous_region_is_always_live(region_idx);
-    // Is_candidate already filters out humongous regions with some remembered set.
-    // This will not lead to humongous object that we mistakenly keep alive because
-    // during young collection the remembered sets will only be added to.
+    bool is_candidate = humongous_region_is_candidate(region_idx);
+    // Is_candidate already filters out humongous object with large remembered sets.
+    // If we have a humongous object with a few remembered sets, we simply flush these
+    // remembered set entries into the DCQS. That will result in automatic
+    // re-evaluation of their remembered set entries during the following evacuation
+    // phase.
     if (is_candidate) {
+      if (!r->rem_set()->is_empty()) {
+        guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries),
+                  "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
+        G1SATBCardTableLoggingModRefBS* bs = g1h->g1_barrier_set();
+        HeapRegionRemSetIterator hrrs(r->rem_set());
+        size_t card_index;
+        while (hrrs.has_next(card_index)) {
+          jbyte* card_ptr = (jbyte*)bs->byte_for_index(card_index);
+          if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
+            *card_ptr = CardTableModRefBS::dirty_card_val();
+            _dcq.enqueue(card_ptr);
+          }
+        }
+        r->rem_set()->clear_locked();
+      }
+      assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
       g1h->register_humongous_region_with_in_cset_fast_test(region_idx);
       _candidate_humongous++;
     }
@@ -3702,23 +3736,32 @@ class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
 
   size_t total_humongous() const { return _total_humongous; }
   size_t candidate_humongous() const { return _candidate_humongous; }
+
+  void flush_rem_set_entries() { _dcq.flush(); }
 };
 
 void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() {
-  if (!G1ReclaimDeadHumongousObjectsAtYoungGC) {
-    g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0, 0);
+  if (!G1EagerReclaimHumongousObjects) {
+    g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0);
     return;
   }
+  double time = os::elapsed_counter();
 
   RegisterHumongousWithInCSetFastTestClosure cl;
   heap_region_iterate(&cl);
-  g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(cl.total_humongous(),
+
+  time = ((double)(os::elapsed_counter() - time) / os::elapsed_frequency()) * 1000.0;
+  g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(time,
+                                                                  cl.total_humongous(),
                                                                   cl.candidate_humongous());
   _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0;
 
-  if (_has_humongous_reclaim_candidates || G1TraceReclaimDeadHumongousObjectsAtYoungGC) {
+  if (_has_humongous_reclaim_candidates || G1TraceEagerReclaimHumongousObjects) {
     clear_humongous_is_live_table();
   }
+
+  // Finally flush all remembered set entries to re-check into the global DCQS.
+  cl.flush_rem_set_entries();
 }
 
 void
@@ -6273,22 +6316,20 @@ class G1FreeHumongousRegionClosure : public HeapRegionClosure {
     // are completely up-to-date wrt to references to the humongous object.
     //
     // Other implementation considerations:
-    // - never consider object arrays: while they are a valid target, they have not
-    // been observed to be used as temporary objects.
-    // - they would also pose considerable effort for cleaning up the the remembered
-    // sets.
-    // While this cleanup is not strictly necessary to be done (or done instantly),
-    // given that their occurrence is very low, this saves us this additional
-    // complexity.
+    // - never consider object arrays at this time because they would pose
+    // considerable effort for cleaning up the the remembered sets. This is
+    // required because stale remembered sets might reference locations that
+    // are currently allocated into.
     uint region_idx = r->hrm_index();
     if (g1h->humongous_is_live(region_idx) ||
         g1h->humongous_region_is_always_live(region_idx)) {
 
-      if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) {
-        gclog_or_tty->print_cr("Live humongous %d region %d size "SIZE_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d live-other %d obj array %d",
-                               r->isHumongous(),
+      if (G1TraceEagerReclaimHumongousObjects) {
+        gclog_or_tty->print_cr("Live humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d live-other %d obj array %d",
                                region_idx,
                                obj->size()*HeapWordSize,
+                               r->bottom(),
+                               r->region_num(),
                                r->rem_set()->occupied(),
                                r->rem_set()->strong_code_roots_list_length(),
                                next_bitmap->isMarked(r->bottom()),
@@ -6304,12 +6345,11 @@ class G1FreeHumongousRegionClosure : public HeapRegionClosure {
               err_msg("Eagerly reclaiming object arrays is not supported, but the object "PTR_FORMAT" is.",
                       r->bottom()));
 
-    if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) {
-      gclog_or_tty->print_cr("Reclaim humongous region %d size "SIZE_FORMAT" start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d live-other ",
-                             r->isHumongous(),
+    if (G1TraceEagerReclaimHumongousObjects) {
+      gclog_or_tty->print_cr("Dead humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d live-other %d obj array %d",
+                             region_idx,
                              obj->size()*HeapWordSize,
                              r->bottom(),
-                             region_idx,
                              r->region_num(),
                              r->rem_set()->occupied(),
                              r->rem_set()->strong_code_roots_list_length(),
@@ -6346,8 +6386,8 @@ class G1FreeHumongousRegionClosure : public HeapRegionClosure {
 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
   assert_at_safepoint(true);
 
-  if (!G1ReclaimDeadHumongousObjectsAtYoungGC ||
-      (!_has_humongous_reclaim_candidates && !G1TraceReclaimDeadHumongousObjectsAtYoungGC)) {
+  if (!G1EagerReclaimHumongousObjects ||
+      (!_has_humongous_reclaim_candidates && !G1TraceEagerReclaimHumongousObjects)) {
     g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0);
     return;
   }

src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

@@ -664,6 +664,9 @@ public:
   // Returns whether the given region (which must be a humongous (start) region)
   // is to be considered conservatively live regardless of any other conditions.
   bool humongous_region_is_always_live(uint index);
+  // Returns whether the given region (which must be a humongous (start) region)
+  // is considered a candidate for eager reclamation.
+  bool humongous_region_is_candidate(uint index);
   // Register the given region to be part of the collection set.
   inline void register_humongous_region_with_in_cset_fast_test(uint index);
   // Register regions with humongous objects (actually on the start region) in

src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp

@@ -552,11 +552,15 @@ void G1GCPhaseTimes::print(double pause_time_sec) {
   print_stats(2, "Ref Enq", _cur_ref_enq_time_ms);
   print_stats(2, "Redirty Cards", _recorded_redirty_logged_cards_time_ms);
   par_phase_printer.print(RedirtyCards);
-  if (G1ReclaimDeadHumongousObjectsAtYoungGC) {
-    print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms);
+
+  if (G1EagerReclaimHumongousObjects) {
+    print_stats(2, "Humongous Register", _cur_fast_reclaim_humongous_register_time_ms);
     if (G1Log::finest()) {
       print_stats(3, "Humongous Total", _cur_fast_reclaim_humongous_total);
       print_stats(3, "Humongous Candidate", _cur_fast_reclaim_humongous_candidates);
+    }
+    print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms);
+    if (G1Log::finest()) {
       print_stats(3, "Humongous Reclaimed", _cur_fast_reclaim_humongous_reclaimed);
     }
   }

src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp

@@ -107,6 +107,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
   double _recorded_non_young_free_cset_time_ms;
 
   double _cur_fast_reclaim_humongous_time_ms;
+  double _cur_fast_reclaim_humongous_register_time_ms;
   size_t _cur_fast_reclaim_humongous_total;
   size_t _cur_fast_reclaim_humongous_candidates;
   size_t _cur_fast_reclaim_humongous_reclaimed;
@@ -202,7 +203,8 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     _recorded_non_young_free_cset_time_ms = time_ms;
   }
 
-  void record_fast_reclaim_humongous_stats(size_t total, size_t candidates) {
+  void record_fast_reclaim_humongous_stats(double time_ms, size_t total, size_t candidates) {
+    _cur_fast_reclaim_humongous_register_time_ms = time_ms;
     _cur_fast_reclaim_humongous_total = total;
     _cur_fast_reclaim_humongous_candidates = candidates;
   }

src/share/vm/gc_implementation/g1/g1_globals.hpp

@@ -274,10 +274,14 @@
   product(uintx, G1MixedGCCountTarget, 8,                                   \
           "The target number of mixed GCs after a marking cycle.")          \
                                                                             \
-  experimental(bool, G1ReclaimDeadHumongousObjectsAtYoungGC, true,          \
+  experimental(bool, G1EagerReclaimHumongousObjects, true,                  \
           "Try to reclaim dead large objects at every young GC.")           \
                                                                             \
-  experimental(bool, G1TraceReclaimDeadHumongousObjectsAtYoungGC, false,    \
+  experimental(bool, G1EagerReclaimHumongousObjectsWithStaleRefs, true,     \
+          "Try to reclaim dead large objects that have a few stale "        \
+          "references at every young GC.")                                  \
+                                                                            \
+  experimental(bool, G1TraceEagerReclaimHumongousObjects, false,            \
           "Print some information about large object liveness "             \
           "at every young GC.")                                             \
                                                                             \

src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp

@@ -694,6 +694,18 @@ void OtherRegionsTable::scrub(CardTableModRefBS* ctbs,
   clear_fcc();
 }
 
+bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const {
+  if (limit <= (size_t)G1RSetSparseRegionEntries) {
+    return occ_coarse() == 0 && _first_all_fine_prts == NULL && occ_sparse() <= limit;
+  } else {
+    // Current uses of this method may only use values less than G1RSetSparseRegionEntries
+    // for the limit. The solution, comparing against occupied() would be too slow
+    // at this time.
+    Unimplemented();
+    return false;
+  }
+}
+
 bool OtherRegionsTable::is_empty() const {
   return occ_sparse() == 0 && occ_coarse() == 0 && _first_all_fine_prts == NULL;
 }

src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp

@@ -181,6 +181,10 @@ public:
   // sense.
   void add_reference(OopOrNarrowOopStar from, int tid);
 
+  // Returns whether this remembered set (and all sub-sets) have an occupancy
+  // that is less or equal than the given occupancy.
+  bool occupancy_less_or_equal_than(size_t limit) const;
+
   // Removes any entries shown by the given bitmaps to contain only dead
   // objects.
   void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
@@ -276,6 +280,10 @@ public:
     return (strong_code_roots_list_length() == 0) && _other_regions.is_empty();
   }
 
+  bool occupancy_less_or_equal_than(size_t occ) const {
+    return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ);
+  }
+
   size_t occupied() {
     MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
     return occupied_locked();

test/gc/g1/TestEagerReclaimHumongousRegions2.java → test/gc/g1/TestEagerReclaimHumongousRegionsClearMarkBits.java

@@ -22,7 +22,7 @@
  */
 
 /*
- * @test TestEagerReclaimHumongousRegions2
+ * @test TestEagerReclaimHumongousRegionsClearMarkBits
  * @bug 8051973
  * @summary Test to make sure that eager reclaim of humongous objects correctly clears
  * mark bitmaps at reclaim.
@@ -109,7 +109,7 @@ class ReclaimRegionFast {
     }
 }
 
-public class TestEagerReclaimHumongousRegions2 {
+public class TestEagerReclaimHumongousRegionsClearMarkBits {
     public static void main(String[] args) throws Exception {
         ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
             "-XX:+UseG1GC",

test/gc/g1/TestEagerReclaimHumongousRegionsWithRefs.java

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestEagerReclaimHumongousRegionsWithRefs
+ * @bug 8048179
+ * @summary Test to make sure that eager reclaim of humongous objects that have previously
+ * been referenced by other old gen regions work. We simply try to fill
+ * up the heap with humongous objects and create a remembered set entry from an object by
+ * referencing that we know is in the old gen. After changing this reference, the object
+ * should still be eagerly reclaimable to avoid Full GC.
+ * @key gc
+ * @library /testlibrary
+ */
+
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import java.util.LinkedList;
+
+import com.oracle.java.testlibrary.OutputAnalyzer;
+import com.oracle.java.testlibrary.ProcessTools;
+import static com.oracle.java.testlibrary.Asserts.*;
+
+class RefHolder {
+  Object ref;
+}
+
+class ReclaimRegionFast {
+
+    public static final int M = 1024*1024;
+
+    public static LinkedList<Object> garbageList = new LinkedList<Object>();
+
+    public static void genGarbage() {
+        for (int i = 0; i < 32*1024; i++) {
+            garbageList.add(new int[100]);
+        }
+        garbageList.clear();
+    }
+
+
+    // A large object referenced by a static.
+    static int[] filler = new int[10 * M];
+
+    // Old gen object referencing the large object, generating remembered
+    // set entries.
+    static RefHolder fromOld = new RefHolder();
+
+    public static void main(String[] args) {
+
+        int[] large = new int[M];
+
+        Object ref_from_stack = large;
+
+        for (int i = 0; i < 100; i++) {
+            // A large object that will be reclaimed eagerly.
+            large = new int[6*M];
+            fromOld.ref = large;
+            genGarbage();
+        }
+
+        // Keep the reference to the first object alive.
+        System.out.println(ref_from_stack);
+    }
+}
+
+public class TestEagerReclaimHumongousRegionsWithRefs {
+
+    public static void main(String[] args) throws Exception {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+            "-XX:+UseG1GC",
+            "-Xms128M",
+            "-Xmx128M",
+            "-Xmn16M",
+            "-XX:+PrintGC",
+            ReclaimRegionFast.class.getName());
+
+        Pattern p = Pattern.compile("Full GC");
+
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+        int found = 0;
+        Matcher m = p.matcher(output.getStdout());
+        while (m.find()) {
+            found++;
+        }
+        System.out.println("Issued " + found + " Full GCs");
+
+        assertLessThan(found, 10, "Found that " + found + " Full GCs were issued. This is larger than the bound. Eager reclaim of objects once referenced from old gen seems to not work at all");
+        output.shouldHaveExitValue(0);
+    }
+}
+

test/gc/g1/TestG1TraceReclaimDeadHumongousObjectsAtYoungGC.java → test/gc/g1/TestG1TraceEagerReclaimHumongousObjects.java

@@ -22,9 +22,9 @@
  */
 
 /*
- * @test TestG1TraceReclaimDeadHumongousObjectsAtYoungGC
- * @bug 8058801
- * @summary Ensure that the output for a G1TraceReclaimDeadHumongousObjectsAtYoungGC
+ * @test TestG1TraceEagerReclaimHumongousObjects
+ * @bug 8058801 8048179
+ * @summary Ensure that the output for a G1TraceEagerReclaimHumongousObjects
  * includes the expected necessary messages.
  * @key gc
  * @library /testlibrary
@@ -34,7 +34,7 @@ import com.oracle.java.testlibrary.ProcessTools;
 import com.oracle.java.testlibrary.OutputAnalyzer;
 import java.util.LinkedList;
 
-public class TestG1TraceReclaimDeadHumongousObjectsAtYoungGC {
+public class TestG1TraceEagerReclaimHumongousObjects {
   public static void main(String[] args) throws Exception {
     testGCLogs();
     testHumongousObjectGCLogs();
@@ -50,12 +50,12 @@ public class TestG1TraceReclaimDeadHumongousObjectsAtYoungGC {
                                                "-XX:+PrintGC",
                                                "-XX:+UnlockExperimentalVMOptions",
                                                "-XX:G1LogLevel=finest",
-                                               "-XX:+G1TraceReclaimDeadHumongousObjectsAtYoungGC",
+                                               "-XX:+G1TraceEagerReclaimHumongousObjects",
                                                GCTest.class.getName());
 
     OutputAnalyzer output = new OutputAnalyzer(pb.start());
 
-    // As G1ReclaimDeadHumongousObjectsAtYoungGC is set(default), below logs should be displayed.
+    // As G1EagerReclaimHumongousObjects is set(default), below logs should be displayed.
     // And GCTest doesn't have humongous objects, so values should be zero.
     output.shouldContain("[Humongous Reclaim");
     output.shouldContain("[Humongous Total: 0]");
@@ -74,7 +74,7 @@ public class TestG1TraceReclaimDeadHumongousObjectsAtYoungGC {
                                                "-XX:+PrintGC",
                                                "-XX:+UnlockExperimentalVMOptions",
                                                "-XX:G1LogLevel=finest",
-                                               "-XX:+G1TraceReclaimDeadHumongousObjectsAtYoungGC",
+                                               "-XX:+G1TraceEagerReclaimHumongousObjects",
                                                GCWithHumongousObjectTest.class.getName());
 
     OutputAnalyzer output = new OutputAnalyzer(pb.start());
@@ -88,7 +88,7 @@ public class TestG1TraceReclaimDeadHumongousObjectsAtYoungGC {
     // As G1TraceReclaimDeadHumongousObjectsAtYoungGC is set and GCWithHumongousObjectTest has humongous objects,
     // these logs should be displayed.
     output.shouldContain("Live humongous");
-    output.shouldContain("Reclaim humongous region");
+    output.shouldContain("Dead humongous region");
     output.shouldHaveExitValue(0);
   }
 

test/gc/g1/TestGCLogMessages.java

@@ -23,7 +23,7 @@
 
 /*
  * @test TestGCLogMessages
- * @bug 8035406 8027295 8035398 8019342 8027959 8027962
+ * @bug 8035406 8027295 8035398 8019342 8027959 8048179 8027962
  * @summary Ensure that the PrintGCDetails output for a minor GC with G1
  * includes the expected necessary messages.
  * @key gc
@@ -81,6 +81,7 @@ public class TestGCLogMessages {
         new LogMessageWithLevel("Non-Young Free CSet", Level.FINEST),
         // Humongous Eager Reclaim
         new LogMessageWithLevel("Humongous Reclaim", Level.FINER),
+        new LogMessageWithLevel("Humongous Register", Level.FINER),
     };
 
     void checkMessagesAtLevel(OutputAnalyzer output, LogMessageWithLevel messages[], Level level) throws Exception {