8035406: Improve data structure for Code Cache remembered sets

Summary: Change the code cache remembered sets data structure from a GrowableArray to a chunked list of nmethods. This makes the data structure more amenable to parallelization, and decreases freeing time.
Reviewed-by: mgerdin, brutisso

make/excludeSrc.make

 #
-# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@ ifeq ($(INCLUDE_ALL_GCS), false)
 	g1MMUTracker.cpp g1MarkSweep.cpp g1MemoryPool.cpp g1MonitoringSupport.cpp g1OopClosures.cpp \
 	g1RemSet.cpp g1RemSetSummary.cpp g1SATBCardTableModRefBS.cpp g1_globals.cpp heapRegion.cpp \
 	g1BiasedArray.cpp heapRegionRemSet.cpp heapRegionSeq.cpp heapRegionSet.cpp heapRegionSets.cpp \
-	ptrQueue.cpp satbQueue.cpp sparsePRT.cpp survRateGroup.cpp vm_operations_g1.cpp \
+	ptrQueue.cpp satbQueue.cpp sparsePRT.cpp survRateGroup.cpp vm_operations_g1.cpp g1CodeCacheRemSet.cpp \
 	adjoiningGenerations.cpp adjoiningVirtualSpaces.cpp asPSOldGen.cpp asPSYoungGen.cpp \
 	cardTableExtension.cpp gcTaskManager.cpp gcTaskThread.cpp objectStartArray.cpp \
 	parallelScavengeHeap.cpp parMarkBitMap.cpp pcTasks.cpp psAdaptiveSizePolicy.cpp \

src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+#include "precompiled.hpp"
+#include "code/nmethod.hpp"
+#include "gc_implementation/g1/g1CodeCacheRemSet.hpp"
+#include "memory/iterator.hpp"
+
+G1CodeRootChunk::G1CodeRootChunk() : _top(NULL), _next(NULL), _prev(NULL) {
+  _top = bottom();
+}
+
+void G1CodeRootChunk::reset() {
+  _next = _prev = NULL;
+  _top = bottom();
+}
+
+void G1CodeRootChunk::nmethods_do(CodeBlobClosure* cl) {
+  nmethod** cur = bottom();
+  while (cur != _top) {
+    cl->do_code_blob(*cur);
+    cur++;
+  }
+}
+
+FreeList<G1CodeRootChunk> G1CodeRootSet::_free_list;
+size_t G1CodeRootSet::_num_chunks_handed_out = 0;
+
+G1CodeRootChunk* G1CodeRootSet::new_chunk() {
+  G1CodeRootChunk* result = _free_list.get_chunk_at_head();
+  if (result == NULL) {
+    result = new G1CodeRootChunk();
+  }
+  G1CodeRootSet::_num_chunks_handed_out++;
+  result->reset();
+  return result;
+}
+
+void G1CodeRootSet::free_chunk(G1CodeRootChunk* chunk) {
+  _free_list.return_chunk_at_head(chunk);
+  G1CodeRootSet::_num_chunks_handed_out--;
+}
+
+void G1CodeRootSet::free_all_chunks(FreeList<G1CodeRootChunk>* list) {
+  G1CodeRootSet::_num_chunks_handed_out -= list->count();
+  _free_list.prepend(list);
+}
+
+void G1CodeRootSet::purge_chunks(size_t keep_ratio) {
+  size_t keep = G1CodeRootSet::_num_chunks_handed_out * keep_ratio / 100;
+
+  if (keep >= (size_t)_free_list.count()) {
+    return;
+  }
+
+  FreeList<G1CodeRootChunk> temp;
+  temp.initialize();
+  temp.set_size(G1CodeRootChunk::word_size());
+
+  _free_list.getFirstNChunksFromList((size_t)_free_list.count() - keep, &temp);
+
+  G1CodeRootChunk* cur = temp.get_chunk_at_head();
+  while (cur != NULL) {
+    delete cur;
+    cur = temp.get_chunk_at_head();
+  }
+}
+
+size_t G1CodeRootSet::static_mem_size() {
+  return sizeof(_free_list) + sizeof(_num_chunks_handed_out);
+}
+
+size_t G1CodeRootSet::fl_mem_size() {
+  return _free_list.count() * _free_list.size();
+}
+
+void G1CodeRootSet::initialize() {
+  _free_list.initialize();
+  _free_list.set_size(G1CodeRootChunk::word_size());
+}
+
+G1CodeRootSet::G1CodeRootSet() : _list(), _length(0) {
+  _list.initialize();
+  _list.set_size(G1CodeRootChunk::word_size());
+}
+
+G1CodeRootSet::~G1CodeRootSet() {
+  clear();
+}
+
+void G1CodeRootSet::add(nmethod* method) {
+  if (!contains(method)) {
+    // Try to add the nmethod. If there is not enough space, get a new chunk.
+    if (_list.head() == NULL || _list.head()->is_full()) {
+      G1CodeRootChunk* cur = new_chunk();
+      _list.return_chunk_at_head(cur);
+    }
+    bool result = _list.head()->add(method);
+    guarantee(result, err_msg("Not able to add nmethod "PTR_FORMAT" to newly allocated chunk.", method));
+    _length++;
+  }
+}
+
+void G1CodeRootSet::remove(nmethod* method) {
+  G1CodeRootChunk* found = find(method);
+  if (found != NULL) {
+    bool result = found->remove(method);
+    guarantee(result, err_msg("could not find nmethod "PTR_FORMAT" during removal although we previously found it", method));
+    // eventually free completely emptied chunk
+    if (found->is_empty()) {
+      _list.remove_chunk(found);
+      free(found);
+    }
+    _length--;
+  }
+  assert(!contains(method), err_msg(PTR_FORMAT" still contains nmethod "PTR_FORMAT, this, method));
+}
+
+nmethod* G1CodeRootSet::pop() {
+  do {
+    G1CodeRootChunk* cur = _list.head();
+    if (cur == NULL) {
+      assert(_length == 0, "when there are no chunks, there should be no elements");
+      return NULL;
+    }
+    nmethod* result = cur->pop();
+    if (result != NULL) {
+      _length--;
+      return result;
+    } else {
+      free(_list.get_chunk_at_head());
+    }
+  } while (true);
+}
+
+G1CodeRootChunk* G1CodeRootSet::find(nmethod* method) {
+  G1CodeRootChunk* cur = _list.head();
+  while (cur != NULL) {
+    if (cur->contains(method)) {
+      return cur;
+    }
+    cur = (G1CodeRootChunk*)cur->next();
+  }
+  return NULL;
+}
+
+void G1CodeRootSet::free(G1CodeRootChunk* chunk) {
+  free_chunk(chunk);
+}
+
+bool G1CodeRootSet::contains(nmethod* method) {
+  return find(method) != NULL;
+}
+
+void G1CodeRootSet::clear() {
+  free_all_chunks(&_list);
+  _length = 0;
+}
+
+void G1CodeRootSet::nmethods_do(CodeBlobClosure* blk) const {
+  G1CodeRootChunk* cur = _list.head();
+  while (cur != NULL) {
+    cur->nmethods_do(blk);
+    cur = (G1CodeRootChunk*)cur->next();
+  }
+}
+
+size_t G1CodeRootSet::mem_size() {
+  return sizeof(this) + _list.count() * _list.size();
+}
+
+#ifndef PRODUCT
+
+void G1CodeRootSet::test() {
+  initialize();
+
+  assert(_free_list.count() == 0, "Free List must be empty");
+  assert(_num_chunks_handed_out == 0, "No elements must have been handed out yet");
+
+  // The number of chunks that we allocate for purge testing.
+  size_t const num_chunks = 10;
+  {
+    G1CodeRootSet set1;
+    assert(set1.is_empty(), "Code root set must be initially empty but is not.");
+
+    set1.add((nmethod*)1);
+    assert(_num_chunks_handed_out == 1,
+           err_msg("Must have allocated and handed out one chunk, but handed out "
+                   SIZE_FORMAT" chunks", _num_chunks_handed_out));
+    assert(set1.length() == 1, err_msg("Added exactly one element, but set contains "
+                                       SIZE_FORMAT" elements", set1.length()));
+
+    // G1CodeRootChunk::word_size() is larger than G1CodeRootChunk::num_entries which
+    // we cannot access.
+    for (uint i = 0; i < G1CodeRootChunk::word_size() + 1; i++) {
+      set1.add((nmethod*)1);
+    }
+    assert(_num_chunks_handed_out == 1,
+           err_msg("Duplicate detection must have prevented allocation of further "
+                   "chunks but contains "SIZE_FORMAT, _num_chunks_handed_out));
+    assert(set1.length() == 1,
+           err_msg("Duplicate detection should not have increased the set size but "
+                   "is "SIZE_FORMAT, set1.length()));
+
+    size_t num_total_after_add = G1CodeRootChunk::word_size() + 1;
+    for (size_t i = 0; i < num_total_after_add - 1; i++) {
+      set1.add((nmethod*)(2 + i));
+    }
+    assert(_num_chunks_handed_out > 1,
+           "After adding more code roots, more than one chunks should have been handed out");
+    assert(set1.length() == num_total_after_add,
+           err_msg("After adding in total "SIZE_FORMAT" distinct code roots, they "
+                   "need to be in the set, but there are only "SIZE_FORMAT,
+                   num_total_after_add, set1.length()));
+
+    size_t num_popped = 0;
+    while (set1.pop() != NULL) {
+      num_popped++;
+    }
+    assert(num_popped == num_total_after_add,
+           err_msg("Managed to pop "SIZE_FORMAT" code roots, but only "SIZE_FORMAT" "
+                   "were added", num_popped, num_total_after_add));
+    assert(_num_chunks_handed_out == 0,
+           err_msg("After popping all elements, all chunks must have been returned "
+                   "but are still "SIZE_FORMAT, _num_chunks_handed_out));
+
+    purge_chunks(0);
+    assert(_free_list.count() == 0,
+           err_msg("After purging everything, the free list must be empty but still "
+                   "contains "SIZE_FORMAT" chunks", _free_list.count()));
+
+    // Add some more handed out chunks.
+    size_t i = 0;
+    while (_num_chunks_handed_out < num_chunks) {
+      set1.add((nmethod*)i);
+      i++;
+    }
+
+    {
+      // Generate chunks on the free list.
+      G1CodeRootSet set2;
+      size_t i = 0;
+      while (_num_chunks_handed_out < num_chunks * 2) {
+        set2.add((nmethod*)i);
+        i++;
+      }
+      // Exit of the scope of the set2 object will call the destructor that generates
+      // num_chunks elements on the free list.
+    }
+
+    assert(_num_chunks_handed_out == num_chunks,
+           err_msg("Deletion of the second set must have resulted in giving back "
+                   "those, but there is still "SIZE_FORMAT" handed out, expecting "
+                   SIZE_FORMAT, _num_chunks_handed_out, num_chunks));
+    assert((size_t)_free_list.count() == num_chunks,
+           err_msg("After freeing "SIZE_FORMAT" chunks, they must be on the free list "
+                   "but there are only "SIZE_FORMAT, num_chunks, _free_list.count()));
+
+    size_t const test_percentage = 50;
+    purge_chunks(test_percentage);
+    assert(_num_chunks_handed_out == num_chunks,
+           err_msg("Purging must not hand out chunks but there are "SIZE_FORMAT,
+                   _num_chunks_handed_out));
+    assert((size_t)_free_list.count() == (ssize_t)(num_chunks * test_percentage / 100),
+           err_msg("Must have purged "SIZE_FORMAT" percent of "SIZE_FORMAT" chunks"
+                   "but there are "SSIZE_FORMAT, test_percentage, num_chunks,
+                   _free_list.count()));
+    // Purge the remainder of the chunks on the free list.
+    purge_chunks(0);
+    assert(_free_list.count() == 0, "Free List must be empty");
+    assert(_num_chunks_handed_out == num_chunks,
+           err_msg("Expected to be "SIZE_FORMAT" chunks handed out from the first set "
+                   "but there are "SIZE_FORMAT, num_chunks, _num_chunks_handed_out));
+
+    // Exit of the scope of the set1 object will call the destructor that generates
+    // num_chunks additional elements on the free list.
+  }
+
+  assert(_num_chunks_handed_out == 0,
+         err_msg("Deletion of the only set must have resulted in no chunks handed "
+                 "out, but there is still "SIZE_FORMAT" handed out", _num_chunks_handed_out));
+  assert((size_t)_free_list.count() == num_chunks,
+         err_msg("After freeing "SIZE_FORMAT" chunks, they must be on the free list "
+                 "but there are only "SSIZE_FORMAT, num_chunks, _free_list.count()));
+
+  // Restore initial state.
+  purge_chunks(0);
+  assert(_free_list.count() == 0, "Free List must be empty");
+  assert(_num_chunks_handed_out == 0, "No elements must have been handed out yet");
+}
+
+void TestCodeCacheRemSet_test() {
+  G1CodeRootSet::test();
+}
+#endif

src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP
+#define SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/freeList.hpp"
+#include "runtime/globals.hpp"
+
+class CodeBlobClosure;
+
+class G1CodeRootChunk : public CHeapObj<mtGC> {
+ private:
+  static const int NUM_ENTRIES = 32;
+ public:
+  G1CodeRootChunk*     _next;
+  G1CodeRootChunk*     _prev;
+
+  nmethod** _top;
+
+  nmethod* _data[NUM_ENTRIES];
+
+  nmethod** bottom() const {
+    return (nmethod**) &(_data[0]);
+  }
+
+  nmethod** end() const {
+    return (nmethod**) &(_data[NUM_ENTRIES]);
+  }
+
+ public:
+  G1CodeRootChunk();
+  ~G1CodeRootChunk() {}
+
+  static size_t word_size() { return (size_t)(align_size_up_(sizeof(G1CodeRootChunk), HeapWordSize) / HeapWordSize); }
+
+  // FreeList "interface" methods
+
+  G1CodeRootChunk* next() const         { return _next; }
+  G1CodeRootChunk* prev() const         { return _prev; }
+  void set_next(G1CodeRootChunk* v)     { _next = v; assert(v != this, "Boom");}
+  void set_prev(G1CodeRootChunk* v)     { _prev = v; assert(v != this, "Boom");}
+  void clear_next()       { set_next(NULL); }
+  void clear_prev()       { set_prev(NULL); }
+
+  size_t size() const { return word_size(); }
+
+  void link_next(G1CodeRootChunk* ptr)  { set_next(ptr); }
+  void link_prev(G1CodeRootChunk* ptr)  { set_prev(ptr); }
+  void link_after(G1CodeRootChunk* ptr) {
+    link_next(ptr);
+    if (ptr != NULL) ptr->link_prev((G1CodeRootChunk*)this);
+  }
+
+  bool is_free()                 { return true; }
+
+  // New G1CodeRootChunk routines
+
+  void reset();
+
+  bool is_empty() const {
+    return _top == bottom();
+  }
+
+  bool is_full() const {
+    return _top == (nmethod**)end();
+  }
+
+  bool contains(nmethod* method) {
+    nmethod** cur = bottom();
+    while (cur != _top) {
+      if (*cur == method) return true;
+      cur++;
+    }
+    return false;
+  }
+
+  bool add(nmethod* method) {
+    if (is_full()) return false;
+    *_top = method;
+    _top++;
+    return true;
+  }
+
+  bool remove(nmethod* method) {
+    nmethod** cur = bottom();
+    while (cur != _top) {
+      if (*cur == method) {
+        memmove(cur, cur + 1, (_top - (cur + 1)) * sizeof(nmethod**));
+        _top--;
+        return true;
+      }
+      cur++;
+    }
+    return false;
+  }
+
+  void nmethods_do(CodeBlobClosure* blk);
+
+  nmethod* pop() {
+    if (is_empty()) {
+      return NULL;
+    }
+    _top--;
+    return *_top;
+  }
+};
+
+// Implements storage for a set of code roots.
+// All methods that modify the set are not thread-safe except if otherwise noted.
+class G1CodeRootSet VALUE_OBJ_CLASS_SPEC {
+ private:
+  // Global free chunk list management
+  static FreeList<G1CodeRootChunk> _free_list;
+  // Total number of chunks handed out
+  static size_t _num_chunks_handed_out;
+
+  static G1CodeRootChunk* new_chunk();
+  static void free_chunk(G1CodeRootChunk* chunk);
+  // Free all elements of the given list.
+  static void free_all_chunks(FreeList<G1CodeRootChunk>* list);
+
+  // Return the chunk that contains the given nmethod, NULL otherwise.
+  // Scans the list of chunks backwards, as this method is used to add new
+  // entries, which are typically added in bulk for a single nmethod.
+  G1CodeRootChunk* find(nmethod* method);
+  void free(G1CodeRootChunk* chunk);
+
+  size_t _length;
+  FreeList<G1CodeRootChunk> _list;
+
+ public:
+  G1CodeRootSet();
+  ~G1CodeRootSet();
+
+  static void initialize();
+  static void purge_chunks(size_t keep_ratio);
+
+  static size_t static_mem_size();
+  static size_t fl_mem_size();
+
+  // Search for the code blob from the recently allocated ones to find duplicates more quickly, as this
+  // method is likely to be repeatedly called with the same nmethod.
+  void add(nmethod* method);
+
+  void remove(nmethod* method);
+  nmethod* pop();
+
+  bool contains(nmethod* method);
+
+  void clear();
+
+  void nmethods_do(CodeBlobClosure* blk) const;
+
+  bool is_empty() { return length() == 0; }
+
+  // Length in elements
+  size_t length() const { return _length; }
+
+  // Memory size in bytes taken by this set.
+  size_t mem_size();
+
+  static void test() PRODUCT_RETURN;
+};
+
+#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -5934,6 +5934,8 @@ void G1CollectedHeap::evacuate_collection_set(EvacuationInfo& evacuation_info) {
   // strong code roots for a particular heap region.
   migrate_strong_code_roots();
 
+  purge_code_root_memory();
+
   if (g1_policy()->during_initial_mark_pause()) {
     // Reset the claim values set during marking the strong code roots
     reset_heap_region_claim_values();
@@ -6810,6 +6812,13 @@ void G1CollectedHeap::migrate_strong_code_roots() {
   g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms);
 }
 
+void G1CollectedHeap::purge_code_root_memory() {
+  double purge_start = os::elapsedTime();
+  G1CodeRootSet::purge_chunks(G1CodeRootsChunkCacheKeepPercent);
+  double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0;
+  g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms);
+}
+
 // Mark all the code roots that point into regions *not* in the
 // collection set.
 //
@@ -6880,7 +6889,7 @@ public:
       // Code roots should never be attached to a continuation of a humongous region
       assert(hrrs->strong_code_roots_list_length() == 0,
              err_msg("code roots should never be attached to continuations of humongous region "HR_FORMAT
-                     " starting at "HR_FORMAT", but has "INT32_FORMAT,
+                     " starting at "HR_FORMAT", but has "SIZE_FORMAT,
                      HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()),
                      hrrs->strong_code_roots_list_length()));
       return false;

src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1647,6 +1647,9 @@ public:
   // that were not successfullly evacuated are not migrated.
   void migrate_strong_code_roots();
 
+  // Free up superfluous code root memory.
+  void purge_code_root_memory();
+
   // During an initial mark pause, mark all the code roots that
   // point into regions *not* in the collection set.
   void mark_strong_code_roots(uint worker_id);

src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014 Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -250,6 +250,9 @@ double G1GCPhaseTimes::accounted_time_ms() {
     // Strong code root migration time
     misc_time_ms += _cur_strong_code_root_migration_time_ms;
 
+    // Strong code root purge time
+    misc_time_ms += _cur_strong_code_root_purge_time_ms;
+
     // Subtract the time taken to clean the card table from the
     // current value of "other time"
     misc_time_ms += _cur_clear_ct_time_ms;
@@ -299,6 +302,7 @@ void G1GCPhaseTimes::print(double pause_time_sec) {
   }
   print_stats(1, "Code Root Fixup", _cur_collection_code_root_fixup_time_ms);
   print_stats(1, "Code Root Migration", _cur_strong_code_root_migration_time_ms);
+  print_stats(1, "Code Root Purge", _cur_strong_code_root_purge_time_ms);
   print_stats(1, "Clear CT", _cur_clear_ct_time_ms);
   double misc_time_ms = pause_time_sec * MILLIUNITS - accounted_time_ms();
   print_stats(1, "Other", misc_time_ms);

src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014 Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -131,6 +131,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
   double _cur_collection_par_time_ms;
   double _cur_collection_code_root_fixup_time_ms;
   double _cur_strong_code_root_migration_time_ms;
+  double _cur_strong_code_root_purge_time_ms;
 
   double _cur_clear_ct_time_ms;
   double _cur_ref_proc_time_ms;
@@ -223,6 +224,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     _cur_strong_code_root_migration_time_ms = ms;
   }
 
+  void record_strong_code_root_purge_time(double ms) {
+    _cur_strong_code_root_purge_time_ms = ms;
+  }
+
   void record_ref_proc_time(double ms) {
     _cur_ref_proc_time_ms = ms;
   }

src/share/vm/gc_implementation/g1/g1_globals.hpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -285,6 +285,10 @@
   product(uintx, G1MixedGCCountTarget, 8,                                   \
           "The target number of mixed GCs after a marking cycle.")          \
                                                                             \
+  experimental(uintx, G1CodeRootsChunkCacheKeepPercent, 10,                 \
+          "The amount of code root chunks that should be kept at most "     \
+          "as percentage of already allocated.")                            \
+                                                                            \
   experimental(uintx, G1OldCSetRegionThresholdPercent, 10,                  \
           "An upper bound for the number of old CSet regions expressed "    \
           "as a percentage of the heap size.")                              \

src/share/vm/gc_implementation/g1/heapRegion.cpp

@@ -710,14 +710,14 @@ void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const
   }
 
   HeapRegionRemSet* hrrs = rem_set();
-  int strong_code_roots_length = hrrs->strong_code_roots_list_length();
+  size_t strong_code_roots_length = hrrs->strong_code_roots_list_length();
 
   // if this region is empty then there should be no entries
   // on its strong code root list
   if (is_empty()) {
     if (strong_code_roots_length > 0) {
       gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is empty "
-                             "but has "INT32_FORMAT" code root entries",
+                             "but has "SIZE_FORMAT" code root entries",
                              bottom(), end(), strong_code_roots_length);
       *failures = true;
     }
@@ -727,7 +727,7 @@ void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const
   if (continuesHumongous()) {
     if (strong_code_roots_length > 0) {
       gclog_or_tty->print_cr("region "HR_FORMAT" is a continuation of a humongous "
-                             "region but has "INT32_FORMAT" code root entries",
+                             "region but has "SIZE_FORMAT" code root entries",
                              HR_FORMAT_PARAMS(this), strong_code_roots_length);
       *failures = true;
     }

src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -259,10 +259,9 @@ size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;
 size_t OtherRegionsTable::_fine_eviction_stride = 0;
 size_t OtherRegionsTable::_fine_eviction_sample_size = 0;
 
-OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) :
+OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) :
   _g1h(G1CollectedHeap::heap()),
-  _m(Mutex::leaf, "An OtherRegionsTable lock", true),
-  _hr(hr),
+  _hr(hr), _m(m),
   _coarse_map(G1CollectedHeap::heap()->max_regions(),
               false /* in-resource-area */),
   _fine_grain_regions(NULL),
@@ -442,7 +441,7 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
   size_t ind = from_hrs_ind & _mod_max_fine_entries_mask;
   PerRegionTable* prt = find_region_table(ind, from_hr);
   if (prt == NULL) {
-    MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
+    MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag);
     // Confirm that it's really not there...
     prt = find_region_table(ind, from_hr);
     if (prt == NULL) {
@@ -544,7 +543,7 @@ OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const {
 jint OtherRegionsTable::_n_coarsenings = 0;
 
 PerRegionTable* OtherRegionsTable::delete_region_table() {
-  assert(_m.owned_by_self(), "Precondition");
+  assert(_m->owned_by_self(), "Precondition");
   assert(_n_fine_entries == _max_fine_entries, "Precondition");
   PerRegionTable* max = NULL;
   jint max_occ = 0;
@@ -676,8 +675,6 @@ void OtherRegionsTable::scrub(CardTableModRefBS* ctbs,
 
 
 size_t OtherRegionsTable::occupied() const {
-  // Cast away const in this case.
-  MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag);
   size_t sum = occ_fine();
   sum += occ_sparse();
   sum += occ_coarse();
@@ -707,8 +704,6 @@ size_t OtherRegionsTable::occ_sparse() const {
 }
 
 size_t OtherRegionsTable::mem_size() const {
-  // Cast away const in this case.
-  MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag);
   size_t sum = 0;
   // all PRTs are of the same size so it is sufficient to query only one of them.
   if (_first_all_fine_prts != NULL) {
@@ -739,7 +734,6 @@ void OtherRegionsTable::clear_fcc() {
 }
 
 void OtherRegionsTable::clear() {
-  MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
   // if there are no entries, skip this step
   if (_first_all_fine_prts != NULL) {
     guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking");
@@ -759,7 +753,7 @@ void OtherRegionsTable::clear() {
 }
 
 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) {
-  MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
+  MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag);
   size_t hrs_ind = (size_t) from_hr->hrs_index();
   size_t ind = hrs_ind & _mod_max_fine_entries_mask;
   if (del_single_region_table(ind, from_hr)) {
@@ -805,7 +799,7 @@ bool OtherRegionsTable::del_single_region_table(size_t ind,
 
 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const {
   // Cast away const in this case.
-  MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag);
+  MutexLockerEx x((Mutex*)_m, Mutex::_no_safepoint_check_flag);
   return contains_reference_locked(from);
 }
 
@@ -850,7 +844,9 @@ int HeapRegionRemSet::num_par_rem_sets() {
 
 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa,
                                    HeapRegion* hr)
-  : _bosa(bosa), _strong_code_roots_list(NULL), _other_regions(hr) {
+  : _bosa(bosa),
+    _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #"UINT32_FORMAT, hr->hrs_index()), true),
+    _code_roots(), _other_regions(hr, &_m) {
   reset_for_par_iteration();
 }
 
@@ -883,7 +879,7 @@ bool HeapRegionRemSet::iter_is_complete() {
 }
 
 #ifndef PRODUCT
-void HeapRegionRemSet::print() const {
+void HeapRegionRemSet::print() {
   HeapRegionRemSetIterator iter(this);
   size_t card_index;
   while (iter.has_next(card_index)) {
@@ -909,14 +905,14 @@ void HeapRegionRemSet::cleanup() {
 }
 
 void HeapRegionRemSet::clear() {
-  if (_strong_code_roots_list != NULL) {
-    delete _strong_code_roots_list;
-  }
-  _strong_code_roots_list = new (ResourceObj::C_HEAP, mtGC)
-                                GrowableArray<nmethod*>(10, 0, NULL, true);
+  MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
+  clear_locked();
+}
 
+void HeapRegionRemSet::clear_locked() {
+  _code_roots.clear();
   _other_regions.clear();
-  assert(occupied() == 0, "Should be clear.");
+  assert(occupied_locked() == 0, "Should be clear.");
   reset_for_par_iteration();
 }
 
@@ -937,22 +933,14 @@ void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs,
 
 void HeapRegionRemSet::add_strong_code_root(nmethod* nm) {
   assert(nm != NULL, "sanity");
-  // Search for the code blob from the RHS to avoid
-  // duplicate entries as much as possible
-  if (_strong_code_roots_list->find_from_end(nm) < 0) {
-    // Code blob isn't already in the list
-    _strong_code_roots_list->push(nm);
-  }
+  _code_roots.add(nm);
 }
 
 void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) {
   assert(nm != NULL, "sanity");
-  int idx = _strong_code_roots_list->find(nm);
-  if (idx >= 0) {
-    _strong_code_roots_list->remove_at(idx);
-  }
+  _code_roots.remove(nm);
   // Check that there were no duplicates
-  guarantee(_strong_code_roots_list->find(nm) < 0, "duplicate entry found");
+  guarantee(!_code_roots.contains(nm), "duplicate entry found");
 }
 
 class NMethodMigrationOopClosure : public OopClosure {
@@ -1014,8 +1002,8 @@ void HeapRegionRemSet::migrate_strong_code_roots() {
   GrowableArray<nmethod*> to_be_retained(10);
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 
-  while (_strong_code_roots_list->is_nonempty()) {
-    nmethod *nm = _strong_code_roots_list->pop();
+  while (!_code_roots.is_empty()) {
+    nmethod *nm = _code_roots.pop();
     if (nm != NULL) {
       NMethodMigrationOopClosure oop_cl(g1h, hr(), nm);
       nm->oops_do(&oop_cl);
@@ -1038,20 +1026,16 @@ void HeapRegionRemSet::migrate_strong_code_roots() {
 }
 
 void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const {
-  for (int i = 0; i < _strong_code_roots_list->length(); i += 1) {
-    nmethod* nm = _strong_code_roots_list->at(i);
-    blk->do_code_blob(nm);
-  }
+  _code_roots.nmethods_do(blk);
 }
 
 size_t HeapRegionRemSet::strong_code_roots_mem_size() {
-  return sizeof(GrowableArray<nmethod*>) +
-         _strong_code_roots_list->max_length() * sizeof(nmethod*);
+  return _code_roots.mem_size();
 }
 
 //-------------------- Iteration --------------------
 
-HeapRegionRemSetIterator:: HeapRegionRemSetIterator(const HeapRegionRemSet* hrrs) :
+HeapRegionRemSetIterator:: HeapRegionRemSetIterator(HeapRegionRemSet* hrrs) :
   _hrrs(hrrs),
   _g1h(G1CollectedHeap::heap()),
   _coarse_map(&hrrs->_other_regions._coarse_map),

src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP
 #define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP
 
+#include "gc_implementation/g1/g1CodeCacheRemSet.hpp"
 #include "gc_implementation/g1/sparsePRT.hpp"
 
 // Remembered set for a heap region.  Represent a set of "cards" that
@@ -72,7 +73,7 @@ class OtherRegionsTable VALUE_OBJ_CLASS_SPEC {
   friend class HeapRegionRemSetIterator;
 
   G1CollectedHeap* _g1h;
-  Mutex            _m;
+  Mutex*           _m;
   HeapRegion*      _hr;
 
   // These are protected by "_m".
@@ -129,7 +130,7 @@ class OtherRegionsTable VALUE_OBJ_CLASS_SPEC {
   void unlink_from_all(PerRegionTable * prt);
 
 public:
-  OtherRegionsTable(HeapRegion* hr);
+  OtherRegionsTable(HeapRegion* hr, Mutex* m);
 
   HeapRegion* hr() const { return _hr; }
 
@@ -141,7 +142,6 @@ public:
   // objects.
   void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
 
-  // Not const because it takes a lock.
   size_t occupied() const;
   size_t occ_fine() const;
   size_t occ_coarse() const;
@@ -192,9 +192,11 @@ private:
   G1BlockOffsetSharedArray* _bosa;
   G1BlockOffsetSharedArray* bosa() const { return _bosa; }
 
-  // A list of code blobs (nmethods) whose code contains pointers into
+  // A set of code blobs (nmethods) whose code contains pointers into
   // the region that owns this RSet.
-  GrowableArray<nmethod*>* _strong_code_roots_list;
+  G1CodeRootSet _code_roots;
+
+  Mutex _m;
 
   OtherRegionsTable _other_regions;
 
@@ -218,8 +220,7 @@ private:
   static void print_event(outputStream* str, Event evnt);
 
 public:
-  HeapRegionRemSet(G1BlockOffsetSharedArray* bosa,
-                   HeapRegion* hr);
+  HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr);
 
   static int num_par_rem_sets();
   static void setup_remset_size();
@@ -228,7 +229,11 @@ public:
     return _other_regions.hr();
   }
 
-  size_t occupied() const {
+  size_t occupied() {
+    MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
+    return occupied_locked();
+  }
+  size_t occupied_locked() {
     return _other_regions.occupied();
   }
   size_t occ_fine() const {
@@ -260,6 +265,7 @@ public:
   // The region is being reclaimed; clear its remset, and any mention of
   // entries for this region in other remsets.
   void clear();
+  void clear_locked();
 
   // Attempt to claim the region.  Returns true iff this call caused an
   // atomic transition from Unclaimed to Claimed.
@@ -289,6 +295,7 @@ public:
   // The actual # of bytes this hr_remset takes up.
   // Note also includes the strong code root set.
   size_t mem_size() {
+    MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
     return _other_regions.mem_size()
       // This correction is necessary because the above includes the second
       // part.
@@ -299,13 +306,13 @@ public:
   // Returns the memory occupancy of all static data structures associated
   // with remembered sets.
   static size_t static_mem_size() {
-    return OtherRegionsTable::static_mem_size();
+    return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size();
   }
 
   // Returns the memory occupancy of all free_list data structures associated
   // with remembered sets.
   static size_t fl_mem_size() {
-    return OtherRegionsTable::fl_mem_size();
+    return OtherRegionsTable::fl_mem_size() + G1CodeRootSet::fl_mem_size();
   }
 
   bool contains_reference(OopOrNarrowOopStar from) const {
@@ -328,21 +335,21 @@ public:
   void strong_code_roots_do(CodeBlobClosure* blk) const;
 
   // Returns the number of elements in the strong code roots list
-  int strong_code_roots_list_length() {
-    return _strong_code_roots_list->length();
+  size_t strong_code_roots_list_length() {
+    return _code_roots.length();
   }
 
   // Returns true if the strong code roots contains the given
   // nmethod.
   bool strong_code_roots_list_contains(nmethod* nm) {
-    return _strong_code_roots_list->contains(nm);
+    return _code_roots.contains(nm);
   }
 
   // Returns the amount of memory, in bytes, currently
   // consumed by the strong code roots.
   size_t strong_code_roots_mem_size();
 
-  void print() const;
+  void print() PRODUCT_RETURN;
 
   // Called during a stop-world phase to perform any deferred cleanups.
   static void cleanup();
@@ -350,6 +357,7 @@ public:
   // Declare the heap size (in # of regions) to the HeapRegionRemSet(s).
   // (Uses it to initialize from_card_cache).
   static void init_heap(uint max_regions) {
+    G1CodeRootSet::initialize();
     OtherRegionsTable::init_from_card_cache((size_t) max_regions);
   }
 
@@ -384,7 +392,7 @@ public:
 class HeapRegionRemSetIterator : public StackObj {
 
   // The region RSet over which we're iterating.
-  const HeapRegionRemSet* _hrrs;
+  HeapRegionRemSet* _hrrs;
 
   // Local caching of HRRS fields.
   const BitMap*             _coarse_map;
@@ -441,7 +449,7 @@ class HeapRegionRemSetIterator : public StackObj {
 public:
   // We require an iterator to be initialized before use, so the
   // constructor does little.
-  HeapRegionRemSetIterator(const HeapRegionRemSet* hrrs);
+  HeapRegionRemSetIterator(HeapRegionRemSet* hrrs);
 
   // If there remains one or more cards to be yielded, returns true and
   // sets "card_index" to one of those cards (which is then considered

src/share/vm/memory/freeList.cpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,7 @@
 
 #if INCLUDE_ALL_GCS
 #include "gc_implementation/concurrentMarkSweep/freeChunk.hpp"
+#include "gc_implementation/g1/g1CodeCacheRemSet.hpp"
 #endif // INCLUDE_ALL_GCS
 
 // Free list.  A FreeList is used to access a linked list of chunks
@@ -332,4 +333,5 @@ template class FreeList<Metablock>;
 template class FreeList<Metachunk>;
 #if INCLUDE_ALL_GCS
 template class FreeList<FreeChunk>;
+template class FreeList<G1CodeRootChunk>;
 #endif // INCLUDE_ALL_GCS

src/share/vm/prims/jni.cpp

@@ -5083,6 +5083,7 @@ void TestVirtualSpaceNode_test();
 #if INCLUDE_ALL_GCS
 void TestOldFreeSpaceCalculation_test();
 void TestG1BiasedArray_test();
+void TestCodeCacheRemSet_test();
 #endif
 
 void execute_internal_vm_tests() {
@@ -5108,6 +5109,7 @@ void execute_internal_vm_tests() {
     run_unit_test(TestOldFreeSpaceCalculation_test());
     run_unit_test(TestG1BiasedArray_test());
     run_unit_test(HeapRegionRemSet::test_prt());
+    run_unit_test(TestCodeCacheRemSet_test());
 #endif
     tty->print_cr("All internal VM tests passed");
   }

test/gc/g1/TestGCLogMessages.java

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestPrintGCDetails
+ * @bug 8035406
+ * @summary Ensure that the PrintGCDetails output for a minor GC with G1
+ * includes the expected necessary messages.
+ * @key gc
+ * @library /testlibrary
+ */
+
+import com.oracle.java.testlibrary.ProcessTools;
+import com.oracle.java.testlibrary.OutputAnalyzer;
+
+public class TestGCLogMessages {
+  public static void main(String[] args) throws Exception {
+
+    ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-XX:+UseG1GC",
+                                                              "-Xmx10M",
+                                                              "-XX:+PrintGCDetails",
+                                                              GCTest.class.getName());
+
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+    output.shouldContain("[Code Root Purge");
+    output.shouldHaveExitValue(0);
+  }
+
+  static class GCTest {
+    private static byte[] garbage;
+    public static void main(String [] args) {
+      System.out.println("Creating garbage");
+      // create 128MB of garbage. This should result in at least one GC
+      for (int i = 0; i < 1024; i++) {
+        garbage = new byte[128 * 1024];
+      }
+      System.out.println("Done");
+    }
+  }
+}