8008966: NPG: Inefficient Metaspace counter functions cause large young GC regressions

Reviewed-by: mgerdin, coleenp

src/share/vm/classfile/classLoaderData.cpp

@@ -672,6 +672,8 @@ bool ClassLoaderDataGraph::do_unloading(BoolObjectClosure* is_alive_closure) {
     dead->unload();
     data = data->next();
     // Remove from loader list.
+    // This class loader data will no longer be found
+    // in the ClassLoaderDataGraph.
     if (prev != NULL) {
       prev->set_next(data);
     } else {

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -1304,7 +1304,7 @@ bool G1CollectedHeap::do_collection(bool explicit_gc,
 
   print_heap_before_gc();
 
-  size_t metadata_prev_used = MetaspaceAux::used_in_bytes();
+  size_t metadata_prev_used = MetaspaceAux::allocated_used_bytes();
 
   HRSPhaseSetter x(HRSPhaseFullGC);
   verify_region_sets_optional();
@@ -1425,6 +1425,7 @@ bool G1CollectedHeap::do_collection(bool explicit_gc,
 
       // Delete metaspaces for unloaded class loaders and clean up loader_data graph
       ClassLoaderDataGraph::purge();
+    MetaspaceAux::verify_metrics();
 
       // Note: since we've just done a full GC, concurrent
       // marking is no longer active. Therefore we need not

src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp

@@ -177,7 +177,7 @@ bool PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) {
     size_t prev_used = heap->used();
 
     // Capture metadata size before collection for sizing.
-    size_t metadata_prev_used = MetaspaceAux::used_in_bytes();
+    size_t metadata_prev_used = MetaspaceAux::allocated_used_bytes();
 
     // For PrintGCDetails
     size_t old_gen_prev_used = old_gen->used_in_bytes();
@@ -238,6 +238,7 @@ bool PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) {
 
     // Delete metaspaces for unloaded class loaders and clean up loader_data graph
     ClassLoaderDataGraph::purge();
+    MetaspaceAux::verify_metrics();
 
     BiasedLocking::restore_marks();
     Threads::gc_epilogue();

src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp

@@ -891,7 +891,7 @@ public:
     _heap_used      = heap->used();
     _young_gen_used = heap->young_gen()->used_in_bytes();
     _old_gen_used   = heap->old_gen()->used_in_bytes();
-    _metadata_used  = MetaspaceAux::used_in_bytes();
+    _metadata_used  = MetaspaceAux::allocated_used_bytes();
   };
 
   size_t heap_used() const      { return _heap_used; }
@@ -1026,6 +1026,7 @@ void PSParallelCompact::post_compact()
 
   // Delete metaspaces for unloaded class loaders and clean up loader_data graph
   ClassLoaderDataGraph::purge();
+  MetaspaceAux::verify_metrics();
 
   Threads::gc_epilogue();
   CodeCache::gc_epilogue();

src/share/vm/gc_implementation/shared/vmGCOperations.cpp

@@ -225,7 +225,10 @@ void VM_CollectForMetadataAllocation::doit() {
         gclog_or_tty->print_cr("\nCMS full GC for Metaspace");
       }
       heap->collect_as_vm_thread(GCCause::_metadata_GC_threshold);
-      _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype);
+      // After a GC try to allocate without expanding.  Could fail
+      // and expansion will be tried below.
+      _result =
+        _loader_data->metaspace_non_null()->allocate(_size, _mdtype);
     }
     if (_result == NULL && !UseConcMarkSweepGC /* CMS already tried */) {
       // If still failing, allow the Metaspace to expand.

src/share/vm/memory/filemap.cpp

@@ -238,8 +238,8 @@ void FileMapInfo::write_header() {
 
 void FileMapInfo::write_space(int i, Metaspace* space, bool read_only) {
   align_file_position();
-  size_t used = space->used_words(Metaspace::NonClassType) * BytesPerWord;
-  size_t capacity = space->capacity_words(Metaspace::NonClassType) * BytesPerWord;
+  size_t used = space->used_bytes_slow(Metaspace::NonClassType);
+  size_t capacity = space->capacity_bytes_slow(Metaspace::NonClassType);
   struct FileMapInfo::FileMapHeader::space_info* si = &_header._space[i];
   write_region(i, (char*)space->bottom(), used, capacity, read_only, false);
 }

src/share/vm/memory/genCollectedHeap.cpp

@@ -377,7 +377,7 @@ void GenCollectedHeap::do_collection(bool  full,
 
   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());
 
-  const size_t metadata_prev_used = MetaspaceAux::used_in_bytes();
+  const size_t metadata_prev_used = MetaspaceAux::allocated_used_bytes();
 
   print_heap_before_gc();
 
@@ -556,6 +556,7 @@ void GenCollectedHeap::do_collection(bool  full,
     if (complete) {
       // Delete metaspaces for unloaded class loaders and clean up loader_data graph
       ClassLoaderDataGraph::purge();
+      MetaspaceAux::verify_metrics();
       // Resize the metaspace capacity after full collections
       MetaspaceGC::compute_new_size();
       update_full_collections_completed();

src/share/vm/memory/metaspace.hpp

@@ -111,6 +111,10 @@ class Metaspace : public CHeapObj<mtClass> {
   SpaceManager* _class_vsm;
   SpaceManager* class_vsm() const { return _class_vsm; }
 
+  // Allocate space for metadata of type mdtype. This is space
+  // within a Metachunk and is used by
+  //   allocate(ClassLoaderData*, size_t, bool, MetadataType, TRAPS)
+  // which returns a Metablock.
   MetaWord* allocate(size_t word_size, MetadataType mdtype);
 
   // Virtual Space lists for both classes and other metadata
@@ -133,11 +137,14 @@ class Metaspace : public CHeapObj<mtClass> {
   static size_t first_class_chunk_word_size() { return _first_class_chunk_word_size; }
 
   char*  bottom() const;
-  size_t used_words(MetadataType mdtype) const;
+  size_t used_words_slow(MetadataType mdtype) const;
   size_t free_words(MetadataType mdtype) const;
-  size_t capacity_words(MetadataType mdtype) const;
+  size_t capacity_words_slow(MetadataType mdtype) const;
   size_t waste_words(MetadataType mdtype) const;
 
+  size_t used_bytes_slow(MetadataType mdtype) const;
+  size_t capacity_bytes_slow(MetadataType mdtype) const;
+
   static Metablock* allocate(ClassLoaderData* loader_data, size_t size,
                             bool read_only, MetadataType mdtype, TRAPS);
   void deallocate(MetaWord* ptr, size_t byte_size, bool is_class);
@@ -161,28 +168,81 @@ class Metaspace : public CHeapObj<mtClass> {
 class MetaspaceAux : AllStatic {
 
   // Statistics for class space and data space in metaspace.
-  static size_t used_in_bytes(Metaspace::MetadataType mdtype);
+
+  // These methods iterate over the classloader data graph
+  // for the given Metaspace type.  These are slow.
+  static size_t used_bytes_slow(Metaspace::MetadataType mdtype);
   static size_t free_in_bytes(Metaspace::MetadataType mdtype);
-  static size_t capacity_in_bytes(Metaspace::MetadataType mdtype);
+  static size_t capacity_bytes_slow(Metaspace::MetadataType mdtype);
+
+  // Iterates over the virtual space list.
   static size_t reserved_in_bytes(Metaspace::MetadataType mdtype);
 
   static size_t free_chunks_total(Metaspace::MetadataType mdtype);
   static size_t free_chunks_total_in_bytes(Metaspace::MetadataType mdtype);
 
  public:
-  // Total of space allocated to metadata in all Metaspaces
-  static size_t used_in_bytes() {
-    return used_in_bytes(Metaspace::ClassType) +
-           used_in_bytes(Metaspace::NonClassType);
+  // Running sum of space in all Metachunks that has been
+  // allocated to a Metaspace.  This is used instead of
+  // iterating over all the classloaders
+  static size_t _allocated_capacity_words;
+  // Running sum of space in all Metachunks that have
+  // are being used for metadata.
+  static size_t _allocated_used_words;
+
+ public:
+  // Decrement and increment _allocated_capacity_words
+  static void dec_capacity(size_t words);
+  static void inc_capacity(size_t words);
+
+  // Decrement and increment _allocated_used_words
+  static void dec_used(size_t words);
+  static void inc_used(size_t words);
+
+  // Total of space allocated to metadata in all Metaspaces.
+  // This sums the space used in each Metachunk by
+  // iterating over the classloader data graph
+  static size_t used_bytes_slow() {
+    return used_bytes_slow(Metaspace::ClassType) +
+           used_bytes_slow(Metaspace::NonClassType);
+  }
+
+  // Used by MetaspaceCounters
+  static size_t free_chunks_total();
+  static size_t free_chunks_total_in_bytes();
+
+  static size_t allocated_capacity_words() {
+    return _allocated_capacity_words;
+  }
+  static size_t allocated_capacity_bytes() {
+    return _allocated_capacity_words * BytesPerWord;
+  }
+
+  static size_t allocated_used_words() {
+    return _allocated_used_words;
+  }
+  static size_t allocated_used_bytes() {
+    return _allocated_used_words * BytesPerWord;
   }
 
-  // Total of available space in all Metaspaces
-  // Total of capacity allocated to all Metaspaces.  This includes
-  // space in Metachunks not yet allocated and in the Metachunk
-  // freelist.
-  static size_t capacity_in_bytes() {
-    return capacity_in_bytes(Metaspace::ClassType) +
-           capacity_in_bytes(Metaspace::NonClassType);
+  static size_t free_bytes();
+
+  // Total capacity in all Metaspaces
+  static size_t capacity_bytes_slow() {
+#ifdef PRODUCT
+    // Use allocated_capacity_bytes() in PRODUCT instead of this function.
+    guarantee(false, "Should not call capacity_bytes_slow() in the PRODUCT");
+#endif
+    size_t class_capacity = capacity_bytes_slow(Metaspace::ClassType);
+    size_t non_class_capacity = capacity_bytes_slow(Metaspace::NonClassType);
+    assert(allocated_capacity_bytes() == class_capacity + non_class_capacity,
+           err_msg("bad accounting: allocated_capacity_bytes() " SIZE_FORMAT
+             " class_capacity + non_class_capacity " SIZE_FORMAT
+             " class_capacity " SIZE_FORMAT " non_class_capacity " SIZE_FORMAT,
+             allocated_capacity_bytes(), class_capacity + non_class_capacity,
+             class_capacity, non_class_capacity));
+
+    return class_capacity + non_class_capacity;
   }
 
   // Total space reserved in all Metaspaces
@@ -201,6 +261,11 @@ class MetaspaceAux : AllStatic {
   static void print_waste(outputStream* out);
   static void dump(outputStream* out);
   static void verify_free_chunks();
+  // Checks that the values returned by allocated_capacity_bytes() and
+  // capacity_bytes_slow() are the same.
+  static void verify_capacity();
+  static void verify_used();
+  static void verify_metrics();
 };
 
 // Metaspace are deallocated when their class loader are GC'ed.
@@ -235,7 +300,6 @@ class MetaspaceGC : AllStatic {
  public:
 
   static size_t capacity_until_GC() { return _capacity_until_GC; }
-  static size_t capacity_until_GC_in_bytes() { return _capacity_until_GC * BytesPerWord; }
   static void inc_capacity_until_GC(size_t v) { _capacity_until_GC += v; }
   static void dec_capacity_until_GC(size_t v) {
     _capacity_until_GC = _capacity_until_GC > v ? _capacity_until_GC - v : 0;

src/share/vm/memory/metaspaceCounters.cpp

@@ -29,6 +29,16 @@
 
 MetaspaceCounters* MetaspaceCounters::_metaspace_counters = NULL;
 
+size_t MetaspaceCounters::calc_total_capacity() {
+  // The total capacity is the sum of
+  //   1) capacity of Metachunks in use by all Metaspaces
+  //   2) unused space at the end of each Metachunk
+  //   3) space in the freelist
+  size_t total_capacity = MetaspaceAux::allocated_capacity_bytes()
+    + MetaspaceAux::free_bytes() + MetaspaceAux::free_chunks_total_in_bytes();
+  return total_capacity;
+}
+
 MetaspaceCounters::MetaspaceCounters() :
     _capacity(NULL),
     _used(NULL),
@@ -36,8 +46,8 @@ MetaspaceCounters::MetaspaceCounters() :
   if (UsePerfData) {
     size_t min_capacity = MetaspaceAux::min_chunk_size();
     size_t max_capacity = MetaspaceAux::reserved_in_bytes();
-    size_t curr_capacity = MetaspaceAux::capacity_in_bytes();
-    size_t used = MetaspaceAux::used_in_bytes();
+    size_t curr_capacity = calc_total_capacity();
+    size_t used = MetaspaceAux::allocated_used_bytes();
 
     initialize(min_capacity, max_capacity, curr_capacity, used);
   }
@@ -82,15 +92,13 @@ void MetaspaceCounters::initialize(size_t min_capacity,
 
 void MetaspaceCounters::update_capacity() {
   assert(UsePerfData, "Should not be called unless being used");
-  assert(_capacity != NULL, "Should be initialized");
-  size_t capacity_in_bytes = MetaspaceAux::capacity_in_bytes();
-  _capacity->set_value(capacity_in_bytes);
+  size_t total_capacity = calc_total_capacity();
+  _capacity->set_value(total_capacity);
 }
 
 void MetaspaceCounters::update_used() {
   assert(UsePerfData, "Should not be called unless being used");
-  assert(_used != NULL, "Should be initialized");
-  size_t used_in_bytes = MetaspaceAux::used_in_bytes();
+  size_t used_in_bytes = MetaspaceAux::allocated_used_bytes();
   _used->set_value(used_in_bytes);
 }
 

src/share/vm/memory/metaspaceCounters.hpp

@@ -37,6 +37,7 @@ class MetaspaceCounters: public CHeapObj<mtClass> {
                   size_t max_capacity,
                   size_t curr_capacity,
                   size_t used);
+  size_t calc_total_capacity();
  public:
   MetaspaceCounters();
   ~MetaspaceCounters();

src/share/vm/memory/metaspaceShared.cpp

@@ -376,18 +376,17 @@ void VM_PopulateDumpSharedSpace::doit() {
   const char* fmt = "%s space: %9d [ %4.1f%% of total] out of %9d bytes [%4.1f%% used] at " PTR_FORMAT;
   Metaspace* ro_space = _loader_data->ro_metaspace();
   Metaspace* rw_space = _loader_data->rw_metaspace();
-  const size_t BPW = BytesPerWord;
 
   // Allocated size of each space (may not be all occupied)
-  const size_t ro_alloced = ro_space->capacity_words(Metaspace::NonClassType) * BPW;
-  const size_t rw_alloced = rw_space->capacity_words(Metaspace::NonClassType) * BPW;
+  const size_t ro_alloced = ro_space->capacity_bytes_slow(Metaspace::NonClassType);
+  const size_t rw_alloced = rw_space->capacity_bytes_slow(Metaspace::NonClassType);
   const size_t md_alloced = md_end-md_low;
   const size_t mc_alloced = mc_end-mc_low;
   const size_t total_alloced = ro_alloced + rw_alloced + md_alloced + mc_alloced;
 
   // Occupied size of each space.
-  const size_t ro_bytes = ro_space->used_words(Metaspace::NonClassType) * BPW;
-  const size_t rw_bytes = rw_space->used_words(Metaspace::NonClassType) * BPW;
+  const size_t ro_bytes = ro_space->used_bytes_slow(Metaspace::NonClassType);
+  const size_t rw_bytes = rw_space->used_bytes_slow(Metaspace::NonClassType);
   const size_t md_bytes = size_t(md_top - md_low);
   const size_t mc_bytes = size_t(mc_top - mc_low);