8004241: NPG: Metaspace occupies more memory than specified by -XX:MaxMetaspaceSize option

Summary: Enforce MaxMetaspaceSize for both metaspace parts, check MaxMetaspaceSize against "reserved", not "capacity"
Reviewed-by: jmasa, johnc

src/share/vm/memory/metaspace.cpp

@@ -1100,25 +1100,24 @@ size_t MetaspaceGC::delta_capacity_until_GC(size_t word_size) {
 }
 
 bool MetaspaceGC::should_expand(VirtualSpaceList* vsl, size_t word_size) {
-
-  // Class virtual space should always be expanded.  Call GC for the other
-  // metadata virtual space.
-  if (vsl == Metaspace::class_space_list()) return true;
-
   // If the user wants a limit, impose one.
-  size_t max_metaspace_size_words = MaxMetaspaceSize / BytesPerWord;
-  size_t metaspace_size_words = MetaspaceSize / BytesPerWord;
   if (!FLAG_IS_DEFAULT(MaxMetaspaceSize) &&
-      vsl->capacity_words_sum() >= max_metaspace_size_words) {
+      MetaspaceAux::reserved_in_bytes() >= MaxMetaspaceSize) {
     return false;
   }
 
+  // Class virtual space should always be expanded.  Call GC for the other
+  // metadata virtual space.
+  if (vsl == Metaspace::class_space_list()) return true;
+
   // If this is part of an allocation after a GC, expand
   // unconditionally.
   if(MetaspaceGC::expand_after_GC()) {
     return true;
   }
 
+  size_t metaspace_size_words = MetaspaceSize / BytesPerWord;
+
   // If the capacity is below the minimum capacity, allow the
   // expansion.  Also set the high-water-mark (capacity_until_GC)
   // to that minimum capacity so that a GC will not be induced