8217432: MetaspaceGC::_capacity_until_GC exceeds MaxMetaspaceSize

Reviewed-by: tschatzl, stuefe

src/share/vm/memory/metaspace.cpp

 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1422,7 +1422,15 @@ size_t MetaspaceGC::capacity_until_GC() {
   return value;
 }
 
-bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size_t* old_cap_until_GC) {
+// Try to increase the _capacity_until_GC limit counter by v bytes.
+// Returns true if it succeeded. It may fail if either another thread
+// concurrently increased the limit or the new limit would be larger
+// than MaxMetaspaceSize.
+// On success, optionally returns new and old metaspace capacity in
+// new_cap_until_GC and old_cap_until_GC respectively.
+// On error, optionally sets can_retry to indicate whether if there is
+// actually enough space remaining to satisfy the request.
+bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size_t* old_cap_until_GC, bool* can_retry) {
   assert_is_size_aligned(v, Metaspace::commit_alignment());
 
   size_t capacity_until_GC = (size_t) _capacity_until_GC;
@@ -1433,6 +1441,17 @@ bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size
     new_value = align_size_down(max_uintx, Metaspace::commit_alignment());
   }
 
+  if (new_value > MaxMetaspaceSize) {
+    if (can_retry != NULL) {
+      *can_retry = false;
+    }
+    return false;
+  }
+
+  if (can_retry != NULL) {
+    *can_retry = true;
+  }
+
   intptr_t expected = (intptr_t) capacity_until_GC;
   intptr_t actual = Atomic::cmpxchg_ptr((intptr_t) new_value, &_capacity_until_GC, expected);
 
@@ -1520,7 +1539,7 @@ void MetaspaceGC::compute_new_size() {
 
   const double min_tmp = used_after_gc / maximum_used_percentage;
   size_t minimum_desired_capacity =
-    (size_t)MIN2(min_tmp, double(max_uintx));
+    (size_t)MIN2(min_tmp, double(MaxMetaspaceSize));
   // Don't shrink less than the initial generation size
   minimum_desired_capacity = MAX2(minimum_desired_capacity,
                                   MetaspaceSize);
@@ -1579,7 +1598,7 @@ void MetaspaceGC::compute_new_size() {
     const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;
     const double minimum_used_percentage = 1.0 - maximum_free_percentage;
     const double max_tmp = used_after_gc / minimum_used_percentage;
-    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx));
+    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(MaxMetaspaceSize));
     maximum_desired_capacity = MAX2(maximum_desired_capacity,
                                     MetaspaceSize);
     if (PrintGCDetails && Verbose) {
@@ -3408,6 +3427,7 @@ MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype)
 
   size_t before = 0;
   size_t after = 0;
+  bool can_retry = true;
   MetaWord* res;
   bool incremented;
 
@@ -3415,9 +3435,9 @@ MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype)
   // the HWM, an allocation is still attempted. This is because another thread must then
   // have incremented the HWM and therefore the allocation might still succeed.
   do {
-    incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before);
+    incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before, &can_retry);
     res = allocate(word_size, mdtype);
-  } while (!incremented && res == NULL);
+  } while (!incremented && res == NULL && can_retry);
 
   if (incremented) {
     tracer()->report_gc_threshold(before, after,

src/share/vm/memory/metaspace.hpp

 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -416,7 +416,8 @@ class MetaspaceGC : AllStatic {
   static size_t capacity_until_GC();
   static bool inc_capacity_until_GC(size_t v,
                                     size_t* new_cap_until_GC = NULL,
-                                    size_t* old_cap_until_GC = NULL);
+                                    size_t* old_cap_until_GC = NULL,
+                                    bool* can_retry = NULL);
   static size_t dec_capacity_until_GC(size_t v);
 
   static bool should_concurrent_collect() { return _should_concurrent_collect; }