6689523: max heap calculation for compressed oops is off by MaxPermSize

Summary: Need to subtract MaxPermSize from the total heap size when determining whether compressed oops is turned on.
Reviewed-by: jmasa, jcoomes, kvn

src/share/vm/oops/oop.hpp

@@ -138,6 +138,10 @@ class oopDesc {
   // Need this as public for garbage collection.
   template <class T> T* obj_field_addr(int offset) const;
 
+  // Oop encoding heap max
+  static const uint64_t OopEncodingHeapMax =
+              (uint64_t(max_juint) + 1) << LogMinObjAlignmentInBytes;
+
   static bool is_null(oop obj);
   static bool is_null(narrowOop obj);
 

src/share/vm/oops/oop.inline.hpp

@@ -134,8 +134,10 @@ inline bool oopDesc::is_null(narrowOop obj) { return obj == 0; }
 inline narrowOop oopDesc::encode_heap_oop_not_null(oop v) {
   assert(!is_null(v), "oop value can never be zero");
   address heap_base = Universe::heap_base();
-  uint64_t result = (uint64_t)(pointer_delta((void*)v, (void*)heap_base, 1) >> LogMinObjAlignmentInBytes);
-  assert((result & 0xffffffff00000000ULL) == 0, "narrow oop overflow");
+  uint64_t pd = (uint64_t)(pointer_delta((void*)v, (void*)heap_base, 1));
+  assert(OopEncodingHeapMax > pd, "change encoding max if new encoding");
+  uint64_t result = pd >> LogMinObjAlignmentInBytes;
+  assert((result & CONST64(0xffffffff00000000)) == 0, "narrow oop overflow");
   return (narrowOop)result;
 }
 

src/share/vm/runtime/arguments.cpp

@@ -1125,6 +1125,11 @@ void Arguments::set_cms_and_parnew_gc_flags() {
   }
 }
 
+inline uintx max_heap_for_compressed_oops() {
+  LP64_ONLY(return oopDesc::OopEncodingHeapMax - MaxPermSize - os::vm_page_size());
+  NOT_LP64(return DefaultMaxRAM);
+}
+
 bool Arguments::should_auto_select_low_pause_collector() {
   if (UseAutoGCSelectPolicy &&
       !FLAG_IS_DEFAULT(MaxGCPauseMillis) &&
@@ -1169,7 +1174,7 @@ void Arguments::set_ergonomics_flags() {
   // field offset to determine free list chunk markers.
   // Check that UseCompressedOops can be set with the max heap size allocated
   // by ergonomics.
-  if (!UseConcMarkSweepGC && MaxHeapSize <= (32*G - os::vm_page_size())) {
+  if (!UseConcMarkSweepGC && MaxHeapSize <= max_heap_for_compressed_oops()) {
     if (FLAG_IS_DEFAULT(UseCompressedOops)) {
       FLAG_SET_ERGO(bool, UseCompressedOops, true);
     }
@@ -1205,7 +1210,10 @@ void Arguments::set_parallel_gc_flags() {
     if (FLAG_IS_DEFAULT(MaxHeapSize)) {
       const uint64_t reasonable_fraction =
         os::physical_memory() / DefaultMaxRAMFraction;
-      const uint64_t maximum_size = (uint64_t) DefaultMaxRAM;
+      const uint64_t maximum_size = (uint64_t)
+                 (FLAG_IS_DEFAULT(DefaultMaxRAM) && UseCompressedOops ?
+                     MIN2(max_heap_for_compressed_oops(), DefaultMaxRAM) :
+                     DefaultMaxRAM);
       size_t reasonable_max =
         (size_t) os::allocatable_physical_memory(reasonable_fraction);
       if (reasonable_max > maximum_size) {