8008217: CDS: Class data sharing limits the malloc heap on Solaris

Summary: In 64bit VM move CDS archive address to 32G on all platforms using new flag SharedBaseAddress.  In 32bit VM set CDS archive address to 3Gb on Linux and let other OSs pick the address.
Reviewed-by: kvn, dcubed, zgu, hseigel

src/os_cpu/bsd_x86/vm/globals_bsd_x86.hpp

@@ -46,7 +46,7 @@ define_pd_global(uintx, SurvivorRatio,           8);
 
 define_pd_global(uintx, JVMInvokeMethodSlack,    8192);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx, HeapBaseMinAddress,      2*G);
 
 #endif // OS_CPU_BSD_X86_VM_GLOBALS_BSD_X86_HPP

src/os_cpu/bsd_zero/vm/globals_bsd_zero.hpp

@@ -41,7 +41,7 @@ define_pd_global(intx,  VMThreadStackSize,       512);
 define_pd_global(intx,  CompilerThreadStackSize, 0);
 define_pd_global(uintx, JVMInvokeMethodSlack,    8192);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx, HeapBaseMinAddress,      2*G);
 
 #endif // OS_CPU_BSD_ZERO_VM_GLOBALS_BSD_ZERO_HPP

src/os_cpu/linux_sparc/vm/globals_linux_sparc.hpp

@@ -33,7 +33,7 @@
 define_pd_global(uintx, JVMInvokeMethodSlack,    12288);
 define_pd_global(intx, CompilerThreadStackSize,  0);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx, HeapBaseMinAddress,      CONST64(4)*G);
 
 #endif // OS_CPU_LINUX_SPARC_VM_GLOBALS_LINUX_SPARC_HPP

src/os_cpu/linux_x86/vm/globals_linux_x86.hpp

@@ -44,7 +44,7 @@ define_pd_global(intx, CompilerThreadStackSize,  0);
 
 define_pd_global(uintx,JVMInvokeMethodSlack,     8192);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx,HeapBaseMinAddress,       2*G);
 
 #endif // OS_CPU_LINUX_X86_VM_GLOBALS_LINUX_X86_HPP

src/os_cpu/linux_zero/vm/globals_linux_zero.hpp

@@ -41,7 +41,7 @@ define_pd_global(intx,  VMThreadStackSize,       512);
 define_pd_global(intx,  CompilerThreadStackSize, 0);
 define_pd_global(uintx, JVMInvokeMethodSlack,    8192);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx, HeapBaseMinAddress,      2*G);
 
 #endif // OS_CPU_LINUX_ZERO_VM_GLOBALS_LINUX_ZERO_HPP

src/os_cpu/solaris_sparc/vm/globals_solaris_sparc.hpp

@@ -33,7 +33,7 @@
 define_pd_global(uintx, JVMInvokeMethodSlack,    12288);
 define_pd_global(intx, CompilerThreadStackSize,  0);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 #ifdef _LP64
 define_pd_global(uintx, HeapBaseMinAddress,      CONST64(4)*G);
 #else

src/os_cpu/solaris_x86/vm/globals_solaris_x86.hpp

@@ -43,7 +43,7 @@ define_pd_global(uintx,JVMInvokeMethodSlack,     10*K);
 
 define_pd_global(intx, CompilerThreadStackSize,  0);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx,HeapBaseMinAddress,       256*M);
 
 #endif // OS_CPU_SOLARIS_X86_VM_GLOBALS_SOLARIS_X86_HPP

src/os_cpu/windows_x86/vm/globals_windows_x86.hpp

@@ -45,7 +45,7 @@ define_pd_global(intx, CompilerThreadStackSize,  0);
 
 define_pd_global(uintx, JVMInvokeMethodSlack,    8192);
 
-// Used on 64 bit platforms for UseCompressedOops base address or CDS
+// Used on 64 bit platforms for UseCompressedOops base address
 define_pd_global(uintx, HeapBaseMinAddress,      2*G);
 
 #endif // OS_CPU_WINDOWS_X86_VM_GLOBALS_WINDOWS_X86_HPP

src/share/vm/memory/filemap.cpp

@@ -372,7 +372,7 @@ ReservedSpace FileMapInfo::reserve_shared_memory() {
   // other reserved memory (like the code cache).
   ReservedSpace rs(size, alignment, false, requested_addr);
   if (!rs.is_reserved()) {
-    fail_continue(err_msg("Unable to reserved shared space at required address " INTPTR_FORMAT, requested_addr));
+    fail_continue(err_msg("Unable to reserve shared space at required address " INTPTR_FORMAT, requested_addr));
     return rs;
   }
   // the reserved virtual memory is for mapping class data sharing archive

src/share/vm/memory/metaspace.cpp

@@ -337,27 +337,16 @@ VirtualSpaceNode::VirtualSpaceNode(size_t byte_size) : _top(NULL), _next(NULL),
   // align up to vm allocation granularity
   byte_size = align_size_up(byte_size, os::vm_allocation_granularity());
 
-  // This allocates memory with mmap.  For DumpSharedspaces, allocate the
-  // space at low memory so that other shared images don't conflict.
-  // This is the same address as memory needed for UseCompressedOops but
-  // compressed oops don't work with CDS (offsets in metadata are wrong), so
-  // borrow the same address.
+  // This allocates memory with mmap.  For DumpSharedspaces, try to reserve
+  // configurable address, generally at the top of the Java heap so other
+  // memory addresses don't conflict.
   if (DumpSharedSpaces) {
-    char* shared_base = (char*)HeapBaseMinAddress;
+    char* shared_base = (char*)SharedBaseAddress;
     _rs = ReservedSpace(byte_size, 0, false, shared_base, 0);
     if (_rs.is_reserved()) {
-      assert(_rs.base() == shared_base, "should match");
+      assert(shared_base == 0 || _rs.base() == shared_base, "should match");
     } else {
-      // If we are dumping the heap, then allocate a wasted block of address
-      // space in order to push the heap to a lower address.  This extra
-      // address range allows for other (or larger) libraries to be loaded
-      // without them occupying the space required for the shared spaces.
-      uintx reserved = 0;
-      uintx block_size = 64*1024*1024;
-      while (reserved < SharedDummyBlockSize) {
-        char* dummy = os::reserve_memory(block_size);
-        reserved += block_size;
-      }
+      // Get a mmap region anywhere if the SharedBaseAddress fails.
       _rs = ReservedSpace(byte_size);
     }
     MetaspaceShared::set_shared_rs(&_rs);

src/share/vm/runtime/globals.hpp

@@ -3573,8 +3573,9 @@ class CommandLineFlags {
   product(uintx, SharedMiscCodeSize,    120*K,                              \
           "Size of the shared miscellaneous code area (in bytes)")          \
                                                                             \
-  product(uintx, SharedDummyBlockSize, 0,                                   \
-          "Size of dummy block used to shift heap addresses (in bytes)")    \
+  product(uintx, SharedBaseAddress, LP64_ONLY(32*G)                         \
+          NOT_LP64(LINUX_ONLY(2*G) NOT_LINUX(0)),                           \
+          "Address to allocate shared memory region for class data")        \
                                                                             \
   diagnostic(bool, EnableInvokeDynamic, true,                               \
           "support JSR 292 (method handles, invokedynamic, "                \