7197666: java -d64 -version core dumps in a box with lots of memory

Summary: Allow task queues to be mmapped instead of malloced on Solaris
Reviewed-by: coleenp, jmasa, johnc, tschatzl

src/share/vm/memory/allocation.hpp

@@ -611,4 +611,23 @@ public:
   void check()    PRODUCT_RETURN;
 };
 
+// Helper class to allocate arrays that may become large.
+// Uses the OS malloc for allocations smaller than ArrayAllocatorMallocLimit
+// and uses mapped memory for larger allocations.
+// Most OS mallocs do something similar but Solaris malloc does not revert
+// to mapped memory for large allocations. By default ArrayAllocatorMallocLimit
+// is set so that we always use malloc except for Solaris where we set the
+// limit to get mapped memory.
+template <class E, MEMFLAGS F>
+class ArrayAllocator : StackObj {
+  char* _addr;
+  bool _use_malloc;
+  size_t _size;
+ public:
+  ArrayAllocator() : _addr(NULL), _use_malloc(false), _size(0) { }
+  ~ArrayAllocator() { free(); }
+  E* allocate(size_t length);
+  void free();
+};
+
 #endif // SHARE_VM_MEMORY_ALLOCATION_HPP

src/share/vm/memory/allocation.inline.hpp

@@ -108,5 +108,49 @@ template <MEMFLAGS F> void CHeapObj<F>::operator delete(void* p){
    FreeHeap(p, F);
 }
 
+template <class E, MEMFLAGS F>
+E* ArrayAllocator<E, F>::allocate(size_t length) {
+  assert(_addr == NULL, "Already in use");
+
+  _size = sizeof(E) * length;
+  _use_malloc = _size < ArrayAllocatorMallocLimit;
+
+  if (_use_malloc) {
+    _addr = AllocateHeap(_size, F);
+    if (_addr == NULL && _size >=  (size_t)os::vm_allocation_granularity()) {
+      // malloc failed let's try with mmap instead
+      _use_malloc = false;
+    } else {
+      return (E*)_addr;
+    }
+  }
+
+  int alignment = os::vm_allocation_granularity();
+  _size = align_size_up(_size, alignment);
+
+  _addr = os::reserve_memory(_size, NULL, alignment);
+  if (_addr == NULL) {
+    vm_exit_out_of_memory(_size, "Allocator (reserve)");
+  }
+
+  bool success = os::commit_memory(_addr, _size, false /* executable */);
+  if (!success) {
+    vm_exit_out_of_memory(_size, "Allocator (commit)");
+  }
+
+  return (E*)_addr;
+}
+
+template<class E, MEMFLAGS F>
+void ArrayAllocator<E, F>::free() {
+  if (_addr != NULL) {
+    if (_use_malloc) {
+      FreeHeap(_addr, F);
+    } else {
+      os::release_memory(_addr, _size);
+    }
+    _addr = NULL;
+  }
+}
 
 #endif // SHARE_VM_MEMORY_ALLOCATION_INLINE_HPP

src/share/vm/runtime/globals.hpp

@@ -3668,8 +3668,13 @@ class CommandLineFlags {
   product(bool, PrintGCCause, true,                                         \
           "Include GC cause in GC logging")                                 \
                                                                             \
-  product(bool, AllowNonVirtualCalls, false,                                \
-          "Obey the ACC_SUPER flag and allow invokenonvirtual calls")
+  product(bool , AllowNonVirtualCalls, false,                               \
+          "Obey the ACC_SUPER flag and allow invokenonvirtual calls")       \
+                                                                            \
+  experimental(uintx, ArrayAllocatorMallocLimit,                            \
+          SOLARIS_ONLY(64*K) NOT_SOLARIS(max_uintx),                        \
+          "Allocation less than this value will be allocated "              \
+          "using malloc. Larger allocations will use mmap.")
 
 /*
  *  Macros for factoring of globals

src/share/vm/utilities/taskqueue.hpp

@@ -253,6 +253,7 @@ public:
 
 template <class E, MEMFLAGS F, unsigned int N = TASKQUEUE_SIZE>
 class GenericTaskQueue: public TaskQueueSuper<N, F> {
+  ArrayAllocator<E, F> _array_allocator;
 protected:
   typedef typename TaskQueueSuper<N, F>::Age Age;
   typedef typename TaskQueueSuper<N, F>::idx_t idx_t;
@@ -314,7 +315,7 @@ GenericTaskQueue<E, F, N>::GenericTaskQueue() {
 
 template<class E, MEMFLAGS F, unsigned int N>
 void GenericTaskQueue<E, F, N>::initialize() {
-  _elems = NEW_C_HEAP_ARRAY(E, N, F);
+  _elems = _array_allocator.allocate(N);
 }
 
 template<class E, MEMFLAGS F, unsigned int N>