6694099: Hotspot vm_exit_out_of_memory should dump core

Summary: This fix enables the generation of core file when process runs out of C-heap.
Reviewed-by: sbohne

src/share/vm/prims/jni.cpp

@@ -631,7 +631,7 @@ JNI_ENTRY(void, jni_FatalError(JNIEnv *env, const char *msg))
   DTRACE_PROBE2(hotspot_jni, FatalError__entry, env, msg);
   tty->print_cr("FATAL ERROR in native method: %s", msg);
   thread->print_stack();
-  os::abort(false); // Prevent core dump, causes a jck failure.
+  os::abort(); // Dump core and abort
 JNI_END
 
 

src/share/vm/runtime/java.cpp

@@ -502,9 +502,9 @@ void vm_shutdown()
   os::shutdown();
 }
 
-void vm_abort() {
+void vm_abort(bool dump_core) {
   vm_perform_shutdown_actions();
-  os::abort(PRODUCT_ONLY(false));
+  os::abort(dump_core);
   ShouldNotReachHere();
 }
 
@@ -538,18 +538,24 @@ void vm_exit_during_initialization(Handle exception) {
   java_lang_Throwable::print_stack_trace(exception(), tty);
   tty->cr();
   vm_notify_during_shutdown(NULL, NULL);
-  vm_abort();
+
+  // Failure during initialization, we don't want to dump core
+  vm_abort(false);
 }
 
 void vm_exit_during_initialization(symbolHandle ex, const char* message) {
   ResourceMark rm;
   vm_notify_during_shutdown(ex->as_C_string(), message);
-  vm_abort();
+
+  // Failure during initialization, we don't want to dump core
+  vm_abort(false);
 }
 
 void vm_exit_during_initialization(const char* error, const char* message) {
   vm_notify_during_shutdown(error, message);
-  vm_abort();
+
+  // Failure during initialization, we don't want to dump core
+  vm_abort(false);
 }
 
 void vm_shutdown_during_initialization(const char* error, const char* message) {

src/share/vm/runtime/java.hpp

@@ -37,7 +37,7 @@ extern void vm_direct_exit(int code);
 // Shutdown the VM but do not exit the process
 extern void vm_shutdown();
 // Shutdown the VM and abort the process
-extern void vm_abort();
+extern void vm_abort(bool dump_core=true);
 
 // Trigger any necessary notification of the VM being shutdown
 extern void notify_vm_shutdown();

src/share/vm/utilities/debug.cpp

@@ -208,7 +208,9 @@ void report_vm_out_of_memory(const char* file_name, int line_no, size_t size, co
     Thread* thread = ThreadLocalStorage::get_thread_slow();
     VMError(thread, size, message, file_name, line_no).report_and_die();
   }
-  vm_abort();
+
+  // Dump core and abort
+  vm_abort(true);
 }
 
 void report_vm_out_of_memory_vararg(const char* file_name, int line_no, size_t size, const char* format, ...) {