6302804: Hotspot VM dies ungraceful death when C heap is exhausted in various places.

Summary: enhance the error reporting mechanism to help user to fix the problem rather than making it look like a VM error.
Reviewed-by: kvn, kamg

src/os_cpu/solaris_sparc/vm/os_solaris_sparc.cpp

@@ -585,6 +585,13 @@ int JVM_handle_solaris_signal(int sig, siginfo_t* info, void* ucVoid, int abort_
   sigaddset(&newset, sig);
   sigprocmask(SIG_UNBLOCK, &newset, NULL);
 
+  // Determine which sort of error to throw.  Out of swap may signal
+  // on the thread stack, which could get a mapping error when touched.
+  address addr = (address) info->si_addr;
+  if (sig == SIGBUS && info->si_code == BUS_OBJERR && info->si_errno == ENOMEM) {
+    vm_exit_out_of_memory(0, "Out of swap space to map in thread stack.");
+  }
+
   VMError err(t, sig, pc, info, ucVoid);
   err.report_and_die();
 

src/os_cpu/solaris_x86/vm/os_solaris_x86.cpp

@@ -742,6 +742,13 @@ int JVM_handle_solaris_signal(int sig, siginfo_t* info, void* ucVoid, int abort_
   sigaddset(&newset, sig);
   sigprocmask(SIG_UNBLOCK, &newset, NULL);
 
+  // Determine which sort of error to throw.  Out of swap may signal
+  // on the thread stack, which could get a mapping error when touched.
+  address addr = (address) info->si_addr;
+  if (sig == SIGBUS && info->si_code == BUS_OBJERR && info->si_errno == ENOMEM) {
+    vm_exit_out_of_memory(0, "Out of swap space to map in thread stack.");
+  }
+
   VMError err(t, sig, pc, info, ucVoid);
   err.report_and_die();
 

src/share/vm/runtime/arguments.cpp

@@ -2297,14 +2297,15 @@ jint Arguments::parse_each_vm_init_arg(const JavaVMInitArgs* args,
     } else if (match_option(option, "-Xoss", &tail)) {
           // HotSpot does not have separate native and Java stacks, ignore silently for compatibility
     // -Xmaxjitcodesize
-    } else if (match_option(option, "-Xmaxjitcodesize", &tail)) {
+    } else if (match_option(option, "-Xmaxjitcodesize", &tail) ||
+               match_option(option, "-XX:ReservedCodeCacheSize=", &tail)) {
       julong long_ReservedCodeCacheSize = 0;
       ArgsRange errcode = parse_memory_size(tail, &long_ReservedCodeCacheSize,
                                             (size_t)InitialCodeCacheSize);
       if (errcode != arg_in_range) {
         jio_fprintf(defaultStream::error_stream(),
-                    "Invalid maximum code cache size: %s\n",
-                    option->optionString);
+                    "Invalid maximum code cache size: %s. Should be greater than InitialCodeCacheSize=%dK\n",
+                    option->optionString, InitialCodeCacheSize/K);
         describe_range_error(errcode);
         return JNI_EINVAL;
       }

src/share/vm/utilities/debug.cpp

@@ -226,7 +226,7 @@ static jint _exiting_out_of_mem = 0;
 
 void report_vm_out_of_memory(const char* file, int line, size_t size,
                              const char* message) {
-  if (Debugging || error_is_suppressed(file, line)) return;
+  if (Debugging) return;
 
   // We try to gather additional information for the first out of memory
   // error only; gathering additional data might cause an allocation and a

src/share/vm/utilities/vmError.cpp

@@ -67,7 +67,7 @@ const char *env_list[] = {
 // threads are blocked forever inside report_and_die().
 
 // Constructor for crashes
-VMError::VMError(Thread* thread, int sig, address pc, void* siginfo, void* context) {
+VMError::VMError(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context) {
     _thread = thread;
     _id = sig;
     _pc   = pc;
@@ -323,28 +323,50 @@ void VMError::report(outputStream* st) {
   STEP(10, "(printing fatal error message)")
 
     st->print_cr("#");
+    if (should_report_bug(_id)) {
       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
+    } else {
+      st->print_cr("# There is insufficient memory for the Java "
+                   "Runtime Environment to continue.");
+    }
 
   STEP(15, "(printing type of error)")
 
      switch(_id) {
        case oom_error:
-         st->print_cr("#");
-         st->print("# java.lang.OutOfMemoryError: ");
          if (_size) {
-           st->print("requested ");
-           sprintf(buf,SIZE_FORMAT,_size);
+           st->print("# Native memory allocation (malloc) failed to allocate ");
+           jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
            st->print(buf);
            st->print(" bytes");
            if (_message != NULL) {
              st->print(" for ");
              st->print(_message);
            }
-           st->print_cr(". Out of swap space?");
+           st->cr();
          } else {
            if (_message != NULL)
+             st->print("# ");
              st->print_cr(_message);
          }
+         // In error file give some solutions
+         if (_verbose) {
+           st->print_cr("# Possible reasons:");
+           st->print_cr("#   The system is out of physical RAM or swap space");
+           st->print_cr("#   In 32 bit mode, the process size limit was hit");
+           st->print_cr("# Possible solutions:");
+           st->print_cr("#   Reduce memory load on the system");
+           st->print_cr("#   Increase physical memory or swap space");
+           st->print_cr("#   Check if swap backing store is full");
+           st->print_cr("#   Use 64 bit Java on a 64 bit OS");
+           st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
+           st->print_cr("#   Decrease number of Java threads");
+           st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
+           st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
+           st->print_cr("# This output file may be truncated or incomplete.");
+         } else {
+           return;  // that's enough for the screen
+         }
          break;
        case internal_error:
        default:
@@ -361,7 +383,11 @@ void VMError::report(outputStream* st) {
        st->print(" (0x%x)", _id);                // signal number
        st->print(" at pc=" PTR_FORMAT, _pc);
      } else {
+       if (should_report_bug(_id)) {
          st->print("Internal Error");
+       } else {
+         st->print("Out of Memory Error");
+       }
        if (_filename != NULL && _lineno > 0) {
 #ifdef PRODUCT
          // In product mode chop off pathname?
@@ -393,12 +419,14 @@ void VMError::report(outputStream* st) {
 
   STEP(40, "(printing error message)")
 
+     if (should_report_bug(_id)) {  // already printed the message.
        // error message
        if (_detail_msg) {
          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
        } else if (_message) {
          st->print_cr("#  Error: %s", _message);
        }
+    }
 
   STEP(50, "(printing Java version string)")
 
@@ -428,7 +456,9 @@ void VMError::report(outputStream* st) {
 
   STEP(65, "(printing bug submit message)")
 
-     if (_verbose) print_bug_submit_message(st, _thread);
+     if (should_report_bug(_id) && _verbose) {
+       print_bug_submit_message(st, _thread);
+     }
 
   STEP(70, "(printing thread)" )
 
@@ -906,7 +936,7 @@ void VMError::report_and_die() {
     OnError = NULL;
   }
 
-  static bool skip_bug_url = false;
+  static bool skip_bug_url = !should_report_bug(first_error->_id);
   if (!skip_bug_url) {
     skip_bug_url = true;
 
@@ -919,7 +949,8 @@ void VMError::report_and_die() {
     static bool skip_os_abort = false;
     if (!skip_os_abort) {
       skip_os_abort = true;
-      os::abort();
+      bool dump_core = should_report_bug(first_error->_id);
+      os::abort(dump_core);
     }
 
     // if os::abort() doesn't abort, try os::die();

src/share/vm/utilities/vmError.hpp

@@ -87,10 +87,12 @@ class VMError : public StackObj {
   // accessor
   const char* message() const    { return _message; }
   const char* detail_msg() const { return _detail_msg; }
+  bool should_report_bug(unsigned int id) { return id != oom_error; }
 
 public:
   // Constructor for crashes
-  VMError(Thread* thread, int sig, address pc, void* siginfo, void* context);
+  VMError(Thread* thread, unsigned int sig, address pc, void* siginfo,
+          void* context);
   // Constructor for VM internal errors
   VMError(Thread* thread, const char* filename, int lineno,
           const char* message, const char * detail_msg);