6961186: Better VM handling of unexpected exceptions from application native code

Summary: Trap uncaught C++ exception on Windows and Solaris and generate hs_err report.
Reviewed-by: coleenp, bobv, dholmes

src/os/solaris/vm/os_solaris.cpp

@@ -80,6 +80,7 @@
 // put OS-includes here
 # include <dlfcn.h>
 # include <errno.h>
+# include <exception>
 # include <link.h>
 # include <poll.h>
 # include <pthread.h>
@@ -1475,6 +1476,13 @@ sigset_t* os::Solaris::allowdebug_blocked_signals() {
   return &allowdebug_blocked_sigs;
 }
 
+
+void _handle_uncaught_cxx_exception() {
+  VMError err("An uncaught C++ exception");
+  err.report_and_die();
+}
+
+
 // First crack at OS-specific initialization, from inside the new thread.
 void os::initialize_thread() {
   int r = thr_main() ;
@@ -1564,6 +1572,7 @@ void os::initialize_thread() {
    // use the dynamic check for T2 libthread.
 
   os::Solaris::init_thread_fpu_state();
+  std::set_terminate(_handle_uncaught_cxx_exception);
 }
 
 

src/os/windows/vm/os_windows.cpp

 /*
- * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * CopyrighT (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2007,6 +2007,16 @@ struct siglabel {
   int   number;
 };
 
+// All Visual C++ exceptions thrown from code generated by the Microsoft Visual
+// C++ compiler contain this error code. Because this is a compiler-generated
+// error, the code is not listed in the Win32 API header files.
+// The code is actually a cryptic mnemonic device, with the initial "E"
+// standing for "exception" and the final 3 bytes (0x6D7363) representing the
+// ASCII values of "msc".
+
+#define EXCEPTION_UNCAUGHT_CXX_EXCEPTION    0xE06D7363
+
+
 struct siglabel exceptlabels[] = {
     def_excpt(EXCEPTION_ACCESS_VIOLATION),
     def_excpt(EXCEPTION_DATATYPE_MISALIGNMENT),
@@ -2031,6 +2041,7 @@ struct siglabel exceptlabels[] = {
     def_excpt(EXCEPTION_INVALID_DISPOSITION),
     def_excpt(EXCEPTION_GUARD_PAGE),
     def_excpt(EXCEPTION_INVALID_HANDLE),
+    def_excpt(EXCEPTION_UNCAUGHT_CXX_EXCEPTION),
     NULL, 0
 };
 
@@ -2264,7 +2275,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
     }
   }
 
-
   if (t != NULL && t->is_Java_thread()) {
     JavaThread* thread = (JavaThread*) t;
     bool in_java = thread->thread_state() == _thread_in_Java;
@@ -2468,8 +2478,9 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
       } // switch
     }
 #ifndef _WIN64
-    if ((thread->thread_state() == _thread_in_Java) ||
-        (thread->thread_state() == _thread_in_native) )
+    if (((thread->thread_state() == _thread_in_Java) ||
+        (thread->thread_state() == _thread_in_native)) &&
+        exception_code != EXCEPTION_UNCAUGHT_CXX_EXCEPTION)
     {
       LONG result=Handle_FLT_Exception(exceptionInfo);
       if (result==EXCEPTION_CONTINUE_EXECUTION) return result;
@@ -2493,6 +2504,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
       case EXCEPTION_ILLEGAL_INSTRUCTION_2:
       case EXCEPTION_INT_OVERFLOW:
       case EXCEPTION_INT_DIVIDE_BY_ZERO:
+      case EXCEPTION_UNCAUGHT_CXX_EXCEPTION:
       {  report_error(t, exception_code, pc, exceptionInfo->ExceptionRecord,
                        exceptionInfo->ContextRecord);
       }