8049226: com/sun/jdi/OptionTest.java test times out again

Summary: Don't call jni_FatalError if transport initialization fails
Reviewed-by: sspitsyn, sla

src/share/back/debugInit.c

@@ -1013,7 +1013,7 @@ static void
 atexit_finish_logging(void)
 {
     /* Normal exit(0) (not _exit()) may only reach here */
-    finish_logging(0);  /* Only first call matters */
+    finish_logging();  /* Only first call matters */
 }
 
 static jboolean
@@ -1301,43 +1301,49 @@ bad_option_no_msg:
 void
 debugInit_exit(jvmtiError error, const char *msg)
 {
-    int exit_code = 0;
+    enum exit_codes { EXIT_NO_ERRORS = 0, EXIT_JVMTI_ERROR = 1, EXIT_TRANSPORT_ERROR = 2 };
 
-    /* Pick an error code */
-    if ( error != JVMTI_ERROR_NONE ) {
-        exit_code = 1;
-        if ( docoredump ) {
+    // Prepare to exit. Log error and finish logging
+    LOG_MISC(("Exiting with error %s(%d): %s", jvmtiErrorText(error), error,
+                                               ((msg == NULL) ? "" : msg)));
+
+    // coredump requested by command line. Keep JVMTI data dirty
+    if (error != JVMTI_ERROR_NONE && docoredump) {
         LOG_MISC(("Dumping core as requested by command line"));
-            finish_logging(exit_code);
+        finish_logging();
         abort();
     }
-    }
 
-    if ( msg==NULL ) {
-        msg = "";
-    }
-
-    LOG_MISC(("Exiting with error %s(%d): %s", jvmtiErrorText(error), error, msg));
+    finish_logging();
 
+    // Cleanup the JVMTI if we have one
     if (gdata != NULL) {
         gdata->vmDead = JNI_TRUE;
-
-        /* Let's try and cleanup the JVMTI, if we even have one */
-        if ( gdata->jvmti != NULL ) {
-            /* Dispose of jvmti (gdata->jvmti becomes NULL) */
+        if (gdata->jvmti != NULL) {
+            // Dispose of jvmti (gdata->jvmti becomes NULL)
             disposeEnvironment(gdata->jvmti);
         }
     }
 
-    /* Finish up logging. We reach here if JDWP is doing the exiting. */
-    finish_logging(exit_code);  /* Only first call matters */
+    // We are here with no errors. Kill entire process and exit with zero exit code
+    if (error == JVMTI_ERROR_NONE) {
+        forceExit(EXIT_NO_ERRORS);
+        return;
+    }
 
-    /* Let's give the JNI a FatalError if non-exit 0, which is historic way */
-    if ( exit_code != 0 ) {
-        JNIEnv *env = NULL;
-        jniFatalError(env, msg, error, exit_code);
+    // No transport initilized.
+    // As we don't have any details here exiting with separate exit code
+    if (error == AGENT_ERROR_TRANSPORT_INIT) {
+        forceExit(EXIT_TRANSPORT_ERROR);
+        return;
     }
 
-    /* Last chance to die, this kills the entire process. */
-    forceExit(exit_code);
+    // We have JVMTI error. Call hotspot jni_FatalError handler
+    jniFatalError(NULL, msg, error, EXIT_JVMTI_ERROR);
+
+    // hotspot calls os:abort() so we should never reach code below,
+    // but guard against possible hotspot changes
+
+    // Last chance to die, this kills the entire process.
+    forceExit(EXIT_JVMTI_ERROR);
 }

src/share/back/log_messages.c

@@ -230,7 +230,7 @@ setup_logging(const char *filename, unsigned flags)
 
 /* Finish up logging, flush output to the logfile. */
 void
-finish_logging(int exit_code)
+finish_logging()
 {
 #ifdef JDWP_LOGGING
     MUTEX_LOCK(my_mutex);

src/share/back/log_messages.h

@@ -29,7 +29,7 @@
 /* LOG: Must be called like:  LOG_category(("anything")) or LOG_category((format,args)) */
 
 void setup_logging(const char *, unsigned);
-void finish_logging(int);
+void finish_logging();
 
 #define LOG_NULL ((void)0)