8028275: Metaspace ShrinkGrowTest causes fatal error if run with JFR

Summary: Clean up initialization from Threads::create_vm() so that exceptions cause vm_exit_during_initialzation without an exception mark.
Reviewed-by: dholmes, hseigel

hotspot/src/share/vm/oops/constantPool.cpp

 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1874,7 +1874,6 @@ void ConstantPool::preload_and_initialize_all_classes(ConstantPool* obj, TRAPS)
 // Printing
 
 void ConstantPool::print_on(outputStream* st) const {
-  EXCEPTION_MARK;
   assert(is_constantPool(), "must be constantPool");
   st->print_cr(internal_name());
   if (flags() != 0) {

hotspot/src/share/vm/prims/jni.cpp

 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -5053,6 +5053,7 @@ _JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_CreateJavaVM(JavaVM **vm, void **penv, v
   result = Threads::create_vm((JavaVMInitArgs*) args, &can_try_again);
   if (result == JNI_OK) {
     JavaThread *thread = JavaThread::current();
+    assert(!thread->has_pending_exception(), "should have returned not OK");
     /* thread is thread_in_vm here */
     *vm = (JavaVM *)(&main_vm);
     *(JNIEnv**)penv = thread->jni_environment();
@@ -5089,6 +5090,19 @@ _JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_CreateJavaVM(JavaVM **vm, void **penv, v
     // Since this is not a JVM_ENTRY we have to set the thread state manually before leaving.
     ThreadStateTransition::transition_and_fence(thread, _thread_in_vm, _thread_in_native);
   } else {
+    // If create_vm exits because of a pending exception, exit with that
+    // exception.  In the future when we figure out how to reclaim memory,
+    // we may be able to exit with JNI_ERR and allow the calling application
+    // to continue.
+    if (Universe::is_fully_initialized()) {
+      // otherwise no pending exception possible - VM will already have aborted
+      JavaThread* THREAD = JavaThread::current();
+      if (HAS_PENDING_EXCEPTION) {
+        HandleMark hm;
+        vm_exit_during_initialization(Handle(THREAD, PENDING_EXCEPTION));
+      }
+    }
+
     if (can_try_again) {
       // reset safe_to_recreate_vm to 1 so that retrial would be possible
       safe_to_recreate_vm = 1;

hotspot/src/share/vm/runtime/thread.cpp

 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -3301,6 +3301,58 @@ void Threads::threads_do(ThreadClosure* tc) {
   // If CompilerThreads ever become non-JavaThreads, add them here
 }
 
+
+void Threads::initialize_java_lang_classes(JavaThread* main_thread, TRAPS) {
+  TraceTime timer("Initialize java.lang classes", TraceStartupTime);
+
+  if (EagerXrunInit && Arguments::init_libraries_at_startup()) {
+    create_vm_init_libraries();
+  }
+
+  initialize_class(vmSymbols::java_lang_String(), CHECK);
+
+  // Initialize java_lang.System (needed before creating the thread)
+  initialize_class(vmSymbols::java_lang_System(), CHECK);
+  initialize_class(vmSymbols::java_lang_ThreadGroup(), CHECK);
+  Handle thread_group = create_initial_thread_group(CHECK);
+  Universe::set_main_thread_group(thread_group());
+  initialize_class(vmSymbols::java_lang_Thread(), CHECK);
+  oop thread_object = create_initial_thread(thread_group, main_thread, CHECK);
+  main_thread->set_threadObj(thread_object);
+  // Set thread status to running since main thread has
+  // been started and running.
+  java_lang_Thread::set_thread_status(thread_object,
+                                      java_lang_Thread::RUNNABLE);
+
+  // The VM creates & returns objects of this class. Make sure it's initialized.
+  initialize_class(vmSymbols::java_lang_Class(), CHECK);
+
+  // The VM preresolves methods to these classes. Make sure that they get initialized
+  initialize_class(vmSymbols::java_lang_reflect_Method(), CHECK);
+  initialize_class(vmSymbols::java_lang_ref_Finalizer(),  CHECK);
+  call_initializeSystemClass(CHECK);
+
+  // get the Java runtime name after java.lang.System is initialized
+  JDK_Version::set_runtime_name(get_java_runtime_name(THREAD));
+  JDK_Version::set_runtime_version(get_java_runtime_version(THREAD));
+
+  // an instance of OutOfMemory exception has been allocated earlier
+  initialize_class(vmSymbols::java_lang_OutOfMemoryError(), CHECK);
+  initialize_class(vmSymbols::java_lang_NullPointerException(), CHECK);
+  initialize_class(vmSymbols::java_lang_ClassCastException(), CHECK);
+  initialize_class(vmSymbols::java_lang_ArrayStoreException(), CHECK);
+  initialize_class(vmSymbols::java_lang_ArithmeticException(), CHECK);
+  initialize_class(vmSymbols::java_lang_StackOverflowError(), CHECK);
+  initialize_class(vmSymbols::java_lang_IllegalMonitorStateException(), CHECK);
+  initialize_class(vmSymbols::java_lang_IllegalArgumentException(), CHECK);
+}
+
+void Threads::initialize_jsr292_core_classes(TRAPS) {
+  initialize_class(vmSymbols::java_lang_invoke_MethodHandle(), CHECK);
+  initialize_class(vmSymbols::java_lang_invoke_MemberName(), CHECK);
+  initialize_class(vmSymbols::java_lang_invoke_MethodHandleNatives(), CHECK);
+}
+
 jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
 
   extern void JDK_Version_init();
@@ -3470,13 +3522,13 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
     VMThread::execute(&verify_op);
   }
 
-  EXCEPTION_MARK;
+  Thread* THREAD = Thread::current();
 
   // At this point, the Universe is initialized, but we have not executed
   // any byte code.  Now is a good time (the only time) to dump out the
   // internal state of the JVM for sharing.
   if (DumpSharedSpaces) {
-    MetaspaceShared::preload_and_dump(CHECK_0);
+    MetaspaceShared::preload_and_dump(CHECK_JNI_ERR);
     ShouldNotReachHere();
   }
 
@@ -3487,74 +3539,12 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // Notify JVMTI agents that VM has started (JNI is up) - nop if no agents.
   JvmtiExport::post_vm_start();
 
-  {
-    TraceTime timer("Initialize java.lang classes", TraceStartupTime);
-
-    if (EagerXrunInit && Arguments::init_libraries_at_startup()) {
-      create_vm_init_libraries();
-    }
+  initialize_java_lang_classes(main_thread, CHECK_JNI_ERR);
 
-    initialize_class(vmSymbols::java_lang_String(), CHECK_0);
-
-    // Initialize java_lang.System (needed before creating the thread)
-    initialize_class(vmSymbols::java_lang_System(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_ThreadGroup(), CHECK_0);
-    Handle thread_group = create_initial_thread_group(CHECK_0);
-    Universe::set_main_thread_group(thread_group());
-    initialize_class(vmSymbols::java_lang_Thread(), CHECK_0);
-    oop thread_object = create_initial_thread(thread_group, main_thread, CHECK_0);
-    main_thread->set_threadObj(thread_object);
-    // Set thread status to running since main thread has
-    // been started and running.
-    java_lang_Thread::set_thread_status(thread_object,
-                                        java_lang_Thread::RUNNABLE);
-
-    // The VM creates & returns objects of this class. Make sure it's initialized.
-    initialize_class(vmSymbols::java_lang_Class(), CHECK_0);
-
-    // The VM preresolves methods to these classes. Make sure that they get initialized
-    initialize_class(vmSymbols::java_lang_reflect_Method(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_ref_Finalizer(),  CHECK_0);
-    call_initializeSystemClass(CHECK_0);
-
-    // get the Java runtime name after java.lang.System is initialized
-    JDK_Version::set_runtime_name(get_java_runtime_name(THREAD));
-    JDK_Version::set_runtime_version(get_java_runtime_version(THREAD));
-
-    // an instance of OutOfMemory exception has been allocated earlier
-    initialize_class(vmSymbols::java_lang_OutOfMemoryError(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_NullPointerException(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_ClassCastException(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_ArrayStoreException(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_ArithmeticException(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_StackOverflowError(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_IllegalMonitorStateException(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_IllegalArgumentException(), CHECK_0);
-  }
-
-  // See        : bugid 4211085.
-  // Background : the static initializer of java.lang.Compiler tries to read
-  //              property"java.compiler" and read & write property "java.vm.info".
-  //              When a security manager is installed through the command line
-  //              option "-Djava.security.manager", the above properties are not
-  //              readable and the static initializer for java.lang.Compiler fails
-  //              resulting in a NoClassDefFoundError.  This can happen in any
-  //              user code which calls methods in java.lang.Compiler.
-  // Hack :       the hack is to pre-load and initialize this class, so that only
-  //              system domains are on the stack when the properties are read.
-  //              Currently even the AWT code has calls to methods in java.lang.Compiler.
-  //              On the classic VM, java.lang.Compiler is loaded very early to load the JIT.
-  // Future Fix : the best fix is to grant everyone permissions to read "java.compiler" and
-  //              read and write"java.vm.info" in the default policy file. See bugid 4211383
-  //              Once that is done, we should remove this hack.
-  initialize_class(vmSymbols::java_lang_Compiler(), CHECK_0);
-
-  // More hackery - the static initializer of java.lang.Compiler adds the string "nojit" to
-  // the java.vm.info property if no jit gets loaded through java.lang.Compiler (the hotspot
-  // compiler does not get loaded through java.lang.Compiler).  "java -version" with the
-  // hotspot vm says "nojit" all the time which is confusing.  So, we reset it here.
-  // This should also be taken out as soon as 4211383 gets fixed.
-  reset_vm_info_property(CHECK_0);
+  // We need this for ClassDataSharing - the initial vm.info property is set
+  // with the default value of CDS "sharing" which may be reset through
+  // command line options.
+  reset_vm_info_property(CHECK_JNI_ERR);
 
   quicken_jni_functions();
 
@@ -3583,10 +3573,7 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // Note that we do not use CHECK_0 here since we are inside an EXCEPTION_MARK and
   // set_init_completed has just been called, causing exceptions not to be shortcut
   // anymore. We call vm_exit_during_initialization directly instead.
-  SystemDictionary::compute_java_system_loader(THREAD);
-  if (HAS_PENDING_EXCEPTION) {
-    vm_exit_during_initialization(Handle(THREAD, PENDING_EXCEPTION));
-  }
+  SystemDictionary::compute_java_system_loader(CHECK_JNI_ERR);
 
 #if INCLUDE_ALL_GCS
   // Support for ConcurrentMarkSweep. This should be cleaned up
@@ -3594,12 +3581,9 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // once things are properly refactored. XXX YSR
   if (UseConcMarkSweepGC || UseG1GC) {
     if (UseConcMarkSweepGC) {
-      ConcurrentMarkSweepThread::makeSurrogateLockerThread(THREAD);
+      ConcurrentMarkSweepThread::makeSurrogateLockerThread(CHECK_JNI_ERR);
     } else {
-      ConcurrentMarkThread::makeSurrogateLockerThread(THREAD);
-    }
-    if (HAS_PENDING_EXCEPTION) {
-      vm_exit_during_initialization(Handle(THREAD, PENDING_EXCEPTION));
+      ConcurrentMarkThread::makeSurrogateLockerThread(CHECK_JNI_ERR);
     }
   }
 #endif // INCLUDE_ALL_GCS
@@ -3642,19 +3626,16 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   CompileBroker::compilation_init();
 #endif
 
+  // Pre-initialize some JSR292 core classes to avoid deadlock during class loading.
+  // It is done after compilers are initialized, because otherwise compilations of
+  // signature polymorphic MH intrinsics can be missed
+  // (see SystemDictionary::find_method_handle_intrinsic).
   if (EnableInvokeDynamic) {
-    // Pre-initialize some JSR292 core classes to avoid deadlock during class loading.
-    // It is done after compilers are initialized, because otherwise compilations of
-    // signature polymorphic MH intrinsics can be missed
-    // (see SystemDictionary::find_method_handle_intrinsic).
-    initialize_class(vmSymbols::java_lang_invoke_MethodHandle(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_invoke_MemberName(), CHECK_0);
-    initialize_class(vmSymbols::java_lang_invoke_MethodHandleNatives(), CHECK_0);
+    initialize_jsr292_core_classes(CHECK_JNI_ERR);
   }
 
 #if INCLUDE_MANAGEMENT
   Management::initialize(THREAD);
-#endif // INCLUDE_MANAGEMENT
 
   if (HAS_PENDING_EXCEPTION) {
     // management agent fails to start possibly due to
@@ -3662,6 +3643,7 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
     // stack trace if appropriate. Simply exit VM.
     vm_exit(1);
   }
+#endif // INCLUDE_MANAGEMENT
 
   if (Arguments::has_profile())       FlatProfiler::engage(main_thread, true);
   if (MemProfiling)                   MemProfiler::engage();

hotspot/src/share/vm/runtime/thread.hpp

 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1891,6 +1891,8 @@ class Threads: AllStatic {
   static bool        _vm_complete;
 #endif
 
+  static void initialize_java_lang_classes(JavaThread* main_thread, TRAPS);
+  static void initialize_jsr292_core_classes(TRAPS);
  public:
   // Thread management
   // force_daemon is a concession to JNI, where we may need to add a

hotspot/src/share/vm/utilities/exceptions.hpp

 /*
- * Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -200,6 +200,7 @@ class Exceptions {
 #define CHECK_NH                                 CHECK_(Handle())
 #define CHECK_NULL                               CHECK_(NULL)
 #define CHECK_false                              CHECK_(false)
+#define CHECK_JNI_ERR                            CHECK_(JNI_ERR)
 
 #define CHECK_AND_CLEAR                         THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_EXCEPTION; return;        } (void)(0
 #define CHECK_AND_CLEAR_(result)                THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_EXCEPTION; return result; } (void)(0