6914050: jvm assertion "guard pages must be in use" in -Xcomp mode

Summary: Move creating stack guard pages in jni attach thread before potential java call rather than after. Also cleanup stack guard pages when jni attach fails
Reviewed-by: never, dholmes

src/share/vm/prims/jni.cpp

@@ -3401,12 +3401,16 @@ static jint attach_current_thread(JavaVM *vm, void **penv, void *_args, bool dae
   thread->set_thread_state(_thread_in_vm);
   // Must do this before initialize_thread_local_storage
   thread->record_stack_base_and_size();
+
   thread->initialize_thread_local_storage();
 
   if (!os::create_attached_thread(thread)) {
     delete thread;
     return JNI_ERR;
   }
+  // Enable stack overflow checks
+  thread->create_stack_guard_pages();
+
   thread->initialize_tlab();
 
   // Crucial that we do not have a safepoint check for this thread, since it has
@@ -3452,9 +3456,6 @@ static jint attach_current_thread(JavaVM *vm, void **penv, void *_args, bool dae
   // to regrab the threads_lock
   thread->set_attached();
 
-  // Enable stack overflow checks
-  thread->create_stack_guard_pages();
-
   // Set java thread status.
   java_lang_Thread::set_thread_status(thread->threadObj(),
               java_lang_Thread::RUNNABLE);

src/share/vm/runtime/thread.cpp

@@ -1637,6 +1637,9 @@ void JavaThread::cleanup_failed_attach_current_thread() {
     JNIHandleBlock::release_block(block);
   }
 
+  // These have to be removed while this is still a valid thread.
+  remove_stack_guard_pages();
+
   if (UseTLAB) {
     tlab().make_parsable(true);  // retire TLAB, if any
   }