6983240: guarantee((Solaris::min_stack_allowed >= (StackYellowPages+StackRedPages...) wrong

Summary: min_stack_allowed is a compile time constant and Stack*Pages are settable
Reviewed-by: dholmes, kvn

src/cpu/x86/vm/methodHandles_x86.cpp

@@ -346,7 +346,7 @@ void trace_method_handle_stub(const char* adaptername,
     if (stack_dump_count > 64)  stack_dump_count = 48;
     for (i = 0; i < stack_dump_count; i += 4) {
       printf(" dump at SP[%d] "INTPTR_FORMAT": "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT"\n",
-             i, &entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
+             i, (intptr_t)&entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
     }
     print_method_handle(mh);
   }

src/os/linux/vm/os_linux.cpp

@@ -827,8 +827,10 @@ bool os::create_thread(Thread* thread, ThreadType thr_type, size_t stack_size) {
 
       switch (thr_type) {
       case os::java_thread:
-        // Java threads use ThreadStackSize which default value can be changed with the flag -Xss
-        if (JavaThread::stack_size_at_create() > 0) stack_size = JavaThread::stack_size_at_create();
+        // Java threads use ThreadStackSize which default value can be
+        // changed with the flag -Xss
+        assert (JavaThread::stack_size_at_create() > 0, "this should be set");
+        stack_size = JavaThread::stack_size_at_create();
         break;
       case os::compiler_thread:
         if (CompilerThreadStackSize > 0) {
@@ -3922,12 +3924,21 @@ jint os::init_2(void)
   Linux::signal_sets_init();
   Linux::install_signal_handlers();
 
+  // Check minimum allowable stack size for thread creation and to initialize
+  // the java system classes, including StackOverflowError - depends on page
+  // size.  Add a page for compiler2 recursion in main thread.
+  // Add in 2*BytesPerWord times page size to account for VM stack during
+  // class initialization depending on 32 or 64 bit VM.
+  os::Linux::min_stack_allowed = MAX2(os::Linux::min_stack_allowed,
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+                    2*BytesPerWord COMPILER2_PRESENT(+1)) * Linux::page_size());
+
   size_t threadStackSizeInBytes = ThreadStackSize * K;
   if (threadStackSizeInBytes != 0 &&
-      threadStackSizeInBytes < Linux::min_stack_allowed) {
+      threadStackSizeInBytes < os::Linux::min_stack_allowed) {
         tty->print_cr("\nThe stack size specified is too small, "
                       "Specify at least %dk",
-                      Linux::min_stack_allowed / K);
+                      os::Linux::min_stack_allowed/ K);
         return JNI_ERR;
   }
 

src/os/solaris/vm/os_solaris.cpp

@@ -4878,18 +4878,17 @@ jint os::init_2(void) {
   // Check minimum allowable stack size for thread creation and to initialize
   // the java system classes, including StackOverflowError - depends on page
   // size.  Add a page for compiler2 recursion in main thread.
-  // Add in BytesPerWord times page size to account for VM stack during
+  // Add in 2*BytesPerWord times page size to account for VM stack during
   // class initialization depending on 32 or 64 bit VM.
-  guarantee((Solaris::min_stack_allowed >=
-    (StackYellowPages+StackRedPages+StackShadowPages+BytesPerWord
-     COMPILER2_PRESENT(+1)) * page_size),
-    "need to increase Solaris::min_stack_allowed on this platform");
+  os::Solaris::min_stack_allowed = MAX2(os::Solaris::min_stack_allowed,
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+                    2*BytesPerWord COMPILER2_PRESENT(+1)) * page_size);
 
   size_t threadStackSizeInBytes = ThreadStackSize * K;
   if (threadStackSizeInBytes != 0 &&
-    threadStackSizeInBytes < Solaris::min_stack_allowed) {
+    threadStackSizeInBytes < os::Solaris::min_stack_allowed) {
     tty->print_cr("\nThe stack size specified is too small, Specify at least %dk",
-                  Solaris::min_stack_allowed/K);
+                  os::Solaris::min_stack_allowed/K);
     return JNI_ERR;
   }
 

src/os/windows/vm/os_windows.cpp

@@ -3311,7 +3311,6 @@ extern "C" {
   }
 }
 
-
 // this is called _after_ the global arguments have been parsed
 jint os::init_2(void) {
   // Allocate a single page and mark it as readable for safepoint polling
@@ -3390,6 +3389,21 @@ jint os::init_2(void) {
     actual_reserve_size = default_reserve_size;
   }
 
+  // Check minimum allowable stack size for thread creation and to initialize
+  // the java system classes, including StackOverflowError - depends on page
+  // size.  Add a page for compiler2 recursion in main thread.
+  // Add in 2*BytesPerWord times page size to account for VM stack during
+  // class initialization depending on 32 or 64 bit VM.
+  size_t min_stack_allowed =
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+            2*BytesPerWord COMPILER2_PRESENT(+1)) * os::vm_page_size();
+  if (actual_reserve_size < min_stack_allowed) {
+    tty->print_cr("\nThe stack size specified is too small, "
+                  "Specify at least %dk",
+                  min_stack_allowed / K);
+    return JNI_ERR;
+  }
+
   JavaThread::set_stack_size_at_create(stack_commit_size);
 
   // Calculate theoretical max. size of Threads to guard gainst artifical

src/share/vm/runtime/arguments.cpp

@@ -1663,7 +1663,8 @@ bool Arguments::check_stack_pages()
   bool status = true;
   status = status && verify_min_value(StackYellowPages, 1, "StackYellowPages");
   status = status && verify_min_value(StackRedPages, 1, "StackRedPages");
-  status = status && verify_min_value(StackShadowPages, 1, "StackShadowPages");
+  // greater stack shadow pages can't generate instruction to bang stack
+  status = status && verify_interval(StackShadowPages, 1, 50, "StackShadowPages");
   return status;
 }
 

src/share/vm/utilities/exceptions.cpp

@@ -61,6 +61,18 @@ bool Exceptions::special_exception(Thread* thread, const char* file, int line, H
    ShouldNotReachHere();
   }
 
+#ifdef ASSERT
+  // Check for trying to throw stack overflow before initialization is complete
+  // to prevent infinite recursion trying to initialize stack overflow without
+  // adequate stack space.
+  // This can happen with stress testing a large value of StackShadowPages
+  if (h_exception()->klass() == SystemDictionary::StackOverflowError_klass()) {
+    instanceKlass* ik = instanceKlass::cast(h_exception->klass());
+    assert(ik->is_initialized(),
+           "need to increase min_stack_allowed calculation");
+  }
+#endif // ASSERT
+
   if (thread->is_VM_thread()
       || thread->is_Compiler_thread() ) {
     // We do not care what kind of exception we get for the vm-thread or a thread which
@@ -91,7 +103,6 @@ bool Exceptions::special_exception(Thread* thread, const char* file, int line, s
     thread->set_pending_exception(Universe::vm_exception(), file, line);
     return true;
   }
-
   return false;
 }
 
@@ -193,6 +204,7 @@ void Exceptions::throw_stack_overflow_exception(Thread* THREAD, const char* file
     klassOop k = SystemDictionary::StackOverflowError_klass();
     oop e = instanceKlass::cast(k)->allocate_instance(CHECK);
     exception = Handle(THREAD, e);  // fill_in_stack trace does gc
+    assert(instanceKlass::cast(k)->is_initialized(), "need to increase min_stack_allowed calculation");
     if (StackTraceInThrowable) {
       java_lang_Throwable::fill_in_stack_trace(exception);
     }