6727377: VM stack guard pages on Windows should PAGE_READWRITE not PAGE_EXECUTE_READWRITE

Summary: Make reguard_stack change access to RW, not execute and use os::protect_memory with the new parameter when change needed to X.
Reviewed-by: acorn, jcoomes

src/os/linux/vm/os_linux.cpp

@@ -2500,7 +2500,7 @@ bool os::guard_memory(char* addr, size_t size) {
 }
 
 bool os::unguard_memory(char* addr, size_t size) {
-  return linux_mprotect(addr, size, PROT_READ|PROT_WRITE|PROT_EXEC);
+  return linux_mprotect(addr, size, PROT_READ|PROT_WRITE);
 }
 
 // Large page support

src/os/solaris/vm/os_solaris.cpp

@@ -3026,6 +3026,8 @@ static bool solaris_mprotect(char* addr, size_t bytes, int prot) {
 
 // Protect memory (Used to pass readonly pages through
 // JNI GetArray<type>Elements with empty arrays.)
+// Also, used for serialization page and for compressed oops null pointer
+// checking.
 bool os::protect_memory(char* addr, size_t bytes, ProtType prot,
                         bool is_committed) {
   unsigned int p = 0;
@@ -3049,7 +3051,7 @@ bool os::guard_memory(char* addr, size_t bytes) {
 }
 
 bool os::unguard_memory(char* addr, size_t bytes) {
-  return solaris_mprotect(addr, bytes, PROT_READ|PROT_WRITE|PROT_EXEC);
+  return solaris_mprotect(addr, bytes, PROT_READ|PROT_WRITE);
 }
 
 // Large page support

src/os/windows/vm/os_windows.cpp

@@ -2020,10 +2020,11 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
         if (UnguardOnExecutionViolation > 0 && addr != last_addr &&
             (UnguardOnExecutionViolation > 1 || os::address_is_in_vm(addr))) {
 
-          // Unguard and retry
+          // Set memory to RWX and retry
           address page_start =
             (address) align_size_down((intptr_t) addr, (intptr_t) page_size);
-          bool res = os::unguard_memory((char*) page_start, page_size);
+          bool res = os::protect_memory((char*) page_start, page_size,
+                                        os::MEM_PROT_RWX);
 
           if (PrintMiscellaneous && Verbose) {
             char buf[256];
@@ -2755,12 +2756,12 @@ bool os::protect_memory(char* addr, size_t bytes, ProtType prot,
 
 bool os::guard_memory(char* addr, size_t bytes) {
   DWORD old_status;
-  return VirtualProtect(addr, bytes, PAGE_EXECUTE_READWRITE | PAGE_GUARD, &old_status) != 0;
+  return VirtualProtect(addr, bytes, PAGE_READWRITE | PAGE_GUARD, &old_status) != 0;
 }
 
 bool os::unguard_memory(char* addr, size_t bytes) {
   DWORD old_status;
-  return VirtualProtect(addr, bytes, PAGE_EXECUTE_READWRITE, &old_status) != 0;
+  return VirtualProtect(addr, bytes, PAGE_READWRITE, &old_status) != 0;
 }
 
 void os::realign_memory(char *addr, size_t bytes, size_t alignment_hint) { }

src/os_cpu/linux_x86/vm/os_linux_x86.cpp

@@ -422,10 +422,11 @@ JVM_handle_linux_signal(int sig,
       if (addr != last_addr &&
           (UnguardOnExecutionViolation > 1 || os::address_is_in_vm(addr))) {
 
-        // Unguard and retry
+        // Set memory to RWX and retry
         address page_start =
           (address) align_size_down((intptr_t) addr, (intptr_t) page_size);
-        bool res = os::unguard_memory((char*) page_start, page_size);
+        bool res = os::protect_memory((char*) page_start, page_size,
+                                      os::MEM_PROT_RWX);
 
         if (PrintMiscellaneous && Verbose) {
           char buf[256];

src/os_cpu/solaris_x86/vm/os_solaris_x86.cpp

@@ -576,10 +576,11 @@ int JVM_handle_solaris_signal(int sig, siginfo_t* info, void* ucVoid, int abort_
       if (addr != last_addr &&
           (UnguardOnExecutionViolation > 1 || os::address_is_in_vm(addr))) {
 
-        // Unguard and retry
+        // Make memory rwx and retry
         address page_start =
           (address) align_size_down((intptr_t) addr, (intptr_t) page_size);
-        bool res = os::unguard_memory((char*) page_start, page_size);
+        bool res = os::protect_memory((char*) page_start, page_size,
+                                      os::MEM_PROT_RWX);
 
         if (PrintMiscellaneous && Verbose) {
           char buf[256];

src/share/vm/prims/jni.cpp

@@ -2173,7 +2173,8 @@ static char* get_bad_address() {
     size_t size = os::vm_allocation_granularity();
     bad_address = os::reserve_memory(size);
     if (bad_address != NULL) {
-      os::protect_memory(bad_address, size, os::MEM_PROT_READ);
+      os::protect_memory(bad_address, size, os::MEM_PROT_READ,
+                         /*is_committed*/false);
     }
   }
   return bad_address;

src/share/vm/runtime/os.cpp

@@ -932,8 +932,9 @@ void os::serialize_thread_states() {
   // the mutator thread if such case is encountered. See bug 6546278 for details.
   Thread::muxAcquire(&SerializePageLock, "serialize_thread_states");
   os::protect_memory((char *)os::get_memory_serialize_page(),
-                     os::vm_page_size(), MEM_PROT_READ, /*is_committed*/true );
-  os::unguard_memory((char *)os::get_memory_serialize_page(), os::vm_page_size());
+                     os::vm_page_size(), MEM_PROT_READ);
+  os::protect_memory((char *)os::get_memory_serialize_page(),
+                     os::vm_page_size(), MEM_PROT_RW);
   Thread::muxRelease(&SerializePageLock);
 }
 

src/share/vm/runtime/os.hpp

@@ -208,7 +208,7 @@ class os: AllStatic {
 
   enum ProtType { MEM_PROT_NONE, MEM_PROT_READ, MEM_PROT_RW, MEM_PROT_RWX };
   static bool   protect_memory(char* addr, size_t bytes, ProtType prot,
-                               bool is_committed = false);
+                               bool is_committed = true);
 
   static bool   guard_memory(char* addr, size_t bytes);
   static bool   unguard_memory(char* addr, size_t bytes);