8012015: Use PROT_NONE when reserving memory

Summary: Reserved memory had PROT_READ+PROT_WRITE access on Linux/bsd, now changed to PROT_NONE.
Reviewed-by: dholmes, ctornqvi

src/os/bsd/vm/os_bsd.cpp

@@ -2080,9 +2080,10 @@ static char* anon_mmap(char* requested_addr, size_t bytes, bool fixed) {
     flags |= MAP_FIXED;
   }
 
-  // Map uncommitted pages PROT_READ and PROT_WRITE, change access
-  // to PROT_EXEC if executable when we commit the page.
-  addr = (char*)::mmap(requested_addr, bytes, PROT_READ|PROT_WRITE,
+  // Map reserved/uncommitted pages PROT_NONE so we fail early if we
+  // touch an uncommitted page. Otherwise, the read/write might
+  // succeed if we have enough swap space to back the physical page.
+  addr = (char*)::mmap(requested_addr, bytes, PROT_NONE,
                        flags, -1, 0);
 
   if (addr != MAP_FAILED) {

src/os/linux/vm/os_linux.cpp

@@ -2906,9 +2906,10 @@ static char* anon_mmap(char* requested_addr, size_t bytes, bool fixed) {
     flags |= MAP_FIXED;
   }
 
-  // Map uncommitted pages PROT_READ and PROT_WRITE, change access
-  // to PROT_EXEC if executable when we commit the page.
-  addr = (char*)::mmap(requested_addr, bytes, PROT_READ|PROT_WRITE,
+  // Map reserved/uncommitted pages PROT_NONE so we fail early if we
+  // touch an uncommitted page. Otherwise, the read/write might
+  // succeed if we have enough swap space to back the physical page.
+  addr = (char*)::mmap(requested_addr, bytes, PROT_NONE,
                        flags, -1, 0);
 
   if (addr != MAP_FAILED) {

src/share/vm/prims/whitebox.cpp

@@ -320,6 +320,11 @@ WB_ENTRY(void, WB_FullGC(JNIEnv* env, jobject o))
   Universe::heap()->collect(GCCause::_last_ditch_collection);
 WB_END
 
+
+WB_ENTRY(jlong, WB_ReserveMemory(JNIEnv* env, jobject o, jlong size))
+  return (jlong)os::reserve_memory(size, NULL, 0);
+WB_END
+
 //Some convenience methods to deal with objects from java
 int WhiteBox::offset_for_field(const char* field_name, oop object,
     Symbol* signature_symbol) {
@@ -421,6 +426,8 @@ static JNINativeMethod methods[] = {
       CC"(Ljava/lang/reflect/Executable;)V",          (void*)&WB_ClearMethodState},
   {CC"isInStringTable",   CC"(Ljava/lang/String;)Z",  (void*)&WB_IsInStringTable  },
   {CC"fullGC",   CC"()V",                             (void*)&WB_FullGC },
+
+  {CC"reserveMemory", CC"(J)J", (void*)&WB_ReserveMemory },
 };
 
 #undef CC

test/runtime/memory/ReserveMemory.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @key regression
+ * @bug 8012015
+ * @summary Make sure reserved (but uncommitted) memory is not accessible
+ * @library /testlibrary /testlibrary/whitebox
+ * @build ReserveMemory
+ * @run main ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main ReserveMemory
+ */
+
+import com.oracle.java.testlibrary.*;
+
+import java.lang.reflect.Field;
+import sun.hotspot.WhiteBox;
+import sun.misc.Unsafe;
+
+public class ReserveMemory {
+  private static Unsafe getUnsafe() throws Exception {
+    Field f = Unsafe.class.getDeclaredField("theUnsafe");
+    f.setAccessible(true);
+    return (Unsafe)f.get(null);
+  }
+
+  private static boolean isWindows() {
+    return System.getProperty("os.name").toLowerCase().startsWith("win");
+  }
+
+  public static void main(String args[]) throws Exception {
+    if (args.length > 0) {
+      long address = WhiteBox.getWhiteBox().reserveMemory(4096);
+
+      System.out.println("Reserved memory at address: 0x" + Long.toHexString(address));
+      System.out.println("Will now read from the address, expecting a crash!");
+
+      int x = getUnsafe().getInt(address);
+
+      throw new Exception("Read of reserved/uncommitted memory unexpectedly succeeded, expected crash!");
+    }
+
+    ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+          "-Xbootclasspath/a:.",
+          "-XX:+UnlockDiagnosticVMOptions",
+          "-XX:+WhiteBoxAPI",
+          "ReserveMemory",
+          "test");
+
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+    if (isWindows()) {
+      output.shouldContain("EXCEPTION_ACCESS_VIOLATION");
+    } else {
+      output.shouldContain("SIGSEGV");
+    }
+  }
+}

test/testlibrary/whitebox/sun/hotspot/WhiteBox.java

@@ -111,6 +111,9 @@ public class WhiteBox {
   // Intered strings
   public native boolean isInStringTable(String str);
 
+  // Memory
+  public native long reserveMemory(long size);
+
   // force Full GC
   public native void fullGC();
 }