7030610: runtime/6878713/Test6878713.sh fails Error. failed to clean up files after test

7123945: runtime/6878713/Test6878713.sh require about 2G of native memory, swaps and times out
Summary: Add new diagnostic option -XX:MallocMaxTestWords=NNN and fix Test6878713.sh.
Reviewed-by: dcubed, coleenp, dholmes, iklam

src/share/vm/runtime/globals.hpp

@@ -2905,6 +2905,10 @@ class CommandLineFlags {
           "if non-zero, start verifying C heap after Nth call to "          \
           "malloc/realloc/free")                                            \
                                                                             \
+  diagnostic(uintx, MallocMaxTestWords,     0,                              \
+          "if non-zero, max # of Words that malloc/realloc can allocate "   \
+          "(for testing only)")                                             \
+                                                                            \
   product(intx, TypeProfileWidth,     2,                                    \
           "number of receiver types to record in call/cast profile")        \
                                                                             \

src/share/vm/runtime/os.cpp

 /*
- * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -80,6 +80,8 @@ julong os::num_frees = 0;           // # of calls to free
 julong os::free_bytes = 0;          // # of bytes freed
 #endif
 
+static juint cur_malloc_words = 0;  // current size for MallocMaxTestWords
+
 void os_init_globals() {
   // Called from init_globals().
   // See Threads::create_vm() in thread.cpp, and init.cpp.
@@ -570,6 +572,26 @@ void verify_block(void* memblock) {
 }
 #endif
 
+//
+// This function supports testing of the malloc out of memory
+// condition without really running the system out of memory.
+//
+static u_char* testMalloc(size_t alloc_size) {
+
+  if (MallocMaxTestWords > 0 &&
+      (cur_malloc_words + (alloc_size / BytesPerWord)) > MallocMaxTestWords) {
+    return NULL;
+  }
+
+  u_char* ptr = (u_char*)::malloc(alloc_size);
+
+  if (MallocMaxTestWords > 0 && (ptr != NULL)) {
+    Atomic::add(((jint) (alloc_size / BytesPerWord)),
+                (volatile jint *) &cur_malloc_words);
+  }
+  return ptr;
+}
+
 void* os::malloc(size_t size, MEMFLAGS memflags, address caller) {
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
@@ -579,11 +601,22 @@ void* os::malloc(size_t size, MEMFLAGS memflags, address caller) {
     // if NULL is returned the calling functions assume out of memory.
     size = 1;
   }
-  if (size > size + space_before + space_after) { // Check for rollover.
+
+  const size_t alloc_size = size + space_before + space_after;
+
+  if (size > alloc_size) { // Check for rollover.
     return NULL;
   }
+
   NOT_PRODUCT(if (MallocVerifyInterval > 0) check_heap());
-  u_char* ptr = (u_char*)::malloc(size + space_before + space_after);
+
+  u_char* ptr;
+
+  if (MallocMaxTestWords > 0) {
+    ptr = testMalloc(alloc_size);
+  } else {
+    ptr = (u_char*)::malloc(alloc_size);
+  }
 
 #ifdef ASSERT
   if (ptr == NULL) return NULL;

test/runtime/6878713/Test6878713.sh

 #!/bin/sh
 
+# 
+#  Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
+#  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+# 
+#  This code is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License version 2 only, as
+#  published by the Free Software Foundation.
+# 
+#  This code is distributed in the hope that it will be useful, but WITHOUT
+#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#  version 2 for more details (a copy is included in the LICENSE file that
+#  accompanied this code).
+# 
+#  You should have received a copy of the GNU General Public License version
+#  2 along with this work; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+# 
+#  Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+#  or visit www.oracle.com if you need additional information or have any
+#  questions.
+# 
+
+ 
+
 ##
 ## @test
 ## @bug 6878713
+## @bug 7030610
+## @bug 7037122
+## @bug 7123945
 ## @summary Verifier heap corruption, relating to backward jsrs
-## @run shell/timeout=120 Test6878713.sh
+## @run shell Test6878713.sh
 ##
 
 if [ "${TESTSRC}" = "" ]
@@ -49,23 +77,98 @@ case "$OS" in
     ;;
 esac
 
-JEMMYPATH=${CPAPPEND}
-CLASSPATH=.${PS}${TESTCLASSES}${PS}${JEMMYPATH} ; export CLASSPATH
-
-THIS_DIR=`pwd`
+CLASSPATH=.${PS}${TESTCLASSES} ; export CLASSPATH
 
 ${TESTJAVA}${FS}bin${FS}java ${TESTVMOPTS} -version
 
-${TESTJAVA}${FS}bin${FS}jar xvf ${TESTSRC}${FS}testcase.jar
+TARGET_CLASS=OOMCrashClass1960_2
 
-${TESTJAVA}${FS}bin${FS}java ${TESTVMOPTS} OOMCrashClass1960_2 > test.out 2>&1
+echo "INFO: extracting the target class."
+${TESTJAVA}${FS}bin${FS}jar xvf \
+    ${TESTSRC}${FS}testcase.jar ${TARGET_CLASS}.class
 
-if [ -s core -o -s "hs_*.log" ]
-then
-    cat hs*.log
-    echo "Test Failed"
-    exit 1
+# remove any hs_err_pid that might exist here
+rm -f hs_err_pid*.log
+
+echo "INFO: checking for 32-bit versus 64-bit VM."
+${TESTJAVA}${FS}bin${FS}java ${TESTVMOPTS} -version 2>&1 \
+    | grep "64-Bit [^ ][^ ]* VM" > /dev/null 2>&1
+status="$?"
+if [ "$status" = 0 ]; then
+    echo "INFO: testing a 64-bit VM."
+    is_64_bit=true
 else
-    echo "Test Passed"
-    exit 0
+    echo "INFO: testing a 32-bit VM."
+fi
+
+if [ "$is_64_bit" = true ]; then
+    # limit is 768MB in 8-byte words (1024 * 1024 * 768 / 8) == 100663296
+    MALLOC_MAX=100663296
+else
+    # limit is 768MB in 4-byte words (1024 * 1024 * 768 / 4) == 201326592
+    MALLOC_MAX=201326592
+fi
+echo "INFO: MALLOC_MAX=$MALLOC_MAX"
+
+echo "INFO: executing the target class."
+# -XX:+PrintCommandLineFlags for debugging purposes
+# -XX:+IgnoreUnrecognizedVMOptions so test will run on a VM without
+#     the new -XX:MallocMaxTestWords option
+# -XX:+UnlockDiagnosticVMOptions so we can use -XX:MallocMaxTestWords
+# -XX:MallocMaxTestWords limits malloc to $MALLOC_MAX
+${TESTJAVA}${FS}bin${FS}java \
+    -XX:+PrintCommandLineFlags \
+    -XX:+IgnoreUnrecognizedVMOptions \
+    -XX:+UnlockDiagnosticVMOptions \
+    -XX:MallocMaxTestWords=$MALLOC_MAX \
+    ${TESTVMOPTS} ${TARGET_CLASS} > test.out 2>&1
+
+echo "INFO: begin contents of test.out:"
+cat test.out
+echo "INFO: end contents of test.out."
+
+echo "INFO: checking for memory allocation error message."
+# We are looking for this specific memory allocation failure mesg so
+# we know we exercised the right allocation path with the test class:
+MESG1="Native memory allocation (malloc) failed to allocate 25696531[0-9][0-9] bytes"
+grep "$MESG1" test.out
+status="$?"
+if [ "$status" = 0 ]; then
+    echo "INFO: found expected memory allocation error message."
+else
+    echo "INFO: did not find expected memory allocation error message."
+
+    # If we didn't find MESG1 above, then there are several scenarios:
+    # 1) -XX:MallocMaxTestWords is not supported by the current VM and we
+    #    didn't fail TARGET_CLASS's memory allocation attempt; instead
+    #    we failed to find TARGET_CLASS's main() method. The TARGET_CLASS
+    #    is designed to provoke a memory allocation failure during class
+    #    loading; we actually don't care about running the class which is
+    #    why it doesn't have a main() method.
+    # 2) we failed a memory allocation, but not the one we were looking
+    #    so it might be that TARGET_CLASS no longer tickles the same
+    #    memory allocation code path
+    # 3) TARGET_CLASS reproduces the failure mode (SIGSEGV) fixed by
+    #    6878713 because the test is running on a pre-fix VM.
+    echo "INFO: checking for no main() method message."
+    MESG2="Error: Main method not found in class"
+    grep "$MESG2" test.out
+    status="$?"
+    if [ "$status" = 0 ]; then
+        echo "INFO: found no main() method message."
+    else
+        echo "FAIL: did not find no main() method message."
+        # status is non-zero for exit below
+
+        if [ -s hs_err_pid*.log ]; then
+            echo "INFO: begin contents of hs_err_pid file:"
+            cat hs_err_pid*.log
+            echo "INFO: end contents of hs_err_pid file."
+        fi
+    fi
+fi
+
+if [ "$status" = 0 ]; then
+    echo "PASS: test found one of the expected messages."
 fi
+exit "$status"