7102044: G1: VM crashes with assert(old_end != new_end) failed: don't call this otherwise

Summary: arrayOopDesc::max_array_length() should return a value that does not overflow a size_t if it is converted to bytes.
Reviewed-by: kvn, dholmes

make/jprt.properties

@@ -541,9 +541,20 @@ jprt.make.rule.test.targets.standard.server = \
   ${jprt.my.windows.i586}-*-c2-servertest, \
   ${jprt.my.windows.x64}-*-c2-servertest
 
+jprt.make.rule.test.targets.standard.internalvmtests = \
+  ${jprt.my.solaris.sparc}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.solaris.sparcv9}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.solaris.i586}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.solaris.x64}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.linux.i586}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.linux.x64}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.windows.i586}-fastdebug-c2-internalvmtests, \
+  ${jprt.my.windows.x64}-fastdebug-c2-internalvmtests
+  
 jprt.make.rule.test.targets.standard = \
   ${jprt.make.rule.test.targets.standard.client}, \
-  ${jprt.make.rule.test.targets.standard.server}
+  ${jprt.make.rule.test.targets.standard.server}, \
+  ${jprt.make.rule.test.targets.standard.internalvmtests}
 
 jprt.make.rule.test.targets.embedded = \
   ${jprt.make.rule.test.targets.standard.client}

src/share/vm/oops/arrayOop.cpp

@@ -23,9 +23,40 @@
  */
 
 #include "precompiled.hpp"
+
+/////////////// Unit tests ///////////////
+
+#ifndef PRODUCT
+
 #include "oops/arrayOop.hpp"
-#include "oops/objArrayOop.hpp"
-#include "oops/oop.inline.hpp"
-#include "oops/symbol.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+bool arrayOopDesc::check_max_length_overflow(BasicType type) {
+  julong length = max_array_length(type);
+  julong bytes_per_element = type2aelembytes(type);
+  julong bytes = length * bytes_per_element + header_size_in_bytes();
+  return (julong)(size_t)bytes == bytes;
+}
+
+bool arrayOopDesc::test_max_array_length() {
+  tty->print_cr("test_max_array_length");
+
+  assert(check_max_length_overflow(T_BOOLEAN), "size_t overflow for boolean array");
+  assert(check_max_length_overflow(T_CHAR), "size_t overflow for char array");
+  assert(check_max_length_overflow(T_FLOAT), "size_t overflow for float array");
+  assert(check_max_length_overflow(T_DOUBLE), "size_t overflow for double array");
+  assert(check_max_length_overflow(T_BYTE), "size_t overflow for byte array");
+  assert(check_max_length_overflow(T_SHORT), "size_t overflow for short array");
+  assert(check_max_length_overflow(T_INT), "size_t overflow for int array");
+  assert(check_max_length_overflow(T_LONG), "size_t overflow for long array");
+  assert(check_max_length_overflow(T_OBJECT), "size_t overflow for object array");
+  assert(check_max_length_overflow(T_ARRAY), "size_t overflow for array array");
+  assert(check_max_length_overflow(T_NARROWOOP), "size_t overflow for narrowOop array");
+
+  // T_VOID and T_ADDRESS are not supported by max_array_length()
+
+  return true;
+}
+
 
-// <<this page is intentionally left blank>>
+#endif //PRODUCT

src/share/vm/oops/arrayOop.hpp

 /*
- * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -104,20 +104,26 @@ class arrayOopDesc : public oopDesc {
 
   // Return the maximum length of an array of BasicType.  The length can passed
   // to typeArrayOop::object_size(scale, length, header_size) without causing an
-  // overflow.
+  // overflow. We also need to make sure that this will not overflow a size_t on
+  // 32 bit platforms when we convert it to a byte size.
   static int32_t max_array_length(BasicType type) {
     assert(type >= 0 && type < T_CONFLICT, "wrong type");
     assert(type2aelembytes(type) != 0, "wrong type");
-    const int bytes_per_element = type2aelembytes(type);
-    if (bytes_per_element < HeapWordSize) {
+
+    const size_t max_element_words_per_size_t  = align_size_down((SIZE_MAX/HeapWordSize - header_size(type)), MinObjAlignment);
+    const size_t max_elements_per_size_t = HeapWordSize * max_element_words_per_size_t  / type2aelembytes(type);
+    if ((size_t)max_jint < max_elements_per_size_t) {
       return max_jint;
     }
-
-    const int32_t max_words = align_size_down(max_jint, MinObjAlignment);
-    const int32_t max_element_words = max_words - header_size(type);
-    const int32_t words_per_element = bytes_per_element >> LogHeapWordSize;
-    return max_element_words / words_per_element;
+    return (int32_t)max_elements_per_size_t;
   }
+
+// for unit testing
+#ifndef PRODUCT
+  static bool check_max_length_overflow(BasicType type);
+  static int32_t old_max_array_length(BasicType type);
+  static bool test_max_array_length();
+#endif
 };
 
 #endif // SHARE_VM_OOPS_ARRAYOOP_HPP

src/share/vm/prims/jni.cpp

@@ -5042,7 +5042,8 @@ _JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_GetDefaultJavaVMInitArgs(void *args_) {
 void execute_internal_vm_tests() {
   if (ExecuteInternalVMTests) {
     assert(QuickSort::test_quick_sort(), "test_quick_sort failed");
-    tty->print_cr("All tests passed");
+    assert(arrayOopDesc::test_max_array_length(), "test_max_array_length failed");
+    tty->print_cr("All internal VM tests passed");
   }
 }
 

src/share/vm/utilities/quickSort.cpp

@@ -23,13 +23,13 @@
  */
 
 #include "precompiled.hpp"
-#include "utilities/quickSort.hpp"
 
-#ifndef PRODUCT
+/////////////// Unit tests ///////////////
 
-// Unit tests
+#ifndef PRODUCT
 
 #include "runtime/os.hpp"
+#include "utilities/quickSort.hpp"
 #include <stdlib.h>
 
 static int test_comparator(int a, int b) {
@@ -94,7 +94,7 @@ bool QuickSort::sort_and_compare(int* arrayToSort, int* expectedResult, int leng
 }
 
 bool QuickSort::test_quick_sort() {
-  tty->print_cr("test_quick_sort\n");
+  tty->print_cr("test_quick_sort");
   {
     int* test_array = NULL;
     int* expected_array = NULL;

test/Makefile

 #
-# Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 1995, 2011, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -219,6 +219,15 @@ PHONY_LIST += servertest
 
 ################################################################
 
+# internalvmtests (run internal unit tests inside the VM)
+
+internalvmtests: prep $(PRODUCT_HOME)
+	$(PRODUCT_HOME)/bin/java $(JAVA_OPTIONS) -XX:+ExecuteInternalVMTests -version
+
+PHONY_LIST += internalvmtests
+
+################################################################
+
 # packtest
 
 # Expect JPRT to set JPRT_PACKTEST_HOME.