8011773: Some tests on Interned String crashed JVM with OOM

Summary: Instead of terminating the VM, throw OutOfMemoryError exceptions.
Reviewed-by: coleenp, dholmes

src/share/vm/classfile/javaClasses.cpp

@@ -315,14 +315,18 @@ Handle java_lang_String::char_converter(Handle java_string, jchar from_char, jch
   return string;
 }
 
-jchar* java_lang_String::as_unicode_string(oop java_string, int& length) {
+jchar* java_lang_String::as_unicode_string(oop java_string, int& length, TRAPS) {
   typeArrayOop value  = java_lang_String::value(java_string);
   int          offset = java_lang_String::offset(java_string);
                length = java_lang_String::length(java_string);
 
-  jchar* result = NEW_RESOURCE_ARRAY(jchar, length);
-  for (int index = 0; index < length; index++) {
-    result[index] = value->char_at(index + offset);
+  jchar* result = NEW_RESOURCE_ARRAY_RETURN_NULL(jchar, length);
+  if (result != NULL) {
+    for (int index = 0; index < length; index++) {
+      result[index] = value->char_at(index + offset);
+    }
+  } else {
+    THROW_MSG_0(vmSymbols::java_lang_OutOfMemoryError(), "could not allocate Unicode string");
   }
   return result;
 }

src/share/vm/classfile/javaClasses.hpp

@@ -153,7 +153,7 @@ class java_lang_String : AllStatic {
   static char*  as_utf8_string(oop java_string, char* buf, int buflen);
   static char*  as_utf8_string(oop java_string, int start, int len);
   static char*  as_platform_dependent_str(Handle java_string, TRAPS);
-  static jchar* as_unicode_string(oop java_string, int& length);
+  static jchar* as_unicode_string(oop java_string, int& length, TRAPS);
   // produce an ascii string with all other values quoted using \u####
   static char*  as_quoted_ascii(oop java_string);
 

src/share/vm/classfile/symbolTable.cpp

@@ -735,7 +735,7 @@ oop StringTable::intern(oop string, TRAPS)
   ResourceMark rm(THREAD);
   int length;
   Handle h_string (THREAD, string);
-  jchar* chars = java_lang_String::as_unicode_string(string, length);
+  jchar* chars = java_lang_String::as_unicode_string(string, length, CHECK_NULL);
   oop result = intern(h_string, chars, length, CHECK_NULL);
   return result;
 }

src/share/vm/memory/allocation.hpp

@@ -539,6 +539,9 @@ class ResourceObj ALLOCATION_SUPER_CLASS_SPEC {
 #define NEW_RESOURCE_ARRAY(type, size)\
   (type*) resource_allocate_bytes((size) * sizeof(type))
 
+#define NEW_RESOURCE_ARRAY_RETURN_NULL(type, size)\
+  (type*) resource_allocate_bytes((size) * sizeof(type), AllocFailStrategy::RETURN_NULL)
+
 #define NEW_RESOURCE_ARRAY_IN_THREAD(thread, type, size)\
   (type*) resource_allocate_bytes(thread, (size) * sizeof(type))
 

src/share/vm/oops/oop.cpp

 /*
- * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -103,11 +103,17 @@ intptr_t oopDesc::slow_identity_hash() {
 
 // When String table needs to rehash
 unsigned int oopDesc::new_hash(jint seed) {
+  EXCEPTION_MARK;
   ResourceMark rm;
   int length;
-  jchar* chars = java_lang_String::as_unicode_string(this, length);
-  // Use alternate hashing algorithm on the string
-  return AltHashing::murmur3_32(seed, chars, length);
+  jchar* chars = java_lang_String::as_unicode_string(this, length, THREAD);
+  if (chars != NULL) {
+    // Use alternate hashing algorithm on the string
+    return AltHashing::murmur3_32(seed, chars, length);
+  } else {
+    vm_exit_out_of_memory(length, "unable to create Unicode strings for String table rehash");
+    return 0;
+  }
 }
 
 VerifyOopClosure VerifyOopClosure::verify_oop;

src/share/vm/prims/whitebox.cpp

 /*
- * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -310,12 +310,8 @@ WB_END
 WB_ENTRY(jboolean, WB_IsInStringTable(JNIEnv* env, jobject o, jstring javaString))
   ResourceMark rm(THREAD);
   int len;
-  jchar* name = java_lang_String::as_unicode_string(JNIHandles::resolve(javaString), len);
-  oop found_string = StringTable::the_table()->lookup(name, len);
-  if (found_string == NULL) {
-        return false;
-  }
-  return true;
+  jchar* name = java_lang_String::as_unicode_string(JNIHandles::resolve(javaString), len, CHECK_false);
+  return (StringTable::lookup(name, len) != NULL);
 WB_END