8043508: JVM core dumps with very long text in tooltip

Reviewed-by: bae, jgodinez

src/share/native/sun/font/DrawGlyphList.c

@@ -52,7 +52,8 @@
 
 GlyphBlitVector* setupBlitVector(JNIEnv *env, jobject glyphlist) {
 
-    int g, bytesNeeded;
+    int g;
+    size_t bytesNeeded;
     jlong *imagePtrs;
     jfloat* positions = NULL;
     GlyphInfo *ginfo;
@@ -71,6 +72,9 @@ GlyphBlitVector* setupBlitVector(JNIEnv *env, jobject glyphlist) {
 
     bytesNeeded = sizeof(GlyphBlitVector)+sizeof(ImageRef)*len;
     gbv = (GlyphBlitVector*)malloc(bytesNeeded);
+    if (gbv == NULL) {
+        return NULL;
+    }
     gbv->numGlyphs = len;
     gbv->glyphs = (ImageRef*)((unsigned char*)gbv+sizeof(GlyphBlitVector));
 
@@ -479,7 +483,8 @@ Java_sun_java2d_loops_DrawGlyphListLCD_DrawGlyphListLCD
  */
 GlyphBlitVector* setupLCDBlitVector(JNIEnv *env, jobject glyphlist) {
 
-    int g, bytesNeeded;
+    int g;
+    size_t bytesNeeded;
     jlong *imagePtrs;
     jfloat* positions = NULL;
     GlyphInfo *ginfo;
@@ -500,6 +505,9 @@ GlyphBlitVector* setupLCDBlitVector(JNIEnv *env, jobject glyphlist) {
 
     bytesNeeded = sizeof(GlyphBlitVector)+sizeof(ImageRef)*len;
     gbv = (GlyphBlitVector*)malloc(bytesNeeded);
+    if (gbv == NULL) {
+        return NULL;
+    }
     gbv->numGlyphs = len;
     gbv->glyphs = (ImageRef*)((unsigned char*)gbv+sizeof(GlyphBlitVector));
 

test/java/awt/Graphics2D/DrawString/DrawStringCrash.java

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8043508
+ * @summary Drawing a very long string crashes VM
+ */
+
+import java.awt.*;
+import java.awt.image.*;
+
+public class DrawStringCrash {
+
+    public static void main(String[] args) {
+        StringBuffer sb = new StringBuffer();
+        String s = "abcdefghijklmnopqrstuzwxyz";
+        for (int x = 0; x < 100000 ; x++) {
+           sb.append(s);
+        }
+        // Now have a string which uses approx 5Mb memory
+        // Loop again drawing doubling each time until
+        // we reach 8 billion chars or get OOME which means we can't
+        // go any further.
+        // Often there is no crash because Java OOM happens
+        // long before native heap runs out.
+        long maxLen = 8L * 1024 * 1024 * 1024;
+        int len = sb.length();
+
+        BufferedImage bi =
+            new BufferedImage(100, 100, BufferedImage.TYPE_INT_RGB);
+        Graphics2D g2d = bi.createGraphics();
+        while (len < maxLen) {
+            try {
+                g2d.drawString(s, 20, 20);
+            } catch (OutOfMemoryError e) {
+                return;
+            }
+            sb.append(sb);
+            len *= 2;
+        }
+        return;
+    }
+}