7160609: [macosx] JDK crash in libjvm.dylib ( C [GeForceGLDriver+0x675a] gldAttachDrawable+0x941)

Summary: Constrain window dimensions with screen bounds and GL_MAX_TEXTURE_SIZE
Reviewed-by: art, serb

src/macosx/classes/sun/java2d/opengl/CGLGraphicsConfig.java

@@ -31,8 +31,12 @@ import java.awt.Color;
 import java.awt.Component;
 import java.awt.Graphics;
 import java.awt.Graphics2D;
+import java.awt.GraphicsConfiguration;
+import java.awt.GraphicsDevice;
+import java.awt.GraphicsEnvironment;
 import java.awt.Image;
 import java.awt.ImageCapabilities;
+import java.awt.Rectangle;
 import java.awt.Transparency;
 import java.awt.color.ColorSpace;
 import java.awt.image.BufferedImage;
@@ -44,6 +48,7 @@ import java.awt.image.WritableRaster;
 
 import sun.awt.CGraphicsConfig;
 import sun.awt.CGraphicsDevice;
+import sun.awt.TextureSizeConstraining;
 import sun.awt.image.OffScreenImage;
 import sun.awt.image.SunVolatileImage;
 import sun.awt.image.SurfaceManager;
@@ -65,7 +70,7 @@ import sun.java2d.pipe.hw.AccelDeviceEventNotifier;
 import sun.lwawt.macosx.CPlatformView;
 
 public class CGLGraphicsConfig extends CGraphicsConfig
-    implements OGLGraphicsConfig
+    implements OGLGraphicsConfig, TextureSizeConstraining
 {
     //private static final int kOpenGLSwapInterval = RuntimeOptions.getCurrentOptions().OpenGLSwapInterval;
     private static final int kOpenGLSwapInterval = 0; // TODO
@@ -242,6 +247,8 @@ public class CGLGraphicsConfig extends CGraphicsConfig
         } finally {
             rq.unlock();
         }
+
+        updateTotalDisplayBounds();
     }
 
     @Override
@@ -478,4 +485,50 @@ public class CGLGraphicsConfig extends CGraphicsConfig
     public void removeDeviceEventListener(AccelDeviceEventListener l) {
         AccelDeviceEventNotifier.removeListener(l);
     }
+
+    private static final Rectangle totalDisplayBounds = new Rectangle();
+
+    private static void updateTotalDisplayBounds() {
+        synchronized (totalDisplayBounds) {
+            Rectangle virtualBounds = new Rectangle();
+            for (GraphicsDevice gd : GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices()) {
+                for (GraphicsConfiguration gc : gd.getConfigurations()) {
+                    virtualBounds = virtualBounds.union(gc.getBounds());
+                }
+            }
+            totalDisplayBounds.setBounds(virtualBounds);
+        }
+    }
+
+    // 7160609: GL still fails to create a square texture of this size,
+    //          so we use this value to cap the total display bounds.
+    native private static int getMaxTextureSize();
+
+    @Override
+    public int getMaxTextureWidth() {
+        int width;
+
+        synchronized (totalDisplayBounds) {
+            if (totalDisplayBounds.width == 0) {
+                updateTotalDisplayBounds();
+            }
+            width = totalDisplayBounds.width;
+        }
+
+        return Math.min(width, getMaxTextureSize());
+    }
+
+    @Override
+    public int getMaxTextureHeight() {
+        int height;
+
+        synchronized (totalDisplayBounds) {
+            if (totalDisplayBounds.height == 0) {
+                updateTotalDisplayBounds();
+            }
+            height = totalDisplayBounds.height;
+        }
+
+        return Math.min(height, getMaxTextureSize());
+    }
 }

src/macosx/classes/sun/lwawt/LWWindowPeer.java

@@ -338,6 +338,18 @@ public class LWWindowPeer
             h = MINIMUM_HEIGHT;
         }
 
+        if (graphicsConfig instanceof TextureSizeConstraining) {
+            final int maxW = ((TextureSizeConstraining)graphicsConfig).getMaxTextureWidth();
+            final int maxH = ((TextureSizeConstraining)graphicsConfig).getMaxTextureHeight();
+
+            if (w > maxW) {
+                w = maxW;
+            }
+            if (h > maxH) {
+                h = maxH;
+            }
+        }
+
         // Don't post ComponentMoved/Resized and Paint events
         // until we've got a notification from the delegate
         setBounds(x, y, w, h, op, false, false);
@@ -405,14 +417,33 @@ public class LWWindowPeer
 
     @Override
     public void updateMinimumSize() {
-        Dimension d = null;
+        final Dimension min;
         if (getTarget().isMinimumSizeSet()) {
-            d = getTarget().getMinimumSize();
+            min = getTarget().getMinimumSize();
+            min.width = Math.max(min.width, MINIMUM_WIDTH);
+            min.height = Math.max(min.height, MINIMUM_HEIGHT);
+        } else {
+            min = new Dimension(MINIMUM_WIDTH, MINIMUM_HEIGHT);
         }
-        if (d == null) {
-            d = new Dimension(MINIMUM_WIDTH, MINIMUM_HEIGHT);
+
+        final int maxW, maxH;
+        if (graphicsConfig instanceof TextureSizeConstraining) {
+            maxW = ((TextureSizeConstraining)graphicsConfig).getMaxTextureWidth();
+            maxH = ((TextureSizeConstraining)graphicsConfig).getMaxTextureHeight();
+        } else {
+            maxW = maxH = Integer.MAX_VALUE;
+        }
+
+        final Dimension max;
+        if (getTarget().isMaximumSizeSet()) {
+            max = getTarget().getMaximumSize();
+            max.width = Math.min(max.width, maxW);
+            max.height = Math.min(max.height, maxH);
+        } else {
+            max = new Dimension(maxW, maxH);
         }
-        platformWindow.setMinimumSize(d.width, d.height);
+
+        platformWindow.setSizeConstraints(min.width, min.height, max.width, max.height);
     }
 
     @Override

src/macosx/classes/sun/lwawt/PlatformWindow.java

@@ -131,7 +131,10 @@ public interface PlatformWindow {
 
     public void setResizable(boolean resizable);
 
-    public void setMinimumSize(int width, int height);
+    /**
+     * Applies the minimum and maximum size to the platform window.
+     */
+    public void setSizeConstraints(int minW, int minH, int maxW, int maxH);
 
     /**
      * Transforms the given Graphics object according to the native

src/macosx/classes/sun/lwawt/macosx/CPlatformEmbeddedFrame.java

@@ -180,7 +180,7 @@ public class CPlatformEmbeddedFrame implements PlatformWindow {
     public void setResizable(boolean resizable) {}
 
     @Override
-    public void setMinimumSize(int width, int height) {}
+    public void setSizeConstraints(int minW, int minH, int maxW, int maxH) {}
 
     @Override
     public Graphics transformGraphics(Graphics g) {

src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -672,20 +672,15 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
 
         // Re-apply the size constraints and the size to ensure the space
         // occupied by the grow box is counted properly
-        setMinimumSize(1, 1); // the method ignores its arguments
+        peer.updateMinimumSize();
 
         Rectangle bounds = peer.getBounds();
         setBounds(bounds.x, bounds.y, bounds.width, bounds.height);
     }
 
     @Override
-    public void setMinimumSize(int width, int height) {
-        //TODO width, height should be used
-        //NOTE: setResizable() calls setMinimumSize(1,1) relaying on the logic below
-        final long nsWindowPtr = getNSWindowPtr();
-        final Dimension min = target.getMinimumSize();
-        final Dimension max = target.getMaximumSize();
-        nativeSetNSWindowMinMax(nsWindowPtr, min.getWidth(), min.getHeight(), max.getWidth(), max.getHeight());
+    public void setSizeConstraints(int minW, int minH, int maxW, int maxH) {
+        nativeSetNSWindowMinMax(getNSWindowPtr(), minW, minH, maxW, maxH);
     }
 
     @Override

src/macosx/native/sun/java2d/opengl/CGLGraphicsConfig.m

@@ -447,3 +447,20 @@ Java_sun_java2d_opengl_CGLGraphicsConfig_getOGLCapabilities
         return cglinfo->context->caps;
     }
 }
+
+JNIEXPORT jint JNICALL
+Java_sun_java2d_opengl_CGLGraphicsConfig_getMaxTextureSize
+    (JNIEnv *env, jclass cglgc)
+{
+    J2dTraceLn(J2D_TRACE_INFO, "CGLGraphicsConfig_getMaxTextureSize");
+
+    __block int max = 0;
+
+    [JNFRunLoop performOnMainThreadWaiting:YES withBlock:^(){
+        [sharedContext makeCurrentContext];
+        j2d_glGetIntegerv(GL_MAX_TEXTURE_SIZE, &max);
+    }];
+
+    return (jint)max;
+}
+

src/share/classes/sun/awt/TextureSizeConstraining.java

+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.awt;
+
+/**
+ * A GraphicsConfiguration implements the TextureSizeConstraining
+ * interface to indicate that it imposes certain limitations on the
+ * maximum size of supported textures.
+ */
+public interface TextureSizeConstraining {
+
+    /**
+     * Returns the maximum width of any texture image.
+     */
+    public int getMaxTextureWidth();
+
+    /**
+     * Returns the maximum height of any texture image.
+     */
+    public int getMaxTextureHeight();
+
+}

test/java/awt/Frame/HugeFrame/HugeFrame.java

+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+  @test
+  @bug 7160609
+  @summary A window with huge dimensions shouldn't crash JVM
+  @author anthony.petrov@oracle.com: area=awt.toplevel
+  @run main HugeFrame
+*/
+
+import java.awt.*;
+
+public class HugeFrame {
+    public static void main(String[] args) throws Exception {
+        Frame f = new Frame("Huge");
+
+        // 8193+ should already produce a crash, but let's go extreme...
+        f.setBounds(10, 10, 30000, 500000);
+        f.setVisible(true);
+
+        // We would crash by now if the bug wasn't fixed
+        Thread.sleep(1000);
+        System.err.println(f.getBounds());
+
+        // Cleanup
+        f.dispose();
+    }
+}