8007146: [macosx] Setting a display mode crashes JDK under VNC

Reviewed-by: serb

src/macosx/native/sun/awt/CGraphicsDevice.m

@@ -49,6 +49,42 @@ static int getBPPFromModeString(CFStringRef mode)
     return 0;
 }
 
+static BOOL isValidDisplayMode(CGDisplayModeRef mode){
+    return (1 < CGDisplayModeGetWidth(mode) && 1 < CGDisplayModeGetHeight(mode));
+}
+
+static CFMutableArrayRef getAllValidDisplayModes(jint displayID){
+    CFArrayRef allModes = CGDisplayCopyAllDisplayModes(displayID, NULL);
+
+    CFIndex numModes = CFArrayGetCount(allModes);
+    CFMutableArrayRef validModes = CFArrayCreateMutable(kCFAllocatorDefault, numModes + 1, NULL);
+
+    CFIndex n;
+    for (n=0; n < numModes; n++) {
+        CGDisplayModeRef cRef = (CGDisplayModeRef) CFArrayGetValueAtIndex(allModes, n);
+        if (cRef != NULL && isValidDisplayMode(cRef)) {
+            CFArrayAppendValue(validModes, cRef);
+        }
+    }
+
+    CGDisplayModeRef currentMode = CGDisplayCopyDisplayMode(displayID);
+
+    BOOL containsCurrentMode = NO;
+    numModes = CFArrayGetCount(validModes);
+    for (n=0; n < numModes; n++) {
+        if(CFArrayGetValueAtIndex(validModes, n) == currentMode){
+            containsCurrentMode = YES;
+            break;
+        }
+    }
+
+    if (!containsCurrentMode) {
+        CFArrayAppendValue(validModes, currentMode);
+    }
+
+    return validModes;
+}
+
 /*
  * Find the best possible match in the list of display modes that we can switch to based on
  * the provided parameters.
@@ -198,28 +234,30 @@ Java_sun_awt_CGraphicsDevice_nativeSetDisplayMode
 (JNIEnv *env, jclass class, jint displayID, jint w, jint h, jint bpp, jint refrate)
 {
     JNF_COCOA_ENTER(env);
-    CFArrayRef allModes = CGDisplayCopyAllDisplayModes(displayID, NULL);
+    CFArrayRef allModes = getAllValidDisplayModes(displayID);
+
     CGDisplayModeRef closestMatch = getBestModeForParameters(allModes, (int)w, (int)h, (int)bpp, (int)refrate);
+    __block CGError retCode = kCGErrorSuccess;
     if (closestMatch != NULL) {
         [JNFRunLoop performOnMainThreadWaiting:YES withBlock:^(){
             CGDisplayConfigRef config;
-            CGError retCode = CGBeginDisplayConfiguration(&config);
+            retCode = CGBeginDisplayConfiguration(&config);
             if (retCode == kCGErrorSuccess) {
                 CGConfigureDisplayWithDisplayMode(config, displayID, closestMatch, NULL);
-                CGCompleteDisplayConfiguration(config, kCGConfigureForAppOnly);
-                if (config != NULL) {
-                    CFRelease(config);
-                }
+                retCode = CGCompleteDisplayConfiguration(config, kCGConfigureForAppOnly);
             }
         }];
     } else {
         [JNFException raise:env as:kIllegalArgumentException reason:"Invalid display mode"];
     }
 
+    if (retCode != kCGErrorSuccess){
+        [JNFException raise:env as:kIllegalArgumentException reason:"Unable to set display mode!"];
+    }    
+
     CFRelease(allModes);
     JNF_COCOA_EXIT(env);
 }
-
 /*
  * Class:     sun_awt_CGraphicsDevice
  * Method:    nativeGetDisplayMode
@@ -247,7 +285,8 @@ Java_sun_awt_CGraphicsDevice_nativeGetDisplayModes
 {
     jobjectArray jreturnArray = NULL;
     JNF_COCOA_ENTER(env);
-    CFArrayRef allModes = CGDisplayCopyAllDisplayModes(displayID, NULL);
+    CFArrayRef allModes = getAllValidDisplayModes(displayID);
+
     CFIndex numModes = CFArrayGetCount(allModes);
     static JNF_CLASS_CACHE(jc_DisplayMode, "java/awt/DisplayMode");
 

test/java/awt/GraphicsDevice/CheckDisplayModes.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8007146
+ * @summary [macosx] Setting a display mode crashes JDK under VNC
+ * @author Alexander Scherbatiy
+ * @run main CheckDisplayModes
+ */
+import java.awt.DisplayMode;
+import java.awt.GraphicsDevice;
+import java.awt.GraphicsEnvironment;
+
+public class CheckDisplayModes {
+
+    public static void main(String[] args) {
+        GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment();
+        GraphicsDevice graphicDevice = ge.getDefaultScreenDevice();
+        DisplayMode defaultDisplayMode = graphicDevice.getDisplayMode();
+        checkDisplayMode(defaultDisplayMode);
+        graphicDevice.setDisplayMode(defaultDisplayMode);
+
+        DisplayMode[] displayModes = graphicDevice.getDisplayModes();
+        boolean isDefaultDisplayModeIncluded = false;
+        for (DisplayMode displayMode : displayModes) {
+            checkDisplayMode(displayMode);
+            graphicDevice.setDisplayMode(displayMode);
+            if (defaultDisplayMode.equals(displayMode)) {
+                isDefaultDisplayModeIncluded = true;
+            }
+        }
+
+        if (!isDefaultDisplayModeIncluded) {
+            throw new RuntimeException("Default display mode is not included");
+        }
+    }
+
+    static void checkDisplayMode(DisplayMode displayMode) {
+        if (displayMode == null || displayMode.getWidth() <= 1 || displayMode.getHeight() <= 1) {
+            throw new RuntimeException("invalid display mode");
+        }
+    }
+}