8028512: [macosx] Crash in full screen api if incorrect display mode is used

Reviewed-by: anthony, leonidr

src/macosx/classes/sun/awt/CGraphicsDevice.java

@@ -158,12 +158,12 @@ public final class CGraphicsDevice extends GraphicsDevice
         boolean fsSupported = isFullScreenSupported();
 
         if (fsSupported && old != null) {
-            // restore original display mode and enter windowed mode.
+            // enter windowed mode and restore original display mode
+            exitFullScreenExclusive(old);
             if (originalMode != null) {
                 setDisplayMode(originalMode);
                 originalMode = null;
             }
-            exitFullScreenExclusive(old);
         }
 
         super.setFullScreenWindow(w);
@@ -227,14 +227,10 @@ public final class CGraphicsDevice extends GraphicsDevice
             throw new IllegalArgumentException("Invalid display mode");
         }
         if (!Objects.equals(dm, getDisplayMode())) {
-            final Window w = getFullScreenWindow();
-            if (w != null) {
-                exitFullScreenExclusive(w);
-            }
             nativeSetDisplayMode(displayID, dm.getWidth(), dm.getHeight(),
                     dm.getBitDepth(), dm.getRefreshRate());
-            if (isFullScreenSupported() && w != null) {
-                enterFullScreenExclusive(w);
+            if (isFullScreenSupported() && getFullScreenWindow() != null) {
+                getFullScreenWindow().setSize(dm.getWidth(), dm.getHeight());
             }
         }
     }

src/macosx/native/sun/awt/AWTWindow.m

@@ -1236,7 +1236,9 @@ JNF_COCOA_ENTER(env);
             NSRect screenRect = [[nsWindow screen] frame];
             [nsWindow setFrame:screenRect display:YES];
         } else {
-            [JNFException raise:env as:kRuntimeException reason:"Failed to enter full screen."];            
+            [JNFException raise:[ThreadUtilities getJNIEnv]
+                             as:kRuntimeException
+                         reason:"Failed to enter full screen."];
         }
     }];
 
@@ -1261,7 +1263,9 @@ JNF_COCOA_ENTER(env);
 
             // GraphicsDevice takes care of restoring pre full screen bounds
         } else {
-            [JNFException raise:env as:kRuntimeException reason:"Failed to exit full screen."];
+            [JNFException raise:[ThreadUtilities getJNIEnv]
+                             as:kRuntimeException
+                         reason:"Failed to exit full screen."];
         }
     }];