8056151: Switching to GTK L&F on-the-fly leads to X Window System error RenderBadPicture

Reviewed-by: alexsch, serb

src/solaris/classes/sun/awt/UNIXToolkit.java

@@ -29,11 +29,10 @@ import static java.awt.RenderingHints.*;
 import java.awt.color.ColorSpace;
 import java.awt.image.*;
 import java.security.AccessController;
-import java.security.PrivilegedAction;
 import sun.security.action.GetIntegerAction;
 import com.sun.java.swing.plaf.gtk.GTKConstants.TextDirection;
+import sun.awt.X11.XToolkit;
 import sun.java2d.opengl.OGLRenderQueue;
-import java.lang.reflect.InvocationTargetException;
 
 public abstract class UNIXToolkit extends SunToolkit
 {
@@ -73,16 +72,16 @@ public abstract class UNIXToolkit extends SunToolkit
             if (nativeGTKLoaded != null) {
                 // We've already attempted to load GTK, so just return the
                 // status of that attempt.
-                return nativeGTKLoaded.booleanValue();
+                return nativeGTKLoaded;
 
             } else if (nativeGTKAvailable != null) {
                 // We've already checked the availability of the native GTK
                 // libraries, so just return the status of that attempt.
-                return nativeGTKAvailable.booleanValue();
+                return nativeGTKAvailable;
 
             } else {
                 boolean success = check_gtk();
-                nativeGTKAvailable = Boolean.valueOf(success);
+                nativeGTKAvailable = success;
                 return success;
             }
         }
@@ -99,11 +98,15 @@ public abstract class UNIXToolkit extends SunToolkit
     public boolean loadGTK() {
         synchronized (GTK_LOCK) {
             if (nativeGTKLoaded == null) {
-                boolean success = load_gtk();
-                nativeGTKLoaded = Boolean.valueOf(success);
+                XToolkit.awtLock();
+                try {
+                    nativeGTKLoaded = load_gtk();
+                } finally {
+                    XToolkit.awtUnlock();
+                }
             }
         }
-        return nativeGTKLoaded.booleanValue();
+        return nativeGTKLoaded;
     }
 
     /**
@@ -252,6 +255,7 @@ public abstract class UNIXToolkit extends SunToolkit
 
     private native void nativeSync();
 
+    @Override
     public void sync() {
         // flush the X11 buffer
         nativeSync();
@@ -266,6 +270,8 @@ public abstract class UNIXToolkit extends SunToolkit
      * This requires that the Gnome properties have already been gathered.
      */
     public static final String FONTCONFIGAAHINT = "fontconfig/Antialias";
+
+    @Override
     protected RenderingHints getDesktopAAHints() {
 
         Object aaValue = getDesktopProperty("gnome.Xft/Antialias");
@@ -288,8 +294,8 @@ public abstract class UNIXToolkit extends SunToolkit
          * us to default to "OFF". I don't think that's the best guess.
          * So if its !=0 then lets assume AA.
          */
-        boolean aa = Boolean.valueOf(((aaValue instanceof Number) &&
-                                      ((Number)aaValue).intValue() != 0));
+        boolean aa = ((aaValue instanceof Number)
+                        && ((Number) aaValue).intValue() != 0);
         Object aaHint;
         if (aa) {
             String subpixOrder =

test/javax/swing/plaf/gtk/crash/RenderBadPictureCrash.java

+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ @test
+ @bug 8056151
+ @summary Switching to GTK L&F on-the-fly leads to X Window System error RenderBadPicture
+ @run main/othervm -Dswing.defaultlaf=javax.swing.plaf.metal.MetalLookAndFeel -Dsun.java2d.xrender=T RenderBadPictureCrash
+ */
+
+import java.awt.Color;
+import java.lang.reflect.InvocationTargetException;
+import javax.swing.JFrame;
+import javax.swing.SwingUtilities;
+
+import javax.swing.UIManager;
+
+public class RenderBadPictureCrash {
+
+    public static void main(String[] args) throws InterruptedException, InvocationTargetException {
+        SwingUtilities.invokeAndWait(() -> {
+            JFrame f = new JFrame();
+            f.setUndecorated(true);
+            f.setBackground(new Color(0, 0, 0, 0));
+            f.setSize(200, 300);
+            f.setVisible(true);
+
+            try {
+                UIManager.setLookAndFeel("com.sun.java.swing.plaf.gtk.GTKLookAndFeel");
+            } catch (Exception e) {
+                System.err.println(e);
+                System.err.println("Could not set GTKLookAndFeel, skipping this test");
+            } finally {
+                f.dispose();
+            }
+        });
+    }
+
+}