6724890: Deadlock between AWT-EventQueue-1 and AWT-XAWT threads during IDE start

Reviewed-by: art, ant

src/share/classes/java/awt/Frame.java

@@ -36,6 +36,7 @@ import java.io.ObjectInputStream;
 import java.io.IOException;
 import sun.awt.AppContext;
 import sun.awt.SunToolkit;
+import sun.awt.AWTAccessor;
 import java.lang.ref.WeakReference;
 import javax.accessibility.*;
 
@@ -738,11 +739,15 @@ public class Frame extends Window implements MenuContainer {
      * @since   1.4
      * @see java.awt.Window#addWindowStateListener
      */
-    public synchronized void setExtendedState(int state) {
+    public void setExtendedState(int state) {
         if ( !isFrameStateSupported( state ) ) {
             return;
         }
-        this.state = state;
+        synchronized (getObjectLock()) {
+            this.state = state;
+        }
+        // peer.setState must be called outside of object lock
+        // synchronization block to avoid possible deadlock
         FramePeer peer = (FramePeer)this.peer;
         if (peer != null) {
             peer.setState(state);
@@ -804,12 +809,27 @@ public class Frame extends Window implements MenuContainer {
      * @see     #setExtendedState(int)
      * @since 1.4
      */
-    public synchronized int getExtendedState() {
-        FramePeer peer = (FramePeer)this.peer;
-        if (peer != null) {
-            state = peer.getState();
+    public int getExtendedState() {
+        synchronized (getObjectLock()) {
+            return state;
         }
-        return state;
+    }
+
+    static {
+        AWTAccessor.setFrameAccessor(
+            new AWTAccessor.FrameAccessor() {
+                public void setExtendedState(Frame frame, int state) {
+                    synchronized(frame.getObjectLock()) {
+                        frame.state = state;
+                    }
+                }
+                public int getExtendedState(Frame frame) {
+                    synchronized(frame.getObjectLock()) {
+                        return frame.state;
+                    }
+                }
+            }
+        );
     }
 
     /**
@@ -967,7 +987,7 @@ public class Frame extends Window implements MenuContainer {
         if (resizable) {
             str += ",resizable";
         }
-        getExtendedState();     // sync with peer
+        int state = getExtendedState();
         if (state == NORMAL) {
             str += ",normal";
         }

src/share/classes/sun/awt/AWTAccessor.java

@@ -132,6 +132,20 @@ public final class AWTAccessor {
         boolean isSystemGenerated(AWTEvent ev);
     }
 
+    /*
+     * An accessor for the java.awt.Frame class.
+     */
+    public interface FrameAccessor {
+        /*
+         * Sets the state of this frame.
+         */
+        void setExtendedState(Frame frame, int state);
+        /*
+         * Gets the state of this frame.
+         */
+       int getExtendedState(Frame frame);
+    }
+
     /*
      * The java.awt.Component class accessor object.
      */
@@ -147,6 +161,11 @@ public final class AWTAccessor {
      */
     private static AWTEventAccessor awtEventAccessor;
 
+    /*
+     * The java.awt.Frame class accessor object.
+     */
+    private static FrameAccessor frameAccessor;
+
     /*
      * Set an accessor object for the java.awt.Component class.
      */
@@ -195,4 +214,21 @@ public final class AWTAccessor {
     public static AWTEventAccessor getAWTEventAccessor() {
         return awtEventAccessor;
     }
+
+    /*
+     * Set an accessor object for the java.awt.Frame class.
+     */
+    public static void setFrameAccessor(FrameAccessor fa) {
+        frameAccessor = fa;
+    }
+
+    /*
+     * Retrieve the accessor object for the java.awt.Frame class.
+     */
+    public static FrameAccessor getFrameAccessor() {
+        if (frameAccessor == null) {
+            unsafe.ensureClassInitialized(Frame.class);
+        }
+        return frameAccessor;
+    }
 }

src/solaris/classes/sun/awt/X11/XFramePeer.java

@@ -36,6 +36,7 @@ import java.awt.Rectangle;
 import java.awt.peer.FramePeer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import sun.awt.AWTAccessor;
 
 class XFramePeer extends XDecoratedPeer implements FramePeer {
     private static Logger log = Logger.getLogger("sun.awt.X11.XFramePeer");
@@ -231,13 +232,19 @@ class XFramePeer extends XDecoratedPeer implements FramePeer {
         }
     }
 
-    public int getState() { return state; }
+    public int getState() {
+        synchronized(getStateLock()) {
+            return state;
+        }
+    }
 
     public void setState(int newState) {
-        if (!isShowing()) {
-            stateLog.finer("Frame is not showing");
-            state = newState;
-            return;
+        synchronized(getStateLock()) {
+            if (!isShowing()) {
+                stateLog.finer("Frame is not showing");
+                state = newState;
+                return;
+            }
         }
         changeState(newState);
     }
@@ -296,6 +303,9 @@ class XFramePeer extends XDecoratedPeer implements FramePeer {
         int old_state = state;
         state = newState;
 
+        // sync target with peer
+        AWTAccessor.getFrameAccessor().setExtendedState((Frame)target, state);
+
         if ((changed & Frame.ICONIFIED) != 0) {
             if ((state & Frame.ICONIFIED) != 0) {
                 stateLog.finer("Iconified");

src/windows/classes/sun/awt/windows/WFramePeer.java

@@ -25,27 +25,46 @@
 package sun.awt.windows;
 
 import java.util.Vector;
+
 import java.awt.*;
 import java.awt.peer.*;
 import java.awt.image.ImageObserver;
-import sun.awt.image.ImageRepresentation;
-import sun.awt.image.IntegerComponentRaster;
-import sun.awt.image.ToolkitImage;
+
 import java.awt.image.Raster;
 import java.awt.image.DataBuffer;
 import java.awt.image.DataBufferInt;
 import java.awt.image.BufferedImage;
-import sun.awt.im.*;
-import sun.awt.Win32GraphicsDevice;
+
 import java.awt.image.ColorModel;
 
+import sun.awt.image.ImageRepresentation;
+import sun.awt.image.IntegerComponentRaster;
+import sun.awt.image.ToolkitImage;
+import sun.awt.im.*;
+import sun.awt.Win32GraphicsDevice;
+import sun.awt.AWTAccessor;
 
 class WFramePeer extends WWindowPeer implements FramePeer {
 
+    static {
+        initIDs();
+    }
+
+    // initialize JNI field and method IDs
+    private static native void initIDs();
+
     // FramePeer implementation
     public native void setState(int state);
     public native int getState();
 
+    // sync target and peer
+    public void setExtendedState(int state) {
+        AWTAccessor.getFrameAccessor().setExtendedState((Frame)target, state);
+    }
+    public int getExtendedState() {
+        return AWTAccessor.getFrameAccessor().getExtendedState((Frame)target);
+    }
+
     // Convenience methods to save us from trouble of extracting
     // Rectangle fields in native code.
     private native void setMaximizedBounds(int x, int y, int w, int h);

src/windows/native/sun/windows/awt_Frame.cpp

@@ -90,8 +90,10 @@ struct BlockedThreadStruct {
  */
 
 jfieldID AwtFrame::handleID;
-jfieldID AwtFrame::stateID;
+
 jfieldID AwtFrame::undecoratedID;
+jmethodID AwtFrame::getExtendedStateMID;
+jmethodID AwtFrame::setExtendedStateMID;
 
 jmethodID AwtFrame::activateEmbeddingTopLevelMID;
 
@@ -232,7 +234,7 @@ AwtFrame* AwtFrame::Create(jobject self, jobject parent)
                 frame->InitPeerGraphicsConfig(env, self);
                 AwtToolkit::GetInstance().RegisterEmbedderProcessId(hwndParent);
             } else {
-                jint state = env->GetIntField(target, AwtFrame::stateID);
+                jint state = env->CallIntMethod(self, AwtFrame::getExtendedStateMID);
                 DWORD exStyle;
                 DWORD style;
 
@@ -883,6 +885,11 @@ MsgRouting AwtFrame::WmSize(UINT type, int w, int h)
     if (changed != 0) {
         DTRACE_PRINTLN2("AwtFrame::WmSize: reporting state change %x -> %x",
                 oldState, newState);
+
+        // sync target with peer
+        JNIEnv *env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
+        env->CallVoidMethod(GetPeer(env), AwtFrame::setExtendedStateMID, newState);
+
         // report (de)iconification to old clients
         if (changed & java_awt_Frame_ICONIFIED) {
             if (newState & java_awt_Frame_ICONIFIED) {
@@ -1594,7 +1601,7 @@ void AwtFrame::_NotifyModalBlocked(void *param)
 extern "C" {
 
 /*
- * Class:     sun_awt_windows_WFramePeer
+ * Class:     java_awt_Frame
  * Method:    initIDs
  * Signature: ()V
  */
@@ -1603,15 +1610,31 @@ Java_java_awt_Frame_initIDs(JNIEnv *env, jclass cls)
 {
     TRY;
 
-    AwtFrame::stateID = env->GetFieldID(cls, "state", "I");
-    DASSERT(AwtFrame::stateID != NULL);
-
     AwtFrame::undecoratedID = env->GetFieldID(cls,"undecorated","Z");
     DASSERT(AwtFrame::undecoratedID != NULL);
 
     CATCH_BAD_ALLOC;
 }
 
+/*
+ * Class:     sun_awt_windows_WFramePeer
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL
+Java_sun_awt_windows_WFramePeer_initIDs(JNIEnv *env, jclass cls)
+{
+    TRY;
+
+    AwtFrame::setExtendedStateMID = env->GetMethodID(cls, "setExtendedState", "(I)V");
+    AwtFrame::getExtendedStateMID = env->GetMethodID(cls, "getExtendedState", "()I");
+
+    DASSERT(AwtFrame::setExtendedStateMID);
+    DASSERT(AwtFrame::getExtendedStateMID);
+
+    CATCH_BAD_ALLOC;
+}
+
 /*
  * Class:     sun_awt_windows_WFramePeer
  * Method:    setState

src/windows/native/sun/windows/awt_Frame.h

@@ -48,14 +48,14 @@ public:
         FRAME_SETMENUBAR
     };
 
-    /* int handle field for sun.awt.windows.WEmbeddedFrame */
-    static jfieldID handleID;
+    /* java.awt.Frame fields and method IDs */
+    static jfieldID undecoratedID;
 
-    /* int state field for java.awt.Frame */
-    static jfieldID stateID;
+    /* sun.awt.windows.WEmbeddedFrame fields and method IDs */
+    static jfieldID handleID;
 
-    /* boolean undecorated field for java.awt.Frame */
-    static jfieldID undecoratedID;
+    static jmethodID setExtendedStateMID;
+    static jmethodID getExtendedStateMID;
 
     /* method id for WEmbeddedFrame.requestActivate() method */
     static jmethodID activateEmbeddingTopLevelMID;