6796915: Deadlock in XAWT when switching virtual desktops

Reviewed-by: art, anthony

src/solaris/classes/sun/awt/X11/XDropTargetRegistry.java

@@ -534,71 +534,71 @@ final class XDropTargetRegistry {
         return entry.getSite(x, y);
     }
 
+    /*
+     * Note: this method should be called under AWT lock.
+     */
     public void registerDropSite(long window) {
+        assert XToolkit.isAWTLockHeldByCurrentThread();
+
         if (window == 0) {
             throw new IllegalArgumentException();
         }
 
         XDropTargetEventProcessor.activate();
 
-        XToolkit.awtLock();
-        try {
-            long toplevel = getToplevelWindow(window);
-
-            /*
-             * No window with WM_STATE property is found.
-             * Since the window can be a plugin window reparented to the browser
-             * toplevel, we cannot determine which window will eventually have
-             * WM_STATE property set. So we schedule a timer callback that will
-             * periodically attempt to find an ancestor with WM_STATE and
-             * register the drop site appropriately.
-             */
-            if (toplevel == 0) {
-                addDelayedRegistrationEntry(window);
-                return;
-            }
+        long toplevel = getToplevelWindow(window);
 
-            if (toplevel == window) {
-                Iterator dropTargetProtocols =
-                    XDragAndDropProtocols.getDropTargetProtocols();
+        /*
+         * No window with WM_STATE property is found.
+         * Since the window can be a plugin window reparented to the browser
+         * toplevel, we cannot determine which window will eventually have
+         * WM_STATE property set. So we schedule a timer callback that will
+         * periodically attempt to find an ancestor with WM_STATE and
+         * register the drop site appropriately.
+         */
+        if (toplevel == 0) {
+            addDelayedRegistrationEntry(window);
+            return;
+        }
 
-                while (dropTargetProtocols.hasNext()) {
-                    XDropTargetProtocol dropTargetProtocol =
-                        (XDropTargetProtocol)dropTargetProtocols.next();
-                    dropTargetProtocol.registerDropTarget(toplevel);
-                }
-            } else {
-                registerEmbeddedDropSite(toplevel, window);
+        if (toplevel == window) {
+            Iterator dropTargetProtocols =
+                XDragAndDropProtocols.getDropTargetProtocols();
+
+            while (dropTargetProtocols.hasNext()) {
+                XDropTargetProtocol dropTargetProtocol =
+                    (XDropTargetProtocol)dropTargetProtocols.next();
+                dropTargetProtocol.registerDropTarget(toplevel);
             }
-        } finally {
-            XToolkit.awtUnlock();
+        } else {
+            registerEmbeddedDropSite(toplevel, window);
         }
     }
 
+    /*
+     * Note: this method should be called under AWT lock.
+     */
     public void unregisterDropSite(long window) {
+        assert XToolkit.isAWTLockHeldByCurrentThread();
+
         if (window == 0) {
             throw new IllegalArgumentException();
         }
 
-        XToolkit.awtLock();
-        try {
-            long toplevel = getToplevelWindow(window);
+        long toplevel = getToplevelWindow(window);
 
-            if (toplevel == window) {
-                Iterator dropProtocols =
-                    XDragAndDropProtocols.getDropTargetProtocols();
+        if (toplevel == window) {
+            Iterator dropProtocols =
+                XDragAndDropProtocols.getDropTargetProtocols();
 
-                removeDelayedRegistrationEntry(window);
+            removeDelayedRegistrationEntry(window);
 
-                while (dropProtocols.hasNext()) {
-                    XDropTargetProtocol dropProtocol = (XDropTargetProtocol)dropProtocols.next();
-                    dropProtocol.unregisterDropTarget(window);
-                }
-            } else {
-                unregisterEmbeddedDropSite(toplevel, window);
+            while (dropProtocols.hasNext()) {
+                XDropTargetProtocol dropProtocol = (XDropTargetProtocol)dropProtocols.next();
+                dropProtocol.unregisterDropTarget(window);
             }
-        } finally {
-            XToolkit.awtUnlock();
+        } else {
+            unregisterEmbeddedDropSite(toplevel, window);
         }
     }
 

src/solaris/classes/sun/awt/X11/XWindowPeer.java

@@ -1757,25 +1757,36 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         }
     }
 
+    // should be synchronized on awtLock
     private int dropTargetCount = 0;
 
-    public synchronized void addDropTarget() {
-        if (dropTargetCount == 0) {
-            long window = getWindow();
-            if (window != 0) {
-                XDropTargetRegistry.getRegistry().registerDropSite(window);
+    public void addDropTarget() {
+        XToolkit.awtLock();
+        try {
+            if (dropTargetCount == 0) {
+                long window = getWindow();
+                if (window != 0) {
+                    XDropTargetRegistry.getRegistry().registerDropSite(window);
+                }
             }
+            dropTargetCount++;
+        } finally {
+            XToolkit.awtUnlock();
         }
-        dropTargetCount++;
     }
 
-    public synchronized void removeDropTarget() {
-        dropTargetCount--;
-        if (dropTargetCount == 0) {
-            long window = getWindow();
-            if (window != 0) {
-                XDropTargetRegistry.getRegistry().unregisterDropSite(window);
+    public void removeDropTarget() {
+        XToolkit.awtLock();
+        try {
+            dropTargetCount--;
+            if (dropTargetCount == 0) {
+                long window = getWindow();
+                if (window != 0) {
+                    XDropTargetRegistry.getRegistry().unregisterDropSite(window);
+                }
             }
+        } finally {
+            XToolkit.awtUnlock();
         }
     }
     void addRootPropertyEventDispatcher() {
@@ -1838,13 +1849,18 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         }
     }
 
-    protected synchronized void updateDropTarget() {
-        if (dropTargetCount > 0) {
-            long window = getWindow();
-            if (window != 0) {
-                XDropTargetRegistry.getRegistry().unregisterDropSite(window);
-                XDropTargetRegistry.getRegistry().registerDropSite(window);
+    protected void updateDropTarget() {
+        XToolkit.awtLock();
+        try {
+            if (dropTargetCount > 0) {
+                long window = getWindow();
+                if (window != 0) {
+                    XDropTargetRegistry.getRegistry().unregisterDropSite(window);
+                    XDropTargetRegistry.getRegistry().registerDropSite(window);
+                }
             }
+        } finally {
+            XToolkit.awtUnlock();
         }
     }