8169589: [macosx] Activating a JDialog puts to back another dialog

Reviewed-by: aivanov, serb

src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -31,12 +31,16 @@ import java.awt.event.*;
 import java.awt.peer.WindowPeer;
 import java.beans.*;
 import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
 import javax.swing.*;
 
 import sun.awt.*;
+import sun.awt.AWTAccessor.ComponentAccessor;
+import sun.awt.AWTAccessor.WindowAccessor;
 import sun.java2d.SurfaceData;
 import sun.java2d.opengl.CGLSurfaceData;
 import sun.lwawt.*;
@@ -1015,29 +1019,70 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
         return true;
     }
 
-    private void orderAboveSiblings() {
-        if (owner == null) {
-            return;
+    private boolean isOneOfOwnersOrSelf(CPlatformWindow window) {
+        while (window != null) {
+            if (this == window) {
+                return true;
+            }
+            window = window.owner;
         }
+        return false;
+    }
 
-        // NOTE: the logic will fail if we have a hierarchy like:
-        //       visible root owner
-        //          invisible owner
-        //              visible dialog
-        // However, this is an unlikely scenario for real life apps
-        if (owner.isVisible()) {
-            // Recursively pop up the windows from the very bottom so that only
-            // the very top-most one becomes the main window
-            owner.orderAboveSiblings();
-
-            // Order the window to front of the stack of child windows
-            final long nsWindowSelfPtr = getNSWindowPtr();
-            final long nsWindowOwnerPtr = owner.getNSWindowPtr();
-            CWrapper.NSWindow.orderFront(nsWindowOwnerPtr);
-            CWrapper.NSWindow.orderWindow(nsWindowSelfPtr, CWrapper.NSWindow.NSWindowAbove, nsWindowOwnerPtr);
+    private CPlatformWindow getRootOwner() {
+        CPlatformWindow rootOwner = this;
+        while (rootOwner.owner != null) {
+            rootOwner = rootOwner.owner;
         }
+        return rootOwner;
+    }
 
-        applyWindowLevel(target);
+    private void orderAboveSiblings() {
+        // Recursively pop up the windows from the very bottom, (i.e. root owner) so that
+        // the windows are ordered above their nearest owner; ancestors of the window,
+        // which is going to become 'main window', are placed above their siblings.
+        CPlatformWindow rootOwner = getRootOwner();
+        if (rootOwner.isVisible()) {
+            CWrapper.NSWindow.orderFront(rootOwner.getNSWindowPtr());
+        }
+        final WindowAccessor windowAccessor = AWTAccessor.getWindowAccessor();
+        orderAboveSiblingsImpl(windowAccessor.getOwnedWindows(rootOwner.target));
+    }
+
+    private void orderAboveSiblingsImpl(Window[] windows) {
+        ArrayList<Window> childWindows = new ArrayList<Window>();
+
+        final ComponentAccessor componentAccessor = AWTAccessor.getComponentAccessor();
+        final WindowAccessor windowAccessor = AWTAccessor.getWindowAccessor();
+
+        // Go through the list of windows and perform ordering.
+        for (Window w : windows) {
+            final Object p = componentAccessor.getPeer(w);
+            if (p instanceof LWWindowPeer) {
+                CPlatformWindow pw = (CPlatformWindow)((LWWindowPeer)p).getPlatformWindow();
+                if (pw != null && pw.isVisible()) {
+                    // If the window is one of ancestors of 'main window' or is going to become main by itself,
+                    // the window should be ordered above its siblings; otherwise the window is just ordered
+                    // above its nearest parent.
+                    if (pw.isOneOfOwnersOrSelf(this)) {
+                        CWrapper.NSWindow.orderFront(pw.getNSWindowPtr());
+                    } else {
+                        CWrapper.NSWindow.orderWindow(pw.getNSWindowPtr(), CWrapper.NSWindow.NSWindowAbove,
+                                pw.owner.getNSWindowPtr());
+                    }
+                    pw.applyWindowLevel(w);
+                }
+            }
+            // Retrieve the child windows for each window from the list and store them for future use.
+            // Note: we collect data about child windows even for invisible owners, since they may have
+            // visible children.
+            childWindows.addAll(Arrays.asList(windowAccessor.getOwnedWindows(w)));
+        }
+        // If some windows, which have just been ordered, have any child windows, let's start new iteration
+        // and order these child windows.
+        if (!childWindows.isEmpty()) {
+            orderAboveSiblingsImpl(childWindows.toArray(new Window[0]));
+        }
     }
 
     protected void applyWindowLevel(Window target) {

src/share/classes/java/awt/Window.java

@@ -4100,6 +4100,10 @@ public class Window extends Container implements Accessible {
             public void setTrayIconWindow(Window w, boolean isTrayIconWindow) {
                 w.isTrayIconWindow = isTrayIconWindow;
             }
+
+            public Window[] getOwnedWindows(Window w) {
+                return w.getOwnedWindows_NoClientCode();
+            }
         }); // WindowAccessor
     } // static
 

src/share/classes/sun/awt/AWTAccessor.java

@@ -333,6 +333,12 @@ public final class AWTAccessor {
          * Marks the specified window as an utility window for TrayIcon.
          */
         void setTrayIconWindow(Window w, boolean isTrayIconWindow);
+
+        /**
+         * Return an array containing all the windows this
+         * window currently owns.
+         */
+        Window[] getOwnedWindows(Window w);
     }
 
     /**

test/java/awt/Dialog/DialogAboveFrame/DialogAboveFrameTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8169589
+ * @summary Activating a dialog puts to back another dialog owned by the same frame
+ * @author Dmitry Markov
+ * @library ../../regtesthelpers
+ * @build Util
+ * @run main DialogAboveFrameTest
+ */
+
+import java.awt.Color;
+import java.awt.Dialog;
+import java.awt.Frame;
+import java.awt.Point;
+import java.awt.Robot;
+
+import test.java.awt.regtesthelpers.Util;
+
+public class DialogAboveFrameTest {
+    public static void main(String[] args) {
+        Robot robot = Util.createRobot();
+
+        Frame frame = new Frame("Frame");
+        frame.setBackground(Color.BLUE);
+        frame.setBounds(200, 50, 300, 300);
+        frame.setVisible(true);
+
+        Dialog dialog1 = new Dialog(frame, "Dialog 1", false);
+        dialog1.setBackground(Color.RED);
+        dialog1.setBounds(100, 100, 200, 200);
+        dialog1.setVisible(true);
+
+        Dialog dialog2 = new Dialog(frame, "Dialog 2", false);
+        dialog2.setBackground(Color.GREEN);
+        dialog2.setBounds(400, 100, 200, 200);
+        dialog2.setVisible(true);
+
+        Util.waitForIdle(robot);
+
+        Util.clickOnComp(dialog2, robot);
+        Util.waitForIdle(robot);
+
+        Point point = dialog1.getLocationOnScreen();
+        int x = point.x + (int)(dialog1.getWidth() * 0.9);
+        int y = point.y + (int)(dialog1.getHeight() * 0.9);
+
+        try {
+            if (!robot.getPixelColor(x, y).equals(dialog1.getBackground())) {
+                throw new RuntimeException("Test FAILED: Dialog is behind the frame");
+            }
+        } finally {
+            frame.dispose();
+            dialog1.dispose();
+            dialog2.dispose();
+        }
+    }
+}
+