8187803: JDK part of JavaFX-Swing dialogs appearing behind main stage

Reviewed-by: ssadetsky, prr

src/macosx/classes/sun/lwawt/LWLightweightFramePeer.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,10 +34,11 @@ import java.awt.dnd.DropTarget;
 
 import sun.awt.CausedFocusEvent;
 import sun.awt.LightweightFrame;
+import sun.awt.OverrideNativeWindowHandle;
 import sun.swing.JLightweightFrame;
 import sun.swing.SwingAccessor;
 
-public class LWLightweightFramePeer extends LWWindowPeer {
+public class LWLightweightFramePeer extends LWWindowPeer implements OverrideNativeWindowHandle {
 
     public LWLightweightFramePeer(LightweightFrame target,
                                   PlatformComponent platformComponent,
@@ -116,4 +117,16 @@ public class LWLightweightFramePeer extends LWWindowPeer {
     public void updateCursorImmediately() {
         SwingAccessor.getJLightweightFrameAccessor().updateCursor((JLightweightFrame)getLwTarget());
     }
+
+    // SwingNode
+    private volatile long overriddenWindowHandle = 0L;
+
+    @Override
+    public void overrideWindowHandle(final long handle) {
+        this.overriddenWindowHandle = handle;
+    }
+
+    public long getOverriddenWindowHandle() {
+        return overriddenWindowHandle;
+    }
 }

src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -46,6 +46,7 @@ import sun.awt.AWTAccessor.ComponentAccessor;
 import sun.awt.AWTAccessor.WindowAccessor;
 import sun.java2d.SurfaceData;
 import sun.java2d.opengl.CGLSurfaceData;
+import sun.lwawt.LWLightweightFramePeer;
 import sun.lwawt.*;
 import sun.util.logging.PlatformLogger;
 
@@ -584,6 +585,20 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
                     if (!isKeyWindow) {
                         CWrapper.NSWindow.makeKeyWindow(ptr);
                     }
+
+                    if (owner != null
+                            && owner.getPeer() instanceof LWLightweightFramePeer) {
+                        LWLightweightFramePeer peer =
+                                (LWLightweightFramePeer) owner.getPeer();
+
+                        long ownerWindowPtr = peer.getOverriddenWindowHandle();
+                        if (ownerWindowPtr != 0) {
+                            //Place window above JavaFX stage
+                            CWrapper.NSWindow.addChildWindow(
+                                    ownerWindowPtr, ptr,
+                                    CWrapper.NSWindow.NSWindowAbove);
+                        }
+                    }
                 });
             } else {
                 execute(ptr->{

src/share/classes/sun/awt/OverrideNativeWindowHandle.java

+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.awt;
+
+/**
+ * Used for replacing window owner with another non-Swing window.
+ * It is useful in case of JavaFX-Swing interop:
+ * it helps to keep Swing dialogs above its owner(JavaFX stage).
+ */
+
+public interface OverrideNativeWindowHandle {
+
+    /**
+     * Replaces an owner window with a window with provided handle.
+     * @param handle native window handle
+     */
+    void overrideWindowHandle(final long handle);
+}

src/share/classes/sun/swing/JLightweightFrame.java

 /*
- * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -61,8 +61,10 @@ import javax.swing.RepaintManager;
 import javax.swing.RootPaneContainer;
 import javax.swing.SwingUtilities;
 
+import sun.awt.AWTAccessor;
 import sun.awt.DisplayChangedListener;
 import sun.awt.LightweightFrame;
+import sun.awt.OverrideNativeWindowHandle;
 import sun.security.action.GetPropertyAction;
 import sun.swing.SwingUtilities2.RepaintListener;
 
@@ -478,6 +480,17 @@ public final class JLightweightFrame extends LightweightFrame implements RootPan
         }
     }
 
+    //Called by reflection by SwingNode
+    public void overrideNativeWindowHandle(long handle, Runnable closeWindow) {
+        final Object peer = AWTAccessor.getComponentAccessor().getPeer(this);
+        if (peer instanceof OverrideNativeWindowHandle) {
+            ((OverrideNativeWindowHandle) peer).overrideWindowHandle(handle);
+        }
+        if (closeWindow != null) {
+            closeWindow.run();
+        }
+    }
+
     public <T extends DragGestureRecognizer> T createDragGestureRecognizer(
             Class<T> abstractRecognizerClass,
             DragSource ds, Component c, int srcActions,

src/solaris/classes/sun/awt/X11/XLightweightFramePeer.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,10 +29,11 @@ import java.awt.Graphics;
 import java.awt.dnd.DropTarget;
 
 import sun.awt.LightweightFrame;
+import sun.awt.OverrideNativeWindowHandle;
 import sun.swing.JLightweightFrame;
 import sun.swing.SwingAccessor;
 
-public class XLightweightFramePeer extends XFramePeer {
+public class XLightweightFramePeer extends XFramePeer implements OverrideNativeWindowHandle {
 
     XLightweightFramePeer(LightweightFrame target) {
         super(target);
@@ -80,4 +81,15 @@ public class XLightweightFramePeer extends XFramePeer {
     public void removeDropTarget(DropTarget dt) {
         getLwTarget().removeDropTarget(dt);
     }
+
+    private volatile long overriddenWindowHandle = 0L;
+
+    @Override
+    public void overrideWindowHandle(final long handle) {
+        overriddenWindowHandle = handle;
+    }
+
+    public long getOverriddenWindowHandle() {
+        return overriddenWindowHandle;
+    }
 }

src/solaris/classes/sun/awt/X11/XWindowPeer.java

@@ -1648,6 +1648,16 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         while (!XlibUtil.isToplevelWindow(tpw) && !XlibUtil.isXAWTToplevelWindow(tpw)) {
             tpw = XlibUtil.getParentWindow(tpw);
         }
+
+        XBaseWindow parent = transientForWindow;
+        if (parent instanceof XLightweightFramePeer) {
+            XLightweightFramePeer peer = (XLightweightFramePeer) parent;
+            long ownerWindowPtr = peer.getOverriddenWindowHandle();
+            if (ownerWindowPtr != 0) {
+                tpw = ownerWindowPtr;
+            }
+        }
+
         XlibWrapper.XSetTransientFor(XToolkit.getDisplay(), bpw, tpw);
         window.curRealTransientFor = transientForWindow;
     }

src/windows/classes/sun/awt/windows/WLightweightFramePeer.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,10 +32,11 @@ import java.awt.event.ComponentEvent;
 import java.awt.event.MouseEvent;
 
 import sun.awt.LightweightFrame;
+import sun.awt.OverrideNativeWindowHandle;
 import sun.swing.JLightweightFrame;
 import sun.swing.SwingAccessor;
 
-public class WLightweightFramePeer extends WFramePeer {
+public class WLightweightFramePeer extends WFramePeer implements OverrideNativeWindowHandle {
 
     public WLightweightFramePeer(LightweightFrame target) {
         super(target);
@@ -50,6 +51,13 @@ public class WLightweightFramePeer extends WFramePeer {
         return getLwTarget().getGraphics();
     }
 
+    private native void overrideNativeHandle(long hwnd);
+
+    @Override
+    public void overrideWindowHandle(final long handle) {
+        overrideNativeHandle(handle);
+    }
+
     @Override
     public void show() {
         super.show();

src/windows/native/sun/windows/awt_Dialog.cpp

 /*
- * Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -119,7 +119,8 @@ AwtDialog* AwtDialog::Create(jobject peer, jobject parent)
         if (parent != NULL) {
             JNI_CHECK_PEER_GOTO(parent, done);
             awtParent = (AwtWindow *)pData;
-            hwndParent = awtParent->GetHWnd();
+            HWND oHWnd = awtParent->GetOverriddenHWnd();
+            hwndParent = oHWnd ? oHWnd : awtParent->GetHWnd();
         } else {
             // There is no way to prevent a parentless dialog from showing on
             //  the taskbar other than to specify an invisible parent and set

src/windows/native/sun/windows/awt_Frame.cpp

@@ -169,7 +169,8 @@ AwtFrame* AwtFrame::Create(jobject self, jobject parent)
             JNI_CHECK_PEER_GOTO(parent, done);
             {
                 AwtFrame* parent = (AwtFrame *)pData;
-                hwndParent = parent->GetHWnd();
+                HWND oHWnd = parent->GetOverriddenHWnd();
+                hwndParent = oHWnd ? oHWnd : parent->GetHWnd();
             }
         }
 

src/windows/native/sun/windows/awt_Window.cpp

 /*
- * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -148,6 +148,10 @@ struct SetFullScreenExclusiveModeStateStruct {
     jboolean isFSEMState;
 };
 
+struct OverrideHandle {
+    jobject frame;
+    HWND handle;
+};
 
 /************************************************************************
  * AwtWindow fields
@@ -223,6 +227,7 @@ AwtWindow::AwtWindow() {
     m_alwaysOnTop = false;
 
     fullScreenExclusiveModeState = FALSE;
+    m_overriddenHwnd = NULL;
 }
 
 AwtWindow::~AwtWindow()
@@ -2471,6 +2476,24 @@ ret:
    delete rfs;
 }
 
+void AwtWindow::_OverrideHandle(void *param)
+{
+    JNIEnv *env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
+
+    OverrideHandle* oh = (OverrideHandle *)param;
+    jobject self = oh->frame;
+    AwtWindow *f = NULL;
+
+    PDATA pData;
+    JNI_CHECK_PEER_GOTO(self, ret);
+    f = (AwtWindow *)pData;
+    f->OverrideHWnd(oh->handle);
+ret:
+    env->DeleteGlobalRef(self);
+
+    delete oh;
+}
+
 /*
  * This is AwtWindow-specific function that is not intended for reusing
  */
@@ -3108,7 +3131,29 @@ Java_java_awt_Window_initIDs(JNIEnv *env, jclass cls)
     CATCH_BAD_ALLOC;
 }
 
-} /* extern "C" */
+/*
+
+* Class:     sun_awt_windows_WLightweightFramePeer
+* Method:    overrideNativeHandle
+* Signature: (J)V
+*/
+
+JNIEXPORT void JNICALL Java_sun_awt_windows_WLightweightFramePeer_overrideNativeHandle
+(JNIEnv *env, jobject self, jlong hwnd)
+{
+    TRY;
+
+    OverrideHandle *oh = new OverrideHandle;
+    oh->frame = env->NewGlobalRef(self);
+    oh->handle = (HWND)hwnd;
+
+    AwtToolkit::GetInstance().SyncCall(AwtFrame::_OverrideHandle, oh);
+    // global ref and oh are deleted in _OverrideHandle()
+
+    CATCH_BAD_ALLOC;
+}
+
+}/* extern "C" */
 
 
 /************************************************************************

src/windows/native/sun/windows/awt_Window.h

 /*
- * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -241,6 +241,7 @@ public:
     static void _UpdateWindow(void* param);
     static void _RepositionSecurityWarning(void* param);
     static void _SetFullScreenExclusiveModeState(void* param);
+    static void _OverrideHandle(void *param);
 
     inline static BOOL IsResizing() {
         return sm_resizing;
@@ -256,6 +257,9 @@ public:
 
     static void FocusedWindowChanged(HWND from, HWND to);
 
+    inline HWND GetOverriddenHWnd() { return m_overriddenHwnd; }
+    inline void OverrideHWnd(HWND hwnd) { m_overriddenHwnd = hwnd; }
+
 private:
     static int ms_instanceCounter;
     static HHOOK ms_hCBTFilter;
@@ -307,6 +311,9 @@ private:
     // The tooltip that appears when hovering the icon
     HWND securityTooltipWindow;
 
+    //Allows substitute parent window with JavaFX stage to make it below a dialog
+    HWND m_overriddenHwnd;
+
     UINT warningWindowWidth;
     UINT warningWindowHeight;
     void InitSecurityWarningSize(JNIEnv *env);