8075516: Deleting a file from either the open or save java.awt.FileDialog hangs.

Reviewed-by: alexsch, serb

src/windows/native/sun/windows/awt_FileDialog.cpp

@@ -349,9 +349,9 @@ AwtFileDialog::Show(void *p)
 
         // show the Win32 file dialog
         if (mode == java_awt_FileDialog_LOAD) {
-            result = AwtFileDialog::GetOpenFileName(&ofn);
+            result = ::GetOpenFileName(&ofn);
         } else {
-            result = AwtFileDialog::GetSaveFileName(&ofn);
+            result = ::GetSaveFileName(&ofn);
         }
         // Fix for 4181310: FileDialog does not show up.
         // If the dialog is not shown because of invalid file name
@@ -361,9 +361,9 @@ AwtFileDialog::Show(void *p)
             if (dlgerr == FNERR_INVALIDFILENAME) {
                 _tcscpy_s(fileBuffer, bufferLimit, TEXT(""));
                 if (mode == java_awt_FileDialog_LOAD) {
-                    result = AwtFileDialog::GetOpenFileName(&ofn);
+                    result = ::GetOpenFileName(&ofn);
                 } else {
-                    result = AwtFileDialog::GetSaveFileName(&ofn);
+                    result = ::GetSaveFileName(&ofn);
                 }
             }
         }
@@ -422,22 +422,6 @@ AwtFileDialog::Show(void *p)
         delete[] ofn.lpstrFile;
 }
 
-BOOL
-AwtFileDialog::GetOpenFileName(LPOPENFILENAME data) {
-    return static_cast<BOOL>(reinterpret_cast<INT_PTR>(
-        AwtToolkit::GetInstance().InvokeFunction((void*(*)(void*))
-                     ::GetOpenFileName, data)));
-
-}
-
-BOOL
-AwtFileDialog::GetSaveFileName(LPOPENFILENAME data) {
-    return static_cast<BOOL>(reinterpret_cast<INT_PTR>(
-        AwtToolkit::GetInstance().InvokeFunction((void *(*)(void *))
-                     ::GetSaveFileName, data)));
-
-}
-
 BOOL AwtFileDialog::InheritsNativeMouseWheelBehavior() {return true;}
 
 void AwtFileDialog::_DisposeOrHide(void *param)
@@ -585,9 +569,10 @@ Java_sun_awt_windows_WFileDialogPeer__1show(JNIEnv *env, jobject peer)
      */
     jobject peerGlobal = env->NewGlobalRef(peer);
 
-    AwtToolkit::GetInstance().InvokeFunction(AwtFileDialog::Show, peerGlobal);
-
-    env->DeleteGlobalRef(peerGlobal);
+    if (!AwtToolkit::GetInstance().PostMessage(WM_AWT_INVOKE_METHOD,
+                             (WPARAM)AwtFileDialog::Show, (LPARAM)peerGlobal)) {
+        env->DeleteGlobalRef(peerGlobal);
+    }
 
     CATCH_BAD_ALLOC;
 }

src/windows/native/sun/windows/awt_FileDialog.h

@@ -60,9 +60,6 @@ public:
     static void Initialize(JNIEnv *env, jstring filterDescription);
     static void Show(void *peer);
 
-    static BOOL GetOpenFileName(LPOPENFILENAME);
-    static BOOL GetSaveFileName(LPOPENFILENAME);
-
     virtual BOOL InheritsNativeMouseWheelBehavior();
 
     // some methods called on Toolkit thread

test/java/awt/FileDialog/DeleteInsideFileDialog/DeleteInsideFileDialogTest.java

+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+  @test
+  @bug 8075516
+  @requires os.family=="windows"
+  @summary Deleting a file from either the open or save java.awt.FileDialog
+           hangs.
+  @run main/manual DeleteInsideFileDialogTest
+*/
+
+import java.awt.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class DeleteInsideFileDialogTest {
+
+    private static Path dir;
+    private static Path file1;
+    private static Path file2;
+    private static Frame f;
+    private static FileDialog fd;
+
+    public static void main(String[] args) throws Exception {
+
+        String instructions =
+                "1) Delete file deleteMe.tst in the opened File Dialog window" +
+                   " using the right click popup menu\n" +
+                "2) Select thenSelectMe.tst file in the File Dialog and press" +
+                   " Open (if this is not possible the test fails)\n";
+        dir = Files.createTempDirectory("Test");
+        file1 = Files.createFile(Paths.get(dir.toString(), "deleteMe.tst"));
+        file2 = Files.createFile(Paths.get(dir.toString(), "thenSelectMe.tst"));
+        try {
+            f = new Frame("Instructions");
+            f.add(new TextArea(instructions, 6, 60, TextArea.SCROLLBARS_NONE));
+            f.pack();
+            f.setLocation(100, 500);
+            f.setVisible(true);
+
+            fd = new FileDialog((Frame)null);
+            fd.setDirectory(dir.toString());
+            fd.setVisible(true);
+            if (fd.getFile() == null) {
+                throw new RuntimeException("Failed");
+            }
+        } finally {
+            if (fd != null) {
+                fd.dispose();
+            }
+            if (f != null) {
+                f.dispose();
+            }
+            Files.deleteIfExists(file1);
+            Files.deleteIfExists(file2);
+            Files.deleteIfExists(dir);
+        }
+    }
+}