8013827: File.createTempFile hangs with temp file starting with 'com1.4'

8011950: java.io.File.createTempFile enters infinite loop when passed invalid data
Reviewed-by: alanb

src/share/classes/java/io/File.java

@@ -1899,14 +1899,20 @@ public class File
 
         // file name generation
         private static final SecureRandom random = new SecureRandom();
-        static File generateFile(String prefix, String suffix, File dir) {
+        static File generateFile(String prefix, String suffix, File dir)
+            throws IOException
+        {
             long n = random.nextLong();
             if (n == Long.MIN_VALUE) {
                 n = 0;      // corner case
             } else {
                 n = Math.abs(n);
             }
-            return new File(dir, prefix + Long.toString(n) + suffix);
+            String name = prefix + Long.toString(n) + suffix;
+            File f = new File(dir, name);
+            if (!name.equals(f.getName()))
+                throw new IOException("Unable to create temporary file");
+            return f;
         }
     }
 
@@ -1988,25 +1994,21 @@ public class File
         if (suffix == null)
             suffix = ".tmp";
 
-        File tmpdir = (directory != null) ? directory : TempDirectory.location();
-        SecurityManager sm = System.getSecurityManager();
+        File tmpdir = (directory != null) ? directory
+                                          : TempDirectory.location();
         File f;
-        do {
-            f = TempDirectory.generateFile(prefix, suffix, tmpdir);
-            if (sm != null) {
-                try {
-                    sm.checkWrite(f.getPath());
-                } catch (SecurityException se) {
-                    // don't reveal temporary directory location
-                    if (directory == null)
-                        throw new SecurityException("Unable to create temporary file");
-                    throw se;
-                }
-            }
-            if (f.isInvalid()) {
+        try {
+            do {
+                f = TempDirectory.generateFile(prefix, suffix, tmpdir);
+            } while (f.exists());
+            if (!f.createNewFile())
                 throw new IOException("Unable to create temporary file");
-            }
-        } while (!fs.createFileExclusively(f.getPath()));
+        } catch (SecurityException se) {
+            // don't reveal temporary directory location
+            if (directory == null)
+                throw new SecurityException("Unable to create temporary file");
+            throw se;
+        }
         return f;
     }
 

src/windows/native/java/io/WinNTFileSystem_md.c

@@ -541,6 +541,10 @@ Java_java_io_WinNTFileSystem_createFileExclusively(JNIEnv *env, jclass cls,
     WCHAR *pathbuf = pathToNTPath(env, path, JNI_FALSE);
     if (pathbuf == NULL)
         return JNI_FALSE;
+    if (isReservedDeviceNameW(pathbuf)) {
+        free(pathbuf);
+        return JNI_FALSE;
+    }
     h = CreateFileW(
         pathbuf,                              /* Wide char path name */
         GENERIC_READ | GENERIC_WRITE,         /* Read and write permission */

test/java/io/File/CreateNewFile.java

 /*
- * Copyright (c) 1998, 2001, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,7 +22,7 @@
  */
 
 /* @test
-   @bug 4130498 4391178
+   @bug 4130498 4391178 6198547
    @summary Basic test for createNewFile method
  */
 
@@ -51,5 +51,20 @@ public class CreateNewFile {
         } catch (IOException e) {
             // Exception expected
         }
+
+        testCreateExistingDir();
+    }
+
+    // Test JDK-6198547
+    private static void testCreateExistingDir() throws IOException {
+        File tmpFile = new File("hugo");
+        if (tmpFile.exists() && !tmpFile.delete())
+            throw new RuntimeException("Cannot delete " + tmpFile);
+        if (!tmpFile.mkdir())
+            throw new RuntimeException("Cannot create dir " + tmpFile);
+        if (!tmpFile.exists())
+            throw new RuntimeException("Cannot see created dir " + tmpFile);
+        if (tmpFile.createNewFile())
+            throw new RuntimeException("Should fail to create file " + tmpFile);
     }
 }

test/java/io/File/NulFile.java

@@ -612,7 +612,7 @@ public class NulFile {
             try {
                 File.createTempFile(prefix, suffix, directory);
             } catch (IOException ex) {
-                if ("Unable to create temporary file".equals(ex.getMessage()))
+                if (ExceptionMsg.equals(ex.getMessage()))
                     exceptionThrown = true;
             }
         }

test/java/io/File/createTempFile/SpecialTempFile.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8013827 8011950
+ * @summary Check whether File.createTempFile can handle special parameters
+ *          on Windows platforms
+ * @author Dan Xu
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+public class SpecialTempFile {
+
+    private static void test(String name, String[] prefix, String[] suffix) {
+        if (prefix == null || suffix == null
+            || prefix.length != suffix.length)
+        {
+            return;
+        }
+
+        final String exceptionMsg = "Unable to create temporary file";
+        final String errMsg = "IOException is expected";
+
+        for (int i = 0; i < prefix.length; i++) {
+            boolean exceptionThrown = false;
+            File f = null;
+            System.out.println("In test " + name
+                               + ", creating temp file with prefix, "
+                               + prefix[i] + ", suffix, " + suffix[i]);
+            try {
+                f = File.createTempFile(prefix[i], suffix[i]);
+            } catch (IOException e) {
+                if (exceptionMsg.equals(e.getMessage()))
+                    exceptionThrown = true;
+                else
+                    System.out.println("Wrong error message:" + e.getMessage());
+            }
+            if (!exceptionThrown || f != null)
+                throw new RuntimeException(errMsg);
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (!System.getProperty("os.name").startsWith("Windows"))
+            return;
+
+        // Test JDK-8013827
+        String[] resvPre = { "LPT1.package.zip", "com7.4.package.zip" };
+        String[] resvSuf = { ".temp", ".temp" };
+        test("ReservedName", resvPre, resvSuf);
+
+        // Test JDK-8011950
+        String[] slashPre = { "///..///", "temp", "///..///" };
+        String[] slashSuf = { ".temp", "///..///..", "///..///.." };
+        test("SlashedName", slashPre, slashSuf);
+    }
+}