8003992: File and other classes in java.io do not handle embedded nulls properly

Summary: Have every file operation done with File, FileInputStream, FileOutputStream, or RandomAccessFile that involves a file path containing NUL fail. Also reviewed by fweimer@redhat.com
Reviewed-by: alanb, sherman, ahgross, mduigou, dholmes, aph, plevart, martin

src/share/classes/java/io/File.java

 /*
- * Copyright (c) 1994, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -164,6 +164,32 @@ public class File
      */
     private final String path;
 
+    /**
+     * Enum type that indicates the status of a file path.
+     */
+    private static enum PathStatus { INVALID, CHECKED };
+
+    /**
+     * The flag indicating whether the file path is invalid.
+     */
+    private transient PathStatus status = null;
+
+    /**
+     * Check if the file has an invalid path. Currently, the inspection of
+     * a file path is very limited, and it only covers Nul character check.
+     * Returning true means the path is definitely invalid/garbage. But
+     * returning false does not guarantee that the path is valid.
+     *
+     * @return true if the file path is invalid.
+     */
+    final boolean isInvalid() {
+        if (status == null) {
+            status = (this.path.indexOf('\u0000') < 0) ? PathStatus.CHECKED
+                                                       : PathStatus.INVALID;
+        }
+        return status == PathStatus.INVALID;
+    }
+
     /**
      * The length of this abstract pathname's prefix, or zero if it has no
      * prefix.
@@ -586,6 +612,9 @@ public class File
      * @see     Path#toRealPath
      */
     public String getCanonicalPath() throws IOException {
+        if (isInvalid()) {
+            throw new IOException("Invalid file path");
+        }
         return fs.canonicalize(fs.resolve(this));
     }
 
@@ -651,6 +680,9 @@ public class File
      */
     @Deprecated
     public URL toURL() throws MalformedURLException {
+        if (isInvalid()) {
+            throw new MalformedURLException("Invalid file path");
+        }
         return new URL("file", "", slashify(getAbsolutePath(), isDirectory()));
     }
 
@@ -730,6 +762,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.checkAccess(this, FileSystem.ACCESS_READ);
     }
 
@@ -755,6 +790,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.checkAccess(this, FileSystem.ACCESS_WRITE);
     }
 
@@ -775,6 +813,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return ((fs.getBooleanAttributes(this) & FileSystem.BA_EXISTS) != 0);
     }
 
@@ -802,6 +843,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return ((fs.getBooleanAttributes(this) & FileSystem.BA_DIRECTORY)
                 != 0);
     }
@@ -832,6 +876,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return ((fs.getBooleanAttributes(this) & FileSystem.BA_REGULAR) != 0);
     }
 
@@ -858,6 +905,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return ((fs.getBooleanAttributes(this) & FileSystem.BA_HIDDEN) != 0);
     }
 
@@ -887,6 +937,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return 0L;
+        }
         return fs.getLastModifiedTime(this);
     }
 
@@ -915,6 +968,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return 0L;
+        }
         return fs.getLength(this);
     }
 
@@ -950,6 +1006,9 @@ public class File
     public boolean createNewFile() throws IOException {
         SecurityManager security = System.getSecurityManager();
         if (security != null) security.checkWrite(path);
+        if (isInvalid()) {
+            throw new IOException("Invalid file path");
+        }
         return fs.createFileExclusively(path);
     }
 
@@ -976,6 +1035,9 @@ public class File
         if (security != null) {
             security.checkDelete(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.delete(this);
     }
 
@@ -1011,6 +1073,9 @@ public class File
         if (security != null) {
             security.checkDelete(path);
         }
+        if (isInvalid()) {
+            return;
+        }
         DeleteOnExitHook.add(path);
     }
 
@@ -1051,6 +1116,9 @@ public class File
         if (security != null) {
             security.checkRead(path);
         }
+        if (isInvalid()) {
+            return null;
+        }
         return fs.list(this);
     }
 
@@ -1242,6 +1310,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.createDirectory(this);
     }
 
@@ -1317,6 +1388,12 @@ public class File
             security.checkWrite(path);
             security.checkWrite(dest.path);
         }
+        if (dest == null) {
+            throw new NullPointerException();
+        }
+        if (this.isInvalid() || dest.isInvalid()) {
+            return false;
+        }
         return fs.rename(this, dest);
     }
 
@@ -1352,6 +1429,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.setLastModifiedTime(this, time);
     }
 
@@ -1379,6 +1459,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.setReadOnly(this);
     }
 
@@ -1419,6 +1502,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.setPermission(this, FileSystem.ACCESS_WRITE, writable, ownerOnly);
     }
 
@@ -1493,6 +1579,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.setPermission(this, FileSystem.ACCESS_READ, readable, ownerOnly);
     }
 
@@ -1570,6 +1659,9 @@ public class File
         if (security != null) {
             security.checkWrite(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.setPermission(this, FileSystem.ACCESS_EXECUTE, executable, ownerOnly);
     }
 
@@ -1629,6 +1721,9 @@ public class File
         if (security != null) {
             security.checkExec(path);
         }
+        if (isInvalid()) {
+            return false;
+        }
         return fs.checkAccess(this, FileSystem.ACCESS_EXECUTE);
     }
 
@@ -1705,6 +1800,9 @@ public class File
             sm.checkPermission(new RuntimePermission("getFileSystemAttributes"));
             sm.checkRead(path);
         }
+        if (isInvalid()) {
+            return 0L;
+        }
         return fs.getSpace(this, FileSystem.SPACE_TOTAL);
     }
 
@@ -1721,7 +1819,7 @@ public class File
      * makes no guarantee that write operations to this file system
      * will succeed.
      *
-     * @return  The number of unallocated bytes on the partition <tt>0L</tt>
+     * @return  The number of unallocated bytes on the partition or <tt>0L</tt>
      *          if the abstract pathname does not name a partition.  This
      *          value will be less than or equal to the total file system size
      *          returned by {@link #getTotalSpace}.
@@ -1740,6 +1838,9 @@ public class File
             sm.checkPermission(new RuntimePermission("getFileSystemAttributes"));
             sm.checkRead(path);
         }
+        if (isInvalid()) {
+            return 0L;
+        }
         return fs.getSpace(this, FileSystem.SPACE_FREE);
     }
 
@@ -1778,6 +1879,9 @@ public class File
             sm.checkPermission(new RuntimePermission("getFileSystemAttributes"));
             sm.checkRead(path);
         }
+        if (isInvalid()) {
+            return 0L;
+        }
         return fs.getSpace(this, FileSystem.SPACE_USABLE);
     }
 
@@ -1787,8 +1891,8 @@ public class File
         private TempDirectory() { }
 
         // temporary directory location
-        private static final File tmpdir = new File(fs.normalize(AccessController
-            .doPrivileged(new GetPropertyAction("java.io.tmpdir"))));
+        private static final File tmpdir = new File(AccessController
+            .doPrivileged(new GetPropertyAction("java.io.tmpdir")));
         static File location() {
             return tmpdir;
         }
@@ -1899,6 +2003,9 @@ public class File
                     throw se;
                 }
             }
+            if (f.isInvalid()) {
+                throw new IOException("Unable to create temporary file");
+            }
         } while (!fs.createFileExclusively(f.getPath()));
         return f;
     }

src/share/classes/java/io/FileInputStream.java

 /*
- * Copyright (c) 1994, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -123,6 +123,9 @@ class FileInputStream extends InputStream
         if (name == null) {
             throw new NullPointerException();
         }
+        if (file.isInvalid()) {
+            throw new FileNotFoundException("Invalid file path");
+        }
         fd = new FileDescriptor();
         fd.attach(this);
         open(name);

src/share/classes/java/io/FileOutputStream.java

 /*
- * Copyright (c) 1994, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -196,6 +196,9 @@ class FileOutputStream extends OutputStream
         if (name == null) {
             throw new NullPointerException();
         }
+        if (file.isInvalid()) {
+            throw new FileNotFoundException("Invalid file path");
+        }
         this.fd = new FileDescriptor();
         fd.attach(this);
         this.append = append;

src/share/classes/java/io/RandomAccessFile.java

 /*
- * Copyright (c) 1994, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -228,6 +228,9 @@ public class RandomAccessFile implements DataOutput, DataInput, Closeable {
         if (name == null) {
             throw new NullPointerException();
         }
+        if (file.isInvalid()) {
+            throw new FileNotFoundException("Invalid file path");
+        }
         fd = new FileDescriptor();
         fd.attach(this);
         open(name, imode);