8168628: (fc) SIGBUS when extending file size to map it

8171452: (ch) linux io_util_md: Operation not supported exception after 8168628
Reviewed-by: alanb, bpb

make/mapfiles/libnio/mapfile-linux

 #
-# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,7 @@ SUNWprivate_1.1 {
                 Java_sun_nio_ch_FileChannelImpl_position0;
                 Java_sun_nio_ch_FileChannelImpl_transferTo0;
                 Java_sun_nio_ch_FileChannelImpl_unmap0;
+		Java_sun_nio_ch_FileDispatcherImpl_allocate0;
                 Java_sun_nio_ch_FileDispatcherImpl_close0;
                 Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
 		Java_sun_nio_ch_FileDispatcherImpl_force0;

make/mapfiles/libnio/mapfile-macosx

 #
-# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@ SUNWprivate_1.1 {
                 Java_sun_nio_ch_FileChannelImpl_position0;
                 Java_sun_nio_ch_FileChannelImpl_transferTo0;
                 Java_sun_nio_ch_FileChannelImpl_unmap0;
+		Java_sun_nio_ch_FileDispatcherImpl_allocate0;
                 Java_sun_nio_ch_FileDispatcherImpl_close0;
                 Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
 		Java_sun_nio_ch_FileDispatcherImpl_force0;

make/mapfiles/libnio/mapfile-solaris

 #
-# Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -47,6 +47,7 @@ SUNWprivate_1.1 {
                 Java_sun_nio_ch_FileChannelImpl_position0;
                 Java_sun_nio_ch_FileChannelImpl_transferTo0;
                 Java_sun_nio_ch_FileChannelImpl_unmap0;
+		Java_sun_nio_ch_FileDispatcherImpl_allocate0;
                 Java_sun_nio_ch_FileDispatcherImpl_close0;
                 Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
 		Java_sun_nio_ch_FileDispatcherImpl_force0;

src/share/classes/sun/nio/ch/FileChannelImpl.java

 /*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -889,57 +889,62 @@ public class FileChannelImpl
             if (!isOpen())
                 return null;
 
-            long filesize;
-            do {
-                filesize = nd.size(fd);
-            } while ((filesize == IOStatus.INTERRUPTED) && isOpen());
-            if (!isOpen())
-                return null;
-
-            if (filesize < position + size) { // Extend file size
-                if (!writable) {
-                    throw new IOException("Channel not open for writing " +
-                        "- cannot extend file to required size");
-                }
-                int rv;
+            long mapSize;
+            int pagePosition;
+            synchronized (positionLock) {
+                long filesize;
                 do {
-                    rv = nd.truncate(fd, position + size);
-                } while ((rv == IOStatus.INTERRUPTED) && isOpen());
+                    filesize = nd.size(fd);
+                } while ((filesize == IOStatus.INTERRUPTED) && isOpen());
                 if (!isOpen())
                     return null;
-            }
-            if (size == 0) {
-                addr = 0;
-                // a valid file descriptor is not required
-                FileDescriptor dummy = new FileDescriptor();
-                if ((!writable) || (imode == MAP_RO))
-                    return Util.newMappedByteBufferR(0, 0, dummy, null);
-                else
-                    return Util.newMappedByteBuffer(0, 0, dummy, null);
-            }
 
-            int pagePosition = (int)(position % allocationGranularity);
-            long mapPosition = position - pagePosition;
-            long mapSize = size + pagePosition;
-            try {
-                // If no exception was thrown from map0, the address is valid
-                addr = map0(imode, mapPosition, mapSize);
-            } catch (OutOfMemoryError x) {
-                // An OutOfMemoryError may indicate that we've exhausted memory
-                // so force gc and re-attempt map
-                System.gc();
-                try {
-                    Thread.sleep(100);
-                } catch (InterruptedException y) {
-                    Thread.currentThread().interrupt();
+                if (filesize < position + size) { // Extend file size
+                    if (!writable) {
+                        throw new IOException("Channel not open for writing " +
+                            "- cannot extend file to required size");
+                    }
+                    int rv;
+                    do {
+                        rv = nd.allocate(fd, position + size);
+                    } while ((rv == IOStatus.INTERRUPTED) && isOpen());
+                    if (!isOpen())
+                        return null;
+                }
+
+                if (size == 0) {
+                    addr = 0;
+                    // a valid file descriptor is not required
+                    FileDescriptor dummy = new FileDescriptor();
+                    if ((!writable) || (imode == MAP_RO))
+                        return Util.newMappedByteBufferR(0, 0, dummy, null);
+                    else
+                        return Util.newMappedByteBuffer(0, 0, dummy, null);
                 }
+
+                pagePosition = (int)(position % allocationGranularity);
+                long mapPosition = position - pagePosition;
+                mapSize = size + pagePosition;
                 try {
+                    // If map0 did not throw an exception, the address is valid
                     addr = map0(imode, mapPosition, mapSize);
-                } catch (OutOfMemoryError y) {
-                    // After a second OOME, fail
-                    throw new IOException("Map failed", y);
+                } catch (OutOfMemoryError x) {
+                    // An OutOfMemoryError may indicate that we've exhausted
+                    // memory so force gc and re-attempt map
+                    System.gc();
+                    try {
+                        Thread.sleep(100);
+                    } catch (InterruptedException y) {
+                        Thread.currentThread().interrupt();
+                    }
+                    try {
+                        addr = map0(imode, mapPosition, mapSize);
+                    } catch (OutOfMemoryError y) {
+                        // After a second OOME, fail
+                        throw new IOException("Map failed", y);
+                    }
                 }
-            }
+            } // synchronized
 
             // On Windows, and potentially other platforms, we need an open
             // file descriptor for some mapping operations.

src/share/classes/sun/nio/ch/FileDispatcher.java

 /*
- * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,8 @@ abstract class FileDispatcher extends NativeDispatcher {
 
     abstract int truncate(FileDescriptor fd, long size) throws IOException;
 
+    abstract int allocate(FileDescriptor fd, long size) throws IOException;
+
     abstract long size(FileDescriptor fd) throws IOException;
 
     abstract int lock(FileDescriptor fd, boolean blocking, long pos, long size,

src/solaris/classes/sun/nio/ch/FileDispatcherImpl.java

 /*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -80,6 +80,10 @@ class FileDispatcherImpl extends FileDispatcher {
         return truncate0(fd, size);
     }
 
+    int allocate(FileDescriptor fd, long size) throws IOException {
+        return allocate0(fd, size);
+    }
+
     long size(FileDescriptor fd) throws IOException {
         return size0(fd);
     }
@@ -142,6 +146,9 @@ class FileDispatcherImpl extends FileDispatcher {
     static native int truncate0(FileDescriptor fd, long size)
         throws IOException;
 
+    static native int allocate0(FileDescriptor fd, long size)
+        throws IOException;
+
     static native long size0(FileDescriptor fd) throws IOException;
 
     static native int lock0(FileDescriptor fd, boolean blocking, long pos,

src/solaris/native/java/io/io_util_md.c

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,10 @@
  * questions.
  */
 
+#if defined(__linux__)
+#define _FILE_OFFSET_BITS 64
+#endif
+
 #include "jni.h"
 #include "jni_util.h"
 #include "jvm.h"
@@ -208,6 +212,25 @@ jint
 handleSetLength(FD fd, jlong length)
 {
     int result;
+#if defined(__linux__)
+    /*
+     * On Linux, if the file size is being increased, then ftruncate64()
+     * will modify the metadata value of the size without actually allocating
+     * any blocks which can cause a SIGBUS error if the file is subsequently
+     * memory-mapped.
+     */
+    struct stat64 sb;
+
+    if (fstat64(fd, &sb) == 0 && length > sb.st_blocks*512) {
+        RESTARTABLE(posix_fallocate(fd, 0, length), result);
+        // Return on success or if errno is neither EOPNOTSUPP nor ENOSYS
+        if (result == 0) {
+            return 0;
+        } else if (errno != EOPNOTSUPP && errno != ENOSYS) {
+            return result;
+        }
+    }
+#endif
     RESTARTABLE(ftruncate64(fd, length), result);
     return result;
 }

src/solaris/native/sun/nio/ch/FileDispatcherImpl.c

 /*
- * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,10 @@
  * questions.
  */
 
+#if defined(__linux__)
+#define _FILE_OFFSET_BITS 64
+#endif
+
 #include "jni.h"
 #include "jni_util.h"
 #include "jvm.h"
@@ -178,6 +182,30 @@ Java_sun_nio_ch_FileDispatcherImpl_truncate0(JNIEnv *env, jobject this,
                   "Truncation failed");
 }
 
+JNIEXPORT jint JNICALL
+Java_sun_nio_ch_FileDispatcherImpl_allocate0(JNIEnv *env, jobject this,
+                                             jobject fdo, jlong size)
+{
+#if defined(__linux__)
+    /*
+     * On Linux, if the file size is being increased, then ftruncate64()
+     * will modify the metadata value of the size without actually allocating
+     * any blocks which can cause a SIGBUS error if the file is subsequently
+     * memory-mapped.
+     */
+    // Return on success or if errno is neither EOPNOTSUPP nor ENOSYS
+    int result = posix_fallocate(fdval(env, fdo), 0, size);
+    if (result == 0) {
+        return 0;
+    } else if (errno != EOPNOTSUPP && errno != ENOSYS) {
+        return handle(env, result, "Allocation failed");
+    }
+#endif
+    return handle(env,
+                  ftruncate64(fdval(env, fdo), size),
+                  "Truncation failed");
+}
+
 JNIEXPORT jlong JNICALL
 Java_sun_nio_ch_FileDispatcherImpl_size0(JNIEnv *env, jobject this, jobject fdo)
 {

src/windows/classes/sun/nio/ch/FileDispatcherImpl.java

 /*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -93,6 +93,11 @@ class FileDispatcherImpl extends FileDispatcher {
         return truncate0(fd, size);
     }
 
+    int allocate(FileDescriptor fd, long size) throws IOException {
+        // truncate0() works for extending and truncating file size
+        return truncate0(fd, size);
+    }
+
     long size(FileDescriptor fd) throws IOException {
         return size0(fd);
     }

test/java/nio/channels/FileChannel/FileExtensionAndMap.java

+/*
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @ignore This test has huge disk space requirements
+ * @bug 8168628
+ * @summary Test extending files to very large sizes without hitting a SIGBUS
+ * @requires (os.family == "linux")
+ * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap
+ * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap true
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Semaphore;
+import java.util.stream.IntStream;
+
+public class FileExtensionAndMap {
+
+    private static final ExecutorService CACHED_EXECUTORSERVICE =
+        Executors.newCachedThreadPool();
+
+    private static final String TMPDIR = System.getProperty("test.dir", ".");
+
+    private static boolean useRaf = false;
+
+    public static void main(String args[]) throws Exception {
+        if (args.length > 2) {
+            throw new IllegalArgumentException
+                ("Arguments: [true|false [targetFolder]]");
+        }
+
+        String defaultFolder = TMPDIR + File.separator + "target";
+        if (args.length > 0) {
+            useRaf = Boolean.valueOf(args[0]);
+            if (args.length > 1) {
+                defaultFolder = args[1];
+            }
+        }
+        final String targetFolder = defaultFolder;
+        Path p = Paths.get(targetFolder);
+        boolean targetExists = Files.exists(p);
+        if (!targetExists) {
+            Files.createDirectory(p);
+        }
+
+        System.out.printf("Using RandomAccessFile: %s; target folder: %s%n",
+            useRaf, targetFolder);
+
+        ForkJoinPool fjPool = new ForkJoinPool(3);
+        fjPool.submit(() -> {
+            IntStream.range(0, 20).parallel().forEach((index) -> {
+                String fileName = "testBigFile_" + index + ".dat";
+                Path source = null;
+                Path target = null;
+                try {
+                    source = Paths.get(TMPDIR, fileName);
+                    testCreateBigFile(source);
+                    target = Paths.get(targetFolder, fileName);
+                    testFileCopy(source, target);
+                } catch (Throwable th) {
+                    System.err.println("Error copying file with fileName: "
+                        + fileName + " : " + th.getMessage());
+                    th.printStackTrace(System.err);
+                } finally {
+                    try {
+                        if (source != null) {
+                            Files.deleteIfExists(source);
+                        }
+                    } catch (Throwable ignored) {
+                    }
+                    try {
+                        if (target != null) {
+                            Files.deleteIfExists(target);
+                        }
+                    } catch (Throwable ignored) {
+                    }
+                }
+            });
+        }).join();
+
+        if (!targetExists) {
+            Files.delete(p);
+        }
+    }
+
+    private static void testFileCopy(Path source, Path target)
+        throws IOException {
+        Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+        System.out.println("Finished copying file with fileName: "
+                + source.getFileName());
+    }
+
+    private static void testCreateBigFile(Path segmentFile)
+        throws IOException {
+        final Semaphore concurrencySemaphore = new Semaphore(5);
+        long fileSize = 3L * 1024L * 1024L * 1024L;
+        int blockSize = 10 * 1024 * 1024;
+        int loopCount = (int) Math.floorDiv(fileSize, blockSize);
+
+        String fileName = segmentFile.getFileName().toString();
+        if (useRaf) {
+            try (RandomAccessFile raf
+                = new RandomAccessFile(segmentFile.toFile(), "rw")) {
+                raf.setLength(fileSize);
+                try (FileChannel fc = raf.getChannel()) {
+                    for (int i = 0; i < loopCount; i++) {
+                        final long startPosition = 1L * blockSize * i;
+                        concurrencySemaphore.acquireUninterruptibly();
+                        CACHED_EXECUTORSERVICE.submit(() -> {
+                            writeTemplateData(fileName, fc, startPosition,
+                                    blockSize, concurrencySemaphore);
+                        });
+                    }
+                } finally {
+                    concurrencySemaphore.acquireUninterruptibly(5);
+                }
+            }
+        } else {
+            Path file = Files.createFile(segmentFile);
+            try (FileChannel fc = FileChannel.open(file,
+                StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+                for (int i = 0; i < loopCount; i++) {
+                    final long startPosition = 1L * blockSize * i;
+                    concurrencySemaphore.acquireUninterruptibly();
+                    CACHED_EXECUTORSERVICE.submit(() -> {
+                        writeTemplateData(fileName, fc, startPosition,
+                                blockSize, concurrencySemaphore);
+                    });
+                }
+            }
+        }
+    }
+
+    private static void writeTemplateData(String fileName,
+        FileChannel fc, long startPosition, int blockSize,
+        Semaphore concurrencySemaphore) {
+        try {
+            byte[] EMPTY_RECORD = new byte[blockSize / 256];
+
+            MappedByteBuffer mappedByteBuffer = fc.map(MapMode.READ_WRITE,
+                startPosition, blockSize);
+            IntStream.range(0, 256).forEach((recordIndex) -> {
+                try {
+                    mappedByteBuffer.position((int) (recordIndex *
+                        EMPTY_RECORD.length));
+                    mappedByteBuffer.put(EMPTY_RECORD, 0, EMPTY_RECORD.length);
+                } catch (Throwable th) {
+                    System.err.println
+                        ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: "
+                        + fileName + ", startPosition: " + startPosition + ", recordIndex: "
+                        + recordIndex + " : " + th.getMessage());
+                    th.printStackTrace(System.err);
+                }
+            });
+
+            mappedByteBuffer.force();
+        } catch (Throwable th) {
+            if (!(th instanceof ClosedChannelException)) {
+                System.err.println
+                    ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: "
+                    + fileName + ", startPosition: " + startPosition + " : "
+                    + th.getMessage());
+                th.printStackTrace(System.err);
+            }
+        } finally {
+            concurrencySemaphore.release();
+        }
+    }
+}