xfs_file.c 42.2 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_fs.h"
20
#include "xfs_shared.h"
21
#include "xfs_format.h"
22 23
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
L
Linus Torvalds 已提交
24
#include "xfs_mount.h"
25 26
#include "xfs_da_format.h"
#include "xfs_da_btree.h"
L
Linus Torvalds 已提交
27
#include "xfs_inode.h"
28
#include "xfs_trans.h"
29
#include "xfs_inode_item.h"
30
#include "xfs_bmap.h"
D
Dave Chinner 已提交
31
#include "xfs_bmap_util.h"
L
Linus Torvalds 已提交
32
#include "xfs_error.h"
33
#include "xfs_dir2.h"
D
Dave Chinner 已提交
34
#include "xfs_dir2_priv.h"
35
#include "xfs_ioctl.h"
36
#include "xfs_trace.h"
37
#include "xfs_log.h"
38
#include "xfs_icache.h"
39
#include "xfs_pnfs.h"
L
Linus Torvalds 已提交
40 41

#include <linux/dcache.h>
42
#include <linux/falloc.h>
43
#include <linux/pagevec.h>
44
#include <linux/backing-dev.h>
L
Linus Torvalds 已提交
45

46
static const struct vm_operations_struct xfs_file_vm_ops;
L
Linus Torvalds 已提交
47

48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
/*
 * Locking primitives for read and write IO paths to ensure we consistently use
 * and order the inode->i_mutex, ip->i_lock and ip->i_iolock.
 */
static inline void
xfs_rw_ilock(
	struct xfs_inode	*ip,
	int			type)
{
	if (type & XFS_IOLOCK_EXCL)
		mutex_lock(&VFS_I(ip)->i_mutex);
	xfs_ilock(ip, type);
}

static inline void
xfs_rw_iunlock(
	struct xfs_inode	*ip,
	int			type)
{
	xfs_iunlock(ip, type);
	if (type & XFS_IOLOCK_EXCL)
		mutex_unlock(&VFS_I(ip)->i_mutex);
}

static inline void
xfs_rw_ilock_demote(
	struct xfs_inode	*ip,
	int			type)
{
	xfs_ilock_demote(ip, type);
	if (type & XFS_IOLOCK_EXCL)
		mutex_unlock(&VFS_I(ip)->i_mutex);
}

82
/*
83 84 85 86 87
 * xfs_iozero clears the specified range supplied via the page cache (except in
 * the DAX case). Writes through the page cache will allocate blocks over holes,
 * though the callers usually map the holes first and avoid them. If a block is
 * not completely zeroed, then it will be read from disk before being partially
 * zeroed.
88
 *
89 90 91
 * In the DAX case, we can just directly write to the underlying pages. This
 * will not allocate blocks, but will avoid holes and unwritten extents and so
 * not do unnecessary work.
92
 */
93
int
94 95 96 97 98 99 100
xfs_iozero(
	struct xfs_inode	*ip,	/* inode			*/
	loff_t			pos,	/* offset in file		*/
	size_t			count)	/* size of data to zero		*/
{
	struct page		*page;
	struct address_space	*mapping;
101 102
	int			status = 0;

103 104 105 106 107 108 109 110 111 112 113

	mapping = VFS_I(ip)->i_mapping;
	do {
		unsigned offset, bytes;
		void *fsdata;

		offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
		bytes = PAGE_CACHE_SIZE - offset;
		if (bytes > count)
			bytes = count;

114 115 116 117 118 119 120 121 122 123 124
		if (IS_DAX(VFS_I(ip))) {
			status = dax_zero_page_range(VFS_I(ip), pos, bytes,
						     xfs_get_blocks_direct);
			if (status)
				break;
		} else {
			status = pagecache_write_begin(NULL, mapping, pos, bytes,
						AOP_FLAG_UNINTERRUPTIBLE,
						&page, &fsdata);
			if (status)
				break;
125

126
			zero_user(page, offset, bytes);
127

128 129 130 131 132
			status = pagecache_write_end(NULL, mapping, pos, bytes,
						bytes, page, fsdata);
			WARN_ON(status <= 0); /* can't return less than zero! */
			status = 0;
		}
133 134 135 136
		pos += bytes;
		count -= bytes;
	} while (count);

137
	return status;
138 139
}

140 141 142 143 144 145 146 147 148 149 150
int
xfs_update_prealloc_flags(
	struct xfs_inode	*ip,
	enum xfs_prealloc_flags	flags)
{
	struct xfs_trans	*tp;
	int			error;

	tp = xfs_trans_alloc(ip->i_mount, XFS_TRANS_WRITEID);
	error = xfs_trans_reserve(tp, &M_RES(ip->i_mount)->tr_writeid, 0, 0);
	if (error) {
151
		xfs_trans_cancel(tp);
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
		return error;
	}

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);

	if (!(flags & XFS_PREALLOC_INVISIBLE)) {
		ip->i_d.di_mode &= ~S_ISUID;
		if (ip->i_d.di_mode & S_IXGRP)
			ip->i_d.di_mode &= ~S_ISGID;
		xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
	}

	if (flags & XFS_PREALLOC_SET)
		ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
	if (flags & XFS_PREALLOC_CLEAR)
		ip->i_d.di_flags &= ~XFS_DIFLAG_PREALLOC;

	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
	if (flags & XFS_PREALLOC_SYNC)
		xfs_trans_set_sync(tp);
173
	return xfs_trans_commit(tp);
174 175
}

176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
/*
 * Fsync operations on directories are much simpler than on regular files,
 * as there is no file data to flush, and thus also no need for explicit
 * cache flush operations, and there are no non-transaction metadata updates
 * on directories either.
 */
STATIC int
xfs_dir_fsync(
	struct file		*file,
	loff_t			start,
	loff_t			end,
	int			datasync)
{
	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_lsn_t		lsn = 0;

	trace_xfs_dir_fsync(ip);

	xfs_ilock(ip, XFS_ILOCK_SHARED);
	if (xfs_ipincount(ip))
		lsn = ip->i_itemp->ili_last_lsn;
	xfs_iunlock(ip, XFS_ILOCK_SHARED);

	if (!lsn)
		return 0;
D
Dave Chinner 已提交
202
	return _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, NULL);
203 204
}

205 206 207
STATIC int
xfs_file_fsync(
	struct file		*file,
208 209
	loff_t			start,
	loff_t			end,
210 211
	int			datasync)
{
212 213
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
214
	struct xfs_mount	*mp = ip->i_mount;
215 216
	int			error = 0;
	int			log_flushed = 0;
217
	xfs_lsn_t		lsn = 0;
218

C
Christoph Hellwig 已提交
219
	trace_xfs_file_fsync(ip);
220

221 222 223 224
	error = filemap_write_and_wait_range(inode->i_mapping, start, end);
	if (error)
		return error;

225
	if (XFS_FORCED_SHUTDOWN(mp))
E
Eric Sandeen 已提交
226
		return -EIO;
227 228 229

	xfs_iflags_clear(ip, XFS_ITRUNCATED);

230 231 232 233 234 235 236 237 238 239 240 241 242 243
	if (mp->m_flags & XFS_MOUNT_BARRIER) {
		/*
		 * If we have an RT and/or log subvolume we need to make sure
		 * to flush the write cache the device used for file data
		 * first.  This is to ensure newly written file data make
		 * it to disk before logging the new inode size in case of
		 * an extending write.
		 */
		if (XFS_IS_REALTIME_INODE(ip))
			xfs_blkdev_issue_flush(mp->m_rtdev_targp);
		else if (mp->m_logdev_targp != mp->m_ddev_targp)
			xfs_blkdev_issue_flush(mp->m_ddev_targp);
	}

244
	/*
C
Christoph Hellwig 已提交
245 246
	 * All metadata updates are logged, which means that we just have
	 * to flush the log up to the latest LSN that touched the inode.
247 248
	 */
	xfs_ilock(ip, XFS_ILOCK_SHARED);
249 250 251 252 253
	if (xfs_ipincount(ip)) {
		if (!datasync ||
		    (ip->i_itemp->ili_fields & ~XFS_ILOG_TIMESTAMP))
			lsn = ip->i_itemp->ili_last_lsn;
	}
C
Christoph Hellwig 已提交
254
	xfs_iunlock(ip, XFS_ILOCK_SHARED);
255

C
Christoph Hellwig 已提交
256
	if (lsn)
257 258
		error = _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);

259 260 261 262 263 264 265 266 267 268 269 270
	/*
	 * If we only have a single device, and the log force about was
	 * a no-op we might have to flush the data device cache here.
	 * This can only happen for fdatasync/O_DSYNC if we were overwriting
	 * an already allocated file and thus do not have any metadata to
	 * commit.
	 */
	if ((mp->m_flags & XFS_MOUNT_BARRIER) &&
	    mp->m_logdev_targp == mp->m_ddev_targp &&
	    !XFS_IS_REALTIME_INODE(ip) &&
	    !log_flushed)
		xfs_blkdev_issue_flush(mp->m_ddev_targp);
271

D
Dave Chinner 已提交
272
	return error;
273 274
}

275
STATIC ssize_t
A
Al Viro 已提交
276
xfs_file_read_iter(
277
	struct kiocb		*iocb,
A
Al Viro 已提交
278
	struct iov_iter		*to)
279 280 281
{
	struct file		*file = iocb->ki_filp;
	struct inode		*inode = file->f_mapping->host;
282 283
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
A
Al Viro 已提交
284
	size_t			size = iov_iter_count(to);
285
	ssize_t			ret = 0;
286
	int			ioflags = 0;
287
	xfs_fsize_t		n;
A
Al Viro 已提交
288
	loff_t			pos = iocb->ki_pos;
289 290 291

	XFS_STATS_INC(xs_read_calls);

292
	if (unlikely(iocb->ki_flags & IOCB_DIRECT))
D
Dave Chinner 已提交
293
		ioflags |= XFS_IO_ISDIRECT;
294
	if (file->f_mode & FMODE_NOCMTIME)
D
Dave Chinner 已提交
295
		ioflags |= XFS_IO_INVIS;
296

297
	if ((ioflags & XFS_IO_ISDIRECT) && !IS_DAX(inode)) {
298 299 300
		xfs_buftarg_t	*target =
			XFS_IS_REALTIME_INODE(ip) ?
				mp->m_rtdev_targp : mp->m_ddev_targp;
301 302
		/* DIO must be aligned to device logical sector size */
		if ((pos | size) & target->bt_logical_sectormask) {
D
Dave Chinner 已提交
303
			if (pos == i_size_read(inode))
304
				return 0;
E
Eric Sandeen 已提交
305
			return -EINVAL;
306 307 308
		}
	}

D
Dave Chinner 已提交
309
	n = mp->m_super->s_maxbytes - pos;
310
	if (n <= 0 || size == 0)
311 312 313 314 315 316 317 318
		return 0;

	if (n < size)
		size = n;

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

319
	/*
320 321 322 323 324 325 326 327
	 * Locking is a bit tricky here. If we take an exclusive lock for direct
	 * IO, we effectively serialise all new concurrent read IO to this file
	 * and block it behind IO that is currently in progress because IO in
	 * progress holds the IO lock shared. We only need to hold the lock
	 * exclusive to blow away the page cache, so only take lock exclusively
	 * if the page cache needs invalidation. This allows the normal direct
	 * IO case of no page cache pages to proceeed concurrently without
	 * serialisation.
328 329
	 */
	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
D
Dave Chinner 已提交
330
	if ((ioflags & XFS_IO_ISDIRECT) && inode->i_mapping->nrpages) {
331
		xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
332 333
		xfs_rw_ilock(ip, XFS_IOLOCK_EXCL);

334 335 336 337 338 339 340 341 342 343 344
		/*
		 * The generic dio code only flushes the range of the particular
		 * I/O. Because we take an exclusive lock here, this whole
		 * sequence is considerably more expensive for us. This has a
		 * noticeable performance impact for any file with cached pages,
		 * even when outside of the range of the particular I/O.
		 *
		 * Hence, amortize the cost of the lock against a full file
		 * flush and reduce the chances of repeated iolock cycles going
		 * forward.
		 */
345
		if (inode->i_mapping->nrpages) {
346
			ret = filemap_write_and_wait(VFS_I(ip)->i_mapping);
347 348 349 350
			if (ret) {
				xfs_rw_iunlock(ip, XFS_IOLOCK_EXCL);
				return ret;
			}
351 352 353 354 355 356

			/*
			 * Invalidate whole pages. This can return an error if
			 * we fail to invalidate a page, but this should never
			 * happen on XFS. Warn if it does fail.
			 */
357
			ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping);
358 359
			WARN_ON_ONCE(ret);
			ret = 0;
360
		}
361
		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
362
	}
363

D
Dave Chinner 已提交
364
	trace_xfs_file_read(ip, size, pos, ioflags);
365

A
Al Viro 已提交
366
	ret = generic_file_read_iter(iocb, to);
367 368 369
	if (ret > 0)
		XFS_STATS_ADD(xs_read_bytes, ret);

370
	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
371 372 373
	return ret;
}

374 375
STATIC ssize_t
xfs_file_splice_read(
376 377 378 379
	struct file		*infilp,
	loff_t			*ppos,
	struct pipe_inode_info	*pipe,
	size_t			count,
380
	unsigned int		flags)
381
{
382 383
	struct xfs_inode	*ip = XFS_I(infilp->f_mapping->host);
	int			ioflags = 0;
384 385 386
	ssize_t			ret;

	XFS_STATS_INC(xs_read_calls);
387 388

	if (infilp->f_mode & FMODE_NOCMTIME)
D
Dave Chinner 已提交
389
		ioflags |= XFS_IO_INVIS;
390

391 392 393
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
		return -EIO;

394
	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
395 396 397

	trace_xfs_file_splice_read(ip, count, *ppos, ioflags);

398 399 400 401 402
	/* for dax, we need to avoid the page cache */
	if (IS_DAX(VFS_I(ip)))
		ret = default_file_splice_read(infilp, ppos, pipe, count, flags);
	else
		ret = generic_file_splice_read(infilp, ppos, pipe, count, flags);
403 404 405
	if (ret > 0)
		XFS_STATS_ADD(xs_read_bytes, ret);

406
	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
407 408 409 410
	return ret;
}

/*
411 412 413 414
 * This routine is called to handle zeroing any space in the last block of the
 * file that is beyond the EOF.  We do this since the size is being increased
 * without writing anything to that block and we don't want to read the
 * garbage on the disk.
415 416 417
 */
STATIC int				/* error (positive) */
xfs_zero_last_block(
418 419
	struct xfs_inode	*ip,
	xfs_fsize_t		offset,
420 421
	xfs_fsize_t		isize,
	bool			*did_zeroing)
422
{
423 424 425 426 427 428 429
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		last_fsb = XFS_B_TO_FSBT(mp, isize);
	int			zero_offset = XFS_B_FSB_OFFSET(mp, isize);
	int			zero_len;
	int			nimaps = 1;
	int			error = 0;
	struct xfs_bmbt_irec	imap;
430

431
	xfs_ilock(ip, XFS_ILOCK_EXCL);
D
Dave Chinner 已提交
432
	error = xfs_bmapi_read(ip, last_fsb, 1, &imap, &nimaps, 0);
433
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
D
Dave Chinner 已提交
434
	if (error)
435
		return error;
436

437
	ASSERT(nimaps > 0);
438

439 440 441 442
	/*
	 * If the block underlying isize is just a hole, then there
	 * is nothing to zero.
	 */
443
	if (imap.br_startblock == HOLESTARTBLOCK)
444 445 446 447 448
		return 0;

	zero_len = mp->m_sb.sb_blocksize - zero_offset;
	if (isize + zero_len > offset)
		zero_len = offset - isize;
449
	*did_zeroing = true;
450
	return xfs_iozero(ip, isize, zero_len);
451 452 453
}

/*
454 455 456 457 458 459 460 461 462
 * Zero any on disk space between the current EOF and the new, larger EOF.
 *
 * This handles the normal case of zeroing the remainder of the last block in
 * the file and the unusual case of zeroing blocks out beyond the size of the
 * file.  This second case only happens with fixed size extents and when the
 * system crashes before the inode size was updated but after blocks were
 * allocated.
 *
 * Expects the iolock to be held exclusive, and will take the ilock internally.
463 464 465
 */
int					/* error (positive) */
xfs_zero_eof(
466 467
	struct xfs_inode	*ip,
	xfs_off_t		offset,		/* starting I/O offset */
468 469
	xfs_fsize_t		isize,		/* current inode size */
	bool			*did_zeroing)
470
{
471 472 473 474 475 476 477 478 479 480 481 482
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		start_zero_fsb;
	xfs_fileoff_t		end_zero_fsb;
	xfs_fileoff_t		zero_count_fsb;
	xfs_fileoff_t		last_fsb;
	xfs_fileoff_t		zero_off;
	xfs_fsize_t		zero_len;
	int			nimaps;
	int			error = 0;
	struct xfs_bmbt_irec	imap;

	ASSERT(xfs_isilocked(ip, XFS_IOLOCK_EXCL));
483 484
	ASSERT(offset > isize);

485 486
	trace_xfs_zero_eof(ip, isize, offset - isize);

487 488
	/*
	 * First handle zeroing the block on which isize resides.
489
	 *
490 491
	 * We only zero a part of that block so it is handled specially.
	 */
492
	if (XFS_B_FSB_OFFSET(mp, isize) != 0) {
493
		error = xfs_zero_last_block(ip, offset, isize, did_zeroing);
494 495
		if (error)
			return error;
496 497 498
	}

	/*
499 500 501 502 503 504 505
	 * Calculate the range between the new size and the old where blocks
	 * needing to be zeroed may exist.
	 *
	 * To get the block where the last byte in the file currently resides,
	 * we need to subtract one from the size and truncate back to a block
	 * boundary.  We subtract 1 in case the size is exactly on a block
	 * boundary.
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
	 */
	last_fsb = isize ? XFS_B_TO_FSBT(mp, isize - 1) : (xfs_fileoff_t)-1;
	start_zero_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)isize);
	end_zero_fsb = XFS_B_TO_FSBT(mp, offset - 1);
	ASSERT((xfs_sfiloff_t)last_fsb < (xfs_sfiloff_t)start_zero_fsb);
	if (last_fsb == end_zero_fsb) {
		/*
		 * The size was only incremented on its last block.
		 * We took care of that above, so just return.
		 */
		return 0;
	}

	ASSERT(start_zero_fsb <= end_zero_fsb);
	while (start_zero_fsb <= end_zero_fsb) {
		nimaps = 1;
		zero_count_fsb = end_zero_fsb - start_zero_fsb + 1;
523 524

		xfs_ilock(ip, XFS_ILOCK_EXCL);
D
Dave Chinner 已提交
525 526
		error = xfs_bmapi_read(ip, start_zero_fsb, zero_count_fsb,
					  &imap, &nimaps, 0);
527 528
		xfs_iunlock(ip, XFS_ILOCK_EXCL);
		if (error)
529
			return error;
530

531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
		ASSERT(nimaps > 0);

		if (imap.br_state == XFS_EXT_UNWRITTEN ||
		    imap.br_startblock == HOLESTARTBLOCK) {
			start_zero_fsb = imap.br_startoff + imap.br_blockcount;
			ASSERT(start_zero_fsb <= (end_zero_fsb + 1));
			continue;
		}

		/*
		 * There are blocks we need to zero.
		 */
		zero_off = XFS_FSB_TO_B(mp, start_zero_fsb);
		zero_len = XFS_FSB_TO_B(mp, imap.br_blockcount);

		if ((zero_off + zero_len) > offset)
			zero_len = offset - zero_off;

		error = xfs_iozero(ip, zero_off, zero_len);
550 551
		if (error)
			return error;
552

553
		*did_zeroing = true;
554 555 556 557 558 559 560
		start_zero_fsb = imap.br_startoff + imap.br_blockcount;
		ASSERT(start_zero_fsb <= (end_zero_fsb + 1));
	}

	return 0;
}

561 562 563
/*
 * Common pre-write limit and setup checks.
 *
564 565 566
 * Called with the iolocked held either shared and exclusive according to
 * @iolock, and returns with it held.  Might upgrade the iolock to exclusive
 * if called for a direct write beyond i_size.
567 568 569
 */
STATIC ssize_t
xfs_file_aio_write_checks(
570 571
	struct kiocb		*iocb,
	struct iov_iter		*from,
572 573
	int			*iolock)
{
574
	struct file		*file = iocb->ki_filp;
575 576
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
577
	ssize_t			error = 0;
578
	size_t			count = iov_iter_count(from);
579
	bool			drained_dio = false;
580

581
restart:
582 583
	error = generic_write_checks(iocb, from);
	if (error <= 0)
584 585
		return error;

586
	error = xfs_break_layouts(inode, iolock, true);
587 588 589
	if (error)
		return error;

590 591 592 593 594 595 596
	/* For changing security info in file_remove_privs() we need i_mutex */
	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
		xfs_rw_iunlock(ip, *iolock);
		*iolock = XFS_IOLOCK_EXCL;
		xfs_rw_ilock(ip, *iolock);
		goto restart;
	}
597 598 599
	/*
	 * If the offset is beyond the size of the file, we need to zero any
	 * blocks that fall between the existing EOF and the start of this
600
	 * write.  If zeroing is needed and we are currently holding the
601 602
	 * iolock shared, we need to update it to exclusive which implies
	 * having to redo all checks before.
603 604 605 606 607 608 609 610
	 *
	 * We need to serialise against EOF updates that occur in IO
	 * completions here. We want to make sure that nobody is changing the
	 * size while we do this check until we have placed an IO barrier (i.e.
	 * hold the XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.
	 * The spinlock effectively forms a memory barrier once we have the
	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value
	 * and hence be able to correctly determine if we need to run zeroing.
611
	 */
612
	spin_lock(&ip->i_flags_lock);
613
	if (iocb->ki_pos > i_size_read(inode)) {
614 615
		bool	zero = false;

616
		spin_unlock(&ip->i_flags_lock);
617 618 619 620 621 622 623
		if (!drained_dio) {
			if (*iolock == XFS_IOLOCK_SHARED) {
				xfs_rw_iunlock(ip, *iolock);
				*iolock = XFS_IOLOCK_EXCL;
				xfs_rw_ilock(ip, *iolock);
				iov_iter_reexpand(from, count);
			}
624 625 626 627 628 629 630 631 632
			/*
			 * We now have an IO submission barrier in place, but
			 * AIO can do EOF updates during IO completion and hence
			 * we now need to wait for all of them to drain. Non-AIO
			 * DIO will have drained before we are given the
			 * XFS_IOLOCK_EXCL, and so for most cases this wait is a
			 * no-op.
			 */
			inode_dio_wait(inode);
633
			drained_dio = true;
634 635
			goto restart;
		}
636
		error = xfs_zero_eof(ip, iocb->ki_pos, i_size_read(inode), &zero);
637 638
		if (error)
			return error;
639 640
	} else
		spin_unlock(&ip->i_flags_lock);
641

C
Christoph Hellwig 已提交
642 643 644 645 646 647
	/*
	 * Updating the timestamps will grab the ilock again from
	 * xfs_fs_dirty_inode, so we have to call it after dropping the
	 * lock above.  Eventually we should look into a way to avoid
	 * the pointless lock roundtrip.
	 */
648 649 650 651 652
	if (likely(!(file->f_mode & FMODE_NOCMTIME))) {
		error = file_update_time(file);
		if (error)
			return error;
	}
C
Christoph Hellwig 已提交
653

654 655 656 657 658
	/*
	 * If we're writing the file then make sure to clear the setuid and
	 * setgid bits if the process is not being run by root.  This keeps
	 * people from modifying setuid and setgid binaries.
	 */
659 660 661
	if (!IS_NOSEC(inode))
		return file_remove_privs(file);
	return 0;
662 663
}

664 665 666 667
/*
 * xfs_file_dio_aio_write - handle direct IO writes
 *
 * Lock the inode appropriately to prepare for and issue a direct IO write.
668
 * By separating it from the buffered write path we remove all the tricky to
669 670
 * follow locking changes and looping.
 *
671 672 673 674 675 676 677 678 679 680 681 682 683
 * If there are cached pages or we're extending the file, we need IOLOCK_EXCL
 * until we're sure the bytes at the new EOF have been zeroed and/or the cached
 * pages are flushed out.
 *
 * In most cases the direct IO writes will be done holding IOLOCK_SHARED
 * allowing them to be done in parallel with reads and other direct IO writes.
 * However, if the IO is not aligned to filesystem blocks, the direct IO layer
 * needs to do sub-block zeroing and that requires serialisation against other
 * direct IOs to the same block. In this case we need to serialise the
 * submission of the unaligned IOs so that we don't get racing block zeroing in
 * the dio layer.  To avoid the problem with aio, we also need to wait for
 * outstanding IOs to complete so that unwritten extent conversion is completed
 * before we try to map the overlapping block. This is currently implemented by
C
Christoph Hellwig 已提交
684
 * hitting it with a big hammer (i.e. inode_dio_wait()).
685
 *
686 687 688 689 690 691
 * Returns with locks held indicated by @iolock and errors indicated by
 * negative return values.
 */
STATIC ssize_t
xfs_file_dio_aio_write(
	struct kiocb		*iocb,
692
	struct iov_iter		*from)
693 694 695 696 697 698 699
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	ssize_t			ret = 0;
700
	int			unaligned_io = 0;
701
	int			iolock;
702 703
	size_t			count = iov_iter_count(from);
	loff_t			pos = iocb->ki_pos;
704 705
	loff_t			end;
	struct iov_iter		data;
706 707 708
	struct xfs_buftarg	*target = XFS_IS_REALTIME_INODE(ip) ?
					mp->m_rtdev_targp : mp->m_ddev_targp;

709
	/* DIO must be aligned to device logical sector size */
710
	if (!IS_DAX(inode) && ((pos | count) & target->bt_logical_sectormask))
E
Eric Sandeen 已提交
711
		return -EINVAL;
712

713
	/* "unaligned" here means not aligned to a filesystem block */
714 715 716
	if ((pos & mp->m_blockmask) || ((pos + count) & mp->m_blockmask))
		unaligned_io = 1;

717 718 719 720 721 722 723 724
	/*
	 * We don't need to take an exclusive lock unless there page cache needs
	 * to be invalidated or unaligned IO is being executed. We don't need to
	 * consider the EOF extension case here because
	 * xfs_file_aio_write_checks() will relock the inode as necessary for
	 * EOF zeroing cases and fill out the new inode size as appropriate.
	 */
	if (unaligned_io || mapping->nrpages)
725
		iolock = XFS_IOLOCK_EXCL;
726
	else
727 728
		iolock = XFS_IOLOCK_SHARED;
	xfs_rw_ilock(ip, iolock);
729 730 731 732 733 734

	/*
	 * Recheck if there are cached pages that need invalidate after we got
	 * the iolock to protect against other threads adding new pages while
	 * we were waiting for the iolock.
	 */
735 736 737 738
	if (mapping->nrpages && iolock == XFS_IOLOCK_SHARED) {
		xfs_rw_iunlock(ip, iolock);
		iolock = XFS_IOLOCK_EXCL;
		xfs_rw_ilock(ip, iolock);
739
	}
740

741
	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
742
	if (ret)
743
		goto out;
744 745
	count = iov_iter_count(from);
	pos = iocb->ki_pos;
746
	end = pos + count - 1;
747

748 749 750
	/*
	 * See xfs_file_read_iter() for why we do a full-file flush here.
	 */
751
	if (mapping->nrpages) {
752
		ret = filemap_write_and_wait(VFS_I(ip)->i_mapping);
753
		if (ret)
754
			goto out;
755
		/*
756 757 758
		 * Invalidate whole pages. This can return an error if we fail
		 * to invalidate a page, but this should never happen on XFS.
		 * Warn if it does fail.
759
		 */
760
		ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping);
761 762
		WARN_ON_ONCE(ret);
		ret = 0;
763 764
	}

765 766 767 768 769
	/*
	 * If we are doing unaligned IO, wait for all other IO to drain,
	 * otherwise demote the lock if we had to flush cached pages
	 */
	if (unaligned_io)
C
Christoph Hellwig 已提交
770
		inode_dio_wait(inode);
771
	else if (iolock == XFS_IOLOCK_EXCL) {
772
		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
773
		iolock = XFS_IOLOCK_SHARED;
774 775 776 777
	}

	trace_xfs_file_direct_write(ip, count, iocb->ki_pos, 0);

778
	data = *from;
779
	ret = mapping->a_ops->direct_IO(iocb, &data, pos);
780 781 782 783 784 785 786 787 788 789 790 791 792

	/* see generic_file_direct_write() for why this is necessary */
	if (mapping->nrpages) {
		invalidate_inode_pages2_range(mapping,
					      pos >> PAGE_CACHE_SHIFT,
					      end >> PAGE_CACHE_SHIFT);
	}

	if (ret > 0) {
		pos += ret;
		iov_iter_advance(from, ret);
		iocb->ki_pos = pos;
	}
793 794 795
out:
	xfs_rw_iunlock(ip, iolock);

796 797 798 799 800
	/*
	 * No fallback to buffered IO on errors for XFS. DAX can result in
	 * partial writes, but direct IO will either complete fully or fail.
	 */
	ASSERT(ret < 0 || ret == count || IS_DAX(VFS_I(ip)));
801 802 803
	return ret;
}

804
STATIC ssize_t
805
xfs_file_buffered_aio_write(
806
	struct kiocb		*iocb,
807
	struct iov_iter		*from)
808 809 810 811
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
812
	struct xfs_inode	*ip = XFS_I(inode);
813 814
	ssize_t			ret;
	int			enospc = 0;
815
	int			iolock = XFS_IOLOCK_EXCL;
816

817
	xfs_rw_ilock(ip, iolock);
818

819
	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
820
	if (ret)
821
		goto out;
822 823

	/* We can write back this queue in page reclaim */
824
	current->backing_dev_info = inode_to_bdi(inode);
825 826

write_retry:
827 828 829
	trace_xfs_file_buffered_write(ip, iov_iter_count(from),
				      iocb->ki_pos, 0);
	ret = generic_perform_write(file, from, iocb->ki_pos);
830
	if (likely(ret >= 0))
831
		iocb->ki_pos += ret;
832

833
	/*
834 835 836 837 838 839 840
	 * If we hit a space limit, try to free up some lingering preallocated
	 * space before returning an error. In the case of ENOSPC, first try to
	 * write back all dirty inodes to free up some of the excess reserved
	 * metadata space. This reduces the chances that the eofblocks scan
	 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
	 * also behaves as a filter to prevent too many eofblocks scans from
	 * running at the same time.
841
	 */
842 843 844 845 846 847 848
	if (ret == -EDQUOT && !enospc) {
		enospc = xfs_inode_free_quota_eofblocks(ip);
		if (enospc)
			goto write_retry;
	} else if (ret == -ENOSPC && !enospc) {
		struct xfs_eofblocks eofb = {0};

849
		enospc = 1;
D
Dave Chinner 已提交
850
		xfs_flush_inodes(ip->i_mount);
851 852 853
		eofb.eof_scan_owner = ip->i_ino; /* for locking */
		eofb.eof_flags = XFS_EOF_FLAGS_SYNC;
		xfs_icache_free_eofblocks(ip->i_mount, &eofb);
D
Dave Chinner 已提交
854
		goto write_retry;
855
	}
856

857
	current->backing_dev_info = NULL;
858 859
out:
	xfs_rw_iunlock(ip, iolock);
860 861 862 863
	return ret;
}

STATIC ssize_t
A
Al Viro 已提交
864
xfs_file_write_iter(
865
	struct kiocb		*iocb,
A
Al Viro 已提交
866
	struct iov_iter		*from)
867 868 869 870 871 872
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	ssize_t			ret;
A
Al Viro 已提交
873
	size_t			ocount = iov_iter_count(from);
874 875 876 877 878 879

	XFS_STATS_INC(xs_write_calls);

	if (ocount == 0)
		return 0;

A
Al Viro 已提交
880 881
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
		return -EIO;
882

883
	if ((iocb->ki_flags & IOCB_DIRECT) || IS_DAX(inode))
A
Al Viro 已提交
884
		ret = xfs_file_dio_aio_write(iocb, from);
885
	else
A
Al Viro 已提交
886
		ret = xfs_file_buffered_aio_write(iocb, from);
887

888 889
	if (ret > 0) {
		ssize_t err;
890

891
		XFS_STATS_ADD(xs_write_bytes, ret);
892

893
		/* Handle various SYNC-type writes */
894
		err = generic_write_sync(file, iocb->ki_pos - ret, ret);
895 896
		if (err < 0)
			ret = err;
897
	}
898
	return ret;
899 900
}

901 902 903 904 905
#define	XFS_FALLOC_FL_SUPPORTED						\
		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
		 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |	\
		 FALLOC_FL_INSERT_RANGE)

906 907
STATIC long
xfs_file_fallocate(
908 909 910 911
	struct file		*file,
	int			mode,
	loff_t			offset,
	loff_t			len)
912
{
913 914 915
	struct inode		*inode = file_inode(file);
	struct xfs_inode	*ip = XFS_I(inode);
	long			error;
916
	enum xfs_prealloc_flags	flags = 0;
917
	uint			iolock = XFS_IOLOCK_EXCL;
918
	loff_t			new_size = 0;
919
	bool			do_file_insert = 0;
920

921 922
	if (!S_ISREG(inode->i_mode))
		return -EINVAL;
923
	if (mode & ~XFS_FALLOC_FL_SUPPORTED)
924 925
		return -EOPNOTSUPP;

926
	xfs_ilock(ip, iolock);
927
	error = xfs_break_layouts(inode, &iolock, false);
928 929 930
	if (error)
		goto out_unlock;

931 932 933
	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
	iolock |= XFS_MMAPLOCK_EXCL;

934 935 936 937
	if (mode & FALLOC_FL_PUNCH_HOLE) {
		error = xfs_free_file_space(ip, offset, len);
		if (error)
			goto out_unlock;
938 939 940 941
	} else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
		unsigned blksize_mask = (1 << inode->i_blkbits) - 1;

		if (offset & blksize_mask || len & blksize_mask) {
D
Dave Chinner 已提交
942
			error = -EINVAL;
943 944 945
			goto out_unlock;
		}

946 947 948 949 950
		/*
		 * There is no need to overlap collapse range with EOF,
		 * in which case it is effectively a truncate operation
		 */
		if (offset + len >= i_size_read(inode)) {
D
Dave Chinner 已提交
951
			error = -EINVAL;
952 953 954
			goto out_unlock;
		}

955 956 957 958 959
		new_size = i_size_read(inode) - len;

		error = xfs_collapse_file_space(ip, offset, len);
		if (error)
			goto out_unlock;
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
	} else if (mode & FALLOC_FL_INSERT_RANGE) {
		unsigned blksize_mask = (1 << inode->i_blkbits) - 1;

		new_size = i_size_read(inode) + len;
		if (offset & blksize_mask || len & blksize_mask) {
			error = -EINVAL;
			goto out_unlock;
		}

		/* check the new inode size does not wrap through zero */
		if (new_size > inode->i_sb->s_maxbytes) {
			error = -EFBIG;
			goto out_unlock;
		}

		/* Offset should be less than i_size */
		if (offset >= i_size_read(inode)) {
			error = -EINVAL;
			goto out_unlock;
		}
		do_file_insert = 1;
981
	} else {
982 983
		flags |= XFS_PREALLOC_SET;

984 985 986
		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
		    offset + len > i_size_read(inode)) {
			new_size = offset + len;
D
Dave Chinner 已提交
987
			error = inode_newsize_ok(inode, new_size);
988 989 990
			if (error)
				goto out_unlock;
		}
991

992 993 994 995 996
		if (mode & FALLOC_FL_ZERO_RANGE)
			error = xfs_zero_file_space(ip, offset, len);
		else
			error = xfs_alloc_file_space(ip, offset, len,
						     XFS_BMAPI_PREALLOC);
997 998 999 1000
		if (error)
			goto out_unlock;
	}

1001
	if (file->f_flags & O_DSYNC)
1002 1003 1004
		flags |= XFS_PREALLOC_SYNC;

	error = xfs_update_prealloc_flags(ip, flags);
1005 1006 1007 1008 1009 1010 1011 1012 1013
	if (error)
		goto out_unlock;

	/* Change file size if needed */
	if (new_size) {
		struct iattr iattr;

		iattr.ia_valid = ATTR_SIZE;
		iattr.ia_size = new_size;
1014
		error = xfs_setattr_size(ip, &iattr);
1015 1016
		if (error)
			goto out_unlock;
1017 1018
	}

1019 1020 1021 1022 1023 1024 1025 1026 1027
	/*
	 * Perform hole insertion now that the file size has been
	 * updated so that if we crash during the operation we don't
	 * leave shifted extents past EOF and hence losing access to
	 * the data that is contained within them.
	 */
	if (do_file_insert)
		error = xfs_insert_file_space(ip, offset, len);

1028
out_unlock:
1029
	xfs_iunlock(ip, iolock);
D
Dave Chinner 已提交
1030
	return error;
1031 1032 1033
}


L
Linus Torvalds 已提交
1034
STATIC int
1035
xfs_file_open(
L
Linus Torvalds 已提交
1036
	struct inode	*inode,
1037
	struct file	*file)
L
Linus Torvalds 已提交
1038
{
1039
	if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
L
Linus Torvalds 已提交
1040
		return -EFBIG;
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
	if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
		return -EIO;
	return 0;
}

STATIC int
xfs_dir_open(
	struct inode	*inode,
	struct file	*file)
{
	struct xfs_inode *ip = XFS_I(inode);
	int		mode;
	int		error;

	error = xfs_file_open(inode, file);
	if (error)
		return error;

	/*
	 * If there are any blocks, read-ahead block 0 as we're almost
	 * certain to have the next operation be a read there.
	 */
1063
	mode = xfs_ilock_data_map_shared(ip);
1064
	if (ip->i_d.di_nextents > 0)
1065
		xfs_dir3_data_readahead(ip, 0, -1);
1066 1067
	xfs_iunlock(ip, mode);
	return 0;
L
Linus Torvalds 已提交
1068 1069 1070
}

STATIC int
1071
xfs_file_release(
L
Linus Torvalds 已提交
1072 1073 1074
	struct inode	*inode,
	struct file	*filp)
{
D
Dave Chinner 已提交
1075
	return xfs_release(XFS_I(inode));
L
Linus Torvalds 已提交
1076 1077 1078
}

STATIC int
1079
xfs_file_readdir(
A
Al Viro 已提交
1080 1081
	struct file	*file,
	struct dir_context *ctx)
L
Linus Torvalds 已提交
1082
{
A
Al Viro 已提交
1083
	struct inode	*inode = file_inode(file);
1084
	xfs_inode_t	*ip = XFS_I(inode);
C
Christoph Hellwig 已提交
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
	size_t		bufsize;

	/*
	 * The Linux API doesn't pass down the total size of the buffer
	 * we read into down to the filesystem.  With the filldir concept
	 * it's not needed for correct information, but the XFS dir2 leaf
	 * code wants an estimate of the buffer size to calculate it's
	 * readahead window and size the buffers used for mapping to
	 * physical blocks.
	 *
	 * Try to give it an estimate that's good enough, maybe at some
	 * point we can change the ->readdir prototype to include the
E
Eric Sandeen 已提交
1097
	 * buffer size.  For now we use the current glibc buffer size.
C
Christoph Hellwig 已提交
1098
	 */
E
Eric Sandeen 已提交
1099
	bufsize = (size_t)min_t(loff_t, 32768, ip->i_d.di_size);
C
Christoph Hellwig 已提交
1100

1101
	return xfs_readdir(ip, ctx, bufsize);
L
Linus Torvalds 已提交
1102 1103
}

1104 1105
/*
 * This type is designed to indicate the type of offset we would like
1106
 * to search from page cache for xfs_seek_hole_data().
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
 */
enum {
	HOLE_OFF = 0,
	DATA_OFF,
};

/*
 * Lookup the desired type of offset from the given page.
 *
 * On success, return true and the offset argument will point to the
 * start of the region that was found.  Otherwise this function will
 * return false and keep the offset argument unchanged.
 */
STATIC bool
xfs_lookup_buffer_offset(
	struct page		*page,
	loff_t			*offset,
	unsigned int		type)
{
	loff_t			lastoff = page_offset(page);
	bool			found = false;
	struct buffer_head	*bh, *head;

	bh = head = page_buffers(page);
	do {
		/*
		 * Unwritten extents that have data in the page
		 * cache covering them can be identified by the
		 * BH_Unwritten state flag.  Pages with multiple
		 * buffers might have a mix of holes, data and
		 * unwritten extents - any buffer with valid
		 * data in it should have BH_Uptodate flag set
		 * on it.
		 */
		if (buffer_unwritten(bh) ||
		    buffer_uptodate(bh)) {
			if (type == DATA_OFF)
				found = true;
		} else {
			if (type == HOLE_OFF)
				found = true;
		}

		if (found) {
			*offset = lastoff;
			break;
		}
		lastoff += bh->b_size;
	} while ((bh = bh->b_this_page) != head);

	return found;
}

/*
 * This routine is called to find out and return a data or hole offset
 * from the page cache for unwritten extents according to the desired
1163
 * type for xfs_seek_hole_data().
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
 *
 * The argument offset is used to tell where we start to search from the
 * page cache.  Map is used to figure out the end points of the range to
 * lookup pages.
 *
 * Return true if the desired type of offset was found, and the argument
 * offset is filled with that address.  Otherwise, return false and keep
 * offset unchanged.
 */
STATIC bool
xfs_find_get_desired_pgoff(
	struct inode		*inode,
	struct xfs_bmbt_irec	*map,
	unsigned int		type,
	loff_t			*offset)
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	struct pagevec		pvec;
	pgoff_t			index;
	pgoff_t			end;
	loff_t			endoff;
	loff_t			startoff = *offset;
	loff_t			lastoff = startoff;
	bool			found = false;

	pagevec_init(&pvec, 0);

	index = startoff >> PAGE_CACHE_SHIFT;
	endoff = XFS_FSB_TO_B(mp, map->br_startoff + map->br_blockcount);
	end = endoff >> PAGE_CACHE_SHIFT;
	do {
		int		want;
		unsigned	nr_pages;
		unsigned int	i;

		want = min_t(pgoff_t, end - index, PAGEVEC_SIZE);
		nr_pages = pagevec_lookup(&pvec, inode->i_mapping, index,
					  want);
		/*
		 * No page mapped into given range.  If we are searching holes
		 * and if this is the first time we got into the loop, it means
		 * that the given offset is landed in a hole, return it.
		 *
		 * If we have already stepped through some block buffers to find
		 * holes but they all contains data.  In this case, the last
		 * offset is already updated and pointed to the end of the last
		 * mapped page, if it does not reach the endpoint to search,
		 * that means there should be a hole between them.
		 */
		if (nr_pages == 0) {
			/* Data search found nothing */
			if (type == DATA_OFF)
				break;

			ASSERT(type == HOLE_OFF);
			if (lastoff == startoff || lastoff < endoff) {
				found = true;
				*offset = lastoff;
			}
			break;
		}

		/*
		 * At lease we found one page.  If this is the first time we
		 * step into the loop, and if the first page index offset is
		 * greater than the given search offset, a hole was found.
		 */
		if (type == HOLE_OFF && lastoff == startoff &&
		    lastoff < page_offset(pvec.pages[0])) {
			found = true;
			break;
		}

		for (i = 0; i < nr_pages; i++) {
			struct page	*page = pvec.pages[i];
			loff_t		b_offset;

			/*
			 * At this point, the page may be truncated or
			 * invalidated (changing page->mapping to NULL),
			 * or even swizzled back from swapper_space to tmpfs
			 * file mapping. However, page->index will not change
			 * because we have a reference on the page.
			 *
			 * Searching done if the page index is out of range.
			 * If the current offset is not reaches the end of
			 * the specified search range, there should be a hole
			 * between them.
			 */
			if (page->index > end) {
				if (type == HOLE_OFF && lastoff < endoff) {
					*offset = lastoff;
					found = true;
				}
				goto out;
			}

			lock_page(page);
			/*
			 * Page truncated or invalidated(page->mapping == NULL).
			 * We can freely skip it and proceed to check the next
			 * page.
			 */
			if (unlikely(page->mapping != inode->i_mapping)) {
				unlock_page(page);
				continue;
			}

			if (!page_has_buffers(page)) {
				unlock_page(page);
				continue;
			}

			found = xfs_lookup_buffer_offset(page, &b_offset, type);
			if (found) {
				/*
				 * The found offset may be less than the start
				 * point to search if this is the first time to
				 * come here.
				 */
				*offset = max_t(loff_t, startoff, b_offset);
				unlock_page(page);
				goto out;
			}

			/*
			 * We either searching data but nothing was found, or
			 * searching hole but found a data buffer.  In either
			 * case, probably the next page contains the desired
			 * things, update the last offset to it so.
			 */
			lastoff = page_offset(page) + PAGE_SIZE;
			unlock_page(page);
		}

		/*
		 * The number of returned pages less than our desired, search
		 * done.  In this case, nothing was found for searching data,
		 * but we found a hole behind the last offset.
		 */
		if (nr_pages < want) {
			if (type == HOLE_OFF) {
				*offset = lastoff;
				found = true;
			}
			break;
		}

		index = pvec.pages[i - 1]->index + 1;
		pagevec_release(&pvec);
	} while (index <= end);

out:
	pagevec_release(&pvec);
	return found;
}

1322
STATIC loff_t
1323
xfs_seek_hole_data(
1324
	struct file		*file,
1325 1326
	loff_t			start,
	int			whence)
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
{
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	loff_t			uninitialized_var(offset);
	xfs_fsize_t		isize;
	xfs_fileoff_t		fsbno;
	xfs_filblks_t		end;
	uint			lock;
	int			error;

1338 1339 1340
	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

1341
	lock = xfs_ilock_data_map_shared(ip);
1342 1343 1344

	isize = i_size_read(inode);
	if (start >= isize) {
D
Dave Chinner 已提交
1345
		error = -ENXIO;
1346 1347 1348 1349 1350 1351 1352
		goto out_unlock;
	}

	/*
	 * Try to read extents from the first block indicated
	 * by fsbno to the end block of the file.
	 */
1353
	fsbno = XFS_B_TO_FSBT(mp, start);
1354
	end = XFS_B_TO_FSB(mp, isize);
1355

1356 1357 1358 1359
	for (;;) {
		struct xfs_bmbt_irec	map[2];
		int			nmap = 2;
		unsigned int		i;
1360

1361 1362 1363 1364
		error = xfs_bmapi_read(ip, fsbno, end - fsbno, map, &nmap,
				       XFS_BMAPI_ENTIRE);
		if (error)
			goto out_unlock;
1365

1366 1367
		/* No extents at given offset, must be beyond EOF */
		if (nmap == 0) {
D
Dave Chinner 已提交
1368
			error = -ENXIO;
1369 1370 1371 1372 1373 1374 1375
			goto out_unlock;
		}

		for (i = 0; i < nmap; i++) {
			offset = max_t(loff_t, start,
				       XFS_FSB_TO_B(mp, map[i].br_startoff));

1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
			/* Landed in the hole we wanted? */
			if (whence == SEEK_HOLE &&
			    map[i].br_startblock == HOLESTARTBLOCK)
				goto out;

			/* Landed in the data extent we wanted? */
			if (whence == SEEK_DATA &&
			    (map[i].br_startblock == DELAYSTARTBLOCK ||
			     (map[i].br_state == XFS_EXT_NORM &&
			      !isnullstartblock(map[i].br_startblock))))
1386 1387 1388
				goto out;

			/*
1389 1390
			 * Landed in an unwritten extent, try to search
			 * for hole or data from page cache.
1391 1392 1393
			 */
			if (map[i].br_state == XFS_EXT_UNWRITTEN) {
				if (xfs_find_get_desired_pgoff(inode, &map[i],
1394 1395
				      whence == SEEK_HOLE ? HOLE_OFF : DATA_OFF,
							&offset))
1396 1397 1398 1399 1400
					goto out;
			}
		}

		/*
1401 1402
		 * We only received one extent out of the two requested. This
		 * means we've hit EOF and didn't find what we are looking for.
1403
		 */
1404
		if (nmap == 1) {
1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
			/*
			 * If we were looking for a hole, set offset to
			 * the end of the file (i.e., there is an implicit
			 * hole at the end of any file).
		 	 */
			if (whence == SEEK_HOLE) {
				offset = isize;
				break;
			}
			/*
			 * If we were looking for data, it's nowhere to be found
			 */
			ASSERT(whence == SEEK_DATA);
D
Dave Chinner 已提交
1418
			error = -ENXIO;
1419 1420 1421
			goto out_unlock;
		}

1422 1423 1424 1425
		ASSERT(i > 1);

		/*
		 * Nothing was found, proceed to the next round of search
1426
		 * if the next reading offset is not at or beyond EOF.
1427 1428 1429 1430
		 */
		fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
		start = XFS_FSB_TO_B(mp, fsbno);
		if (start >= isize) {
1431 1432 1433 1434 1435
			if (whence == SEEK_HOLE) {
				offset = isize;
				break;
			}
			ASSERT(whence == SEEK_DATA);
D
Dave Chinner 已提交
1436
			error = -ENXIO;
1437 1438
			goto out_unlock;
		}
1439 1440
	}

1441 1442
out:
	/*
1443
	 * If at this point we have found the hole we wanted, the returned
1444
	 * offset may be bigger than the file size as it may be aligned to
1445
	 * page boundary for unwritten extents.  We need to deal with this
1446 1447
	 * situation in particular.
	 */
1448 1449
	if (whence == SEEK_HOLE)
		offset = min_t(loff_t, offset, isize);
J
Jie Liu 已提交
1450
	offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1451 1452

out_unlock:
1453
	xfs_iunlock(ip, lock);
1454 1455

	if (error)
D
Dave Chinner 已提交
1456
		return error;
1457 1458 1459 1460 1461 1462 1463
	return offset;
}

STATIC loff_t
xfs_file_llseek(
	struct file	*file,
	loff_t		offset,
1464
	int		whence)
1465
{
1466
	switch (whence) {
1467 1468 1469
	case SEEK_END:
	case SEEK_CUR:
	case SEEK_SET:
1470
		return generic_file_llseek(file, offset, whence);
1471
	case SEEK_HOLE:
1472
	case SEEK_DATA:
1473
		return xfs_seek_hole_data(file, offset, whence);
1474 1475 1476 1477 1478
	default:
		return -EINVAL;
	}
}

1479 1480 1481 1482 1483
/*
 * Locking for serialisation of IO during page faults. This results in a lock
 * ordering of:
 *
 * mmap_sem (MM)
1484 1485 1486 1487
 *   sb_start_pagefault(vfs, freeze)
 *     i_mmap_lock (XFS - truncate serialisation)
 *       page_lock (MM)
 *         i_lock (XFS - extent map serialisation)
1488 1489
 */

1490 1491 1492 1493 1494
/*
 * mmap()d file has taken write protection fault and is being made writable. We
 * can set the page state up correctly for a writable page, which means we can
 * do correct delalloc accounting (ENOSPC checking!) and unwritten extent
 * mapping.
1495 1496
 */
STATIC int
1497
xfs_filemap_page_mkwrite(
1498 1499 1500
	struct vm_area_struct	*vma,
	struct vm_fault		*vmf)
{
1501
	struct inode		*inode = file_inode(vma->vm_file);
1502
	int			ret;
1503

1504
	trace_xfs_filemap_page_mkwrite(XFS_I(inode));
1505

1506
	sb_start_pagefault(inode->i_sb);
1507
	file_update_time(vma->vm_file);
1508
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1509

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
	if (IS_DAX(inode)) {
		ret = __dax_mkwrite(vma, vmf, xfs_get_blocks_direct,
				    xfs_end_io_dax_write);
	} else {
		ret = __block_page_mkwrite(vma, vmf, xfs_get_blocks);
		ret = block_page_mkwrite_return(ret);
	}

	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	sb_end_pagefault(inode->i_sb);

	return ret;
1522 1523
}

1524
STATIC int
1525
xfs_filemap_fault(
1526 1527 1528
	struct vm_area_struct	*vma,
	struct vm_fault		*vmf)
{
1529
	struct inode		*inode = file_inode(vma->vm_file);
1530
	int			ret;
1531

1532
	trace_xfs_filemap_fault(XFS_I(inode));
1533

1534
	/* DAX can shortcut the normal fault path on write faults! */
1535
	if ((vmf->flags & FAULT_FLAG_WRITE) && IS_DAX(inode))
1536
		return xfs_filemap_page_mkwrite(vma, vmf);
1537

1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	if (IS_DAX(inode)) {
		/*
		 * we do not want to trigger unwritten extent conversion on read
		 * faults - that is unnecessary overhead and would also require
		 * changes to xfs_get_blocks_direct() to map unwritten extent
		 * ioend for conversion on read-only mappings.
		 */
		ret = __dax_fault(vma, vmf, xfs_get_blocks_direct, NULL);
	} else
		ret = filemap_fault(vma, vmf);
	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1550

1551 1552 1553
	return ret;
}

M
Matthew Wilcox 已提交
1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
STATIC int
xfs_filemap_pmd_fault(
	struct vm_area_struct	*vma,
	unsigned long		addr,
	pmd_t			*pmd,
	unsigned int		flags)
{
	struct inode		*inode = file_inode(vma->vm_file);
	struct xfs_inode	*ip = XFS_I(inode);
	int			ret;

	if (!IS_DAX(inode))
		return VM_FAULT_FALLBACK;

	trace_xfs_filemap_pmd_fault(ip);

	sb_start_pagefault(inode->i_sb);
	file_update_time(vma->vm_file);
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	ret = __dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_direct,
				    xfs_end_io_dax_write);
	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	sb_end_pagefault(inode->i_sb);

	return ret;
}

1581 1582
static const struct vm_operations_struct xfs_file_vm_ops = {
	.fault		= xfs_filemap_fault,
M
Matthew Wilcox 已提交
1583
	.pmd_fault	= xfs_filemap_pmd_fault,
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
	.map_pages	= filemap_map_pages,
	.page_mkwrite	= xfs_filemap_page_mkwrite,
};

STATIC int
xfs_file_mmap(
	struct file	*filp,
	struct vm_area_struct *vma)
{
	file_accessed(filp);
	vma->vm_ops = &xfs_file_vm_ops;
	if (IS_DAX(file_inode(filp)))
M
Matthew Wilcox 已提交
1596
		vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
1597
	return 0;
1598 1599
}

1600
const struct file_operations xfs_file_operations = {
1601
	.llseek		= xfs_file_llseek,
A
Al Viro 已提交
1602
	.read_iter	= xfs_file_read_iter,
A
Al Viro 已提交
1603
	.write_iter	= xfs_file_write_iter,
1604
	.splice_read	= xfs_file_splice_read,
A
Al Viro 已提交
1605
	.splice_write	= iter_file_splice_write,
1606
	.unlocked_ioctl	= xfs_file_ioctl,
L
Linus Torvalds 已提交
1607
#ifdef CONFIG_COMPAT
1608
	.compat_ioctl	= xfs_file_compat_ioctl,
L
Linus Torvalds 已提交
1609
#endif
1610 1611 1612 1613
	.mmap		= xfs_file_mmap,
	.open		= xfs_file_open,
	.release	= xfs_file_release,
	.fsync		= xfs_file_fsync,
1614
	.fallocate	= xfs_file_fallocate,
L
Linus Torvalds 已提交
1615 1616
};

1617
const struct file_operations xfs_dir_file_operations = {
1618
	.open		= xfs_dir_open,
L
Linus Torvalds 已提交
1619
	.read		= generic_read_dir,
A
Al Viro 已提交
1620
	.iterate	= xfs_file_readdir,
1621
	.llseek		= generic_file_llseek,
1622
	.unlocked_ioctl	= xfs_file_ioctl,
1623
#ifdef CONFIG_COMPAT
1624
	.compat_ioctl	= xfs_file_compat_ioctl,
1625
#endif
1626
	.fsync		= xfs_dir_fsync,
L
Linus Torvalds 已提交
1627
};