xfs_file.c 42.8 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_fs.h"
20
#include "xfs_shared.h"
21
#include "xfs_format.h"
22 23
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
L
Linus Torvalds 已提交
24
#include "xfs_mount.h"
25 26
#include "xfs_da_format.h"
#include "xfs_da_btree.h"
L
Linus Torvalds 已提交
27
#include "xfs_inode.h"
28
#include "xfs_trans.h"
29
#include "xfs_inode_item.h"
30
#include "xfs_bmap.h"
D
Dave Chinner 已提交
31
#include "xfs_bmap_util.h"
L
Linus Torvalds 已提交
32
#include "xfs_error.h"
33
#include "xfs_dir2.h"
D
Dave Chinner 已提交
34
#include "xfs_dir2_priv.h"
35
#include "xfs_ioctl.h"
36
#include "xfs_trace.h"
37
#include "xfs_log.h"
38
#include "xfs_icache.h"
39
#include "xfs_pnfs.h"
40
#include "xfs_iomap.h"
L
Linus Torvalds 已提交
41 42

#include <linux/dcache.h>
43
#include <linux/falloc.h>
44
#include <linux/pagevec.h>
45
#include <linux/backing-dev.h>
L
Linus Torvalds 已提交
46

47
static const struct vm_operations_struct xfs_file_vm_ops;
L
Linus Torvalds 已提交
48

49 50 51 52 53 54 55 56 57 58
/*
 * Locking primitives for read and write IO paths to ensure we consistently use
 * and order the inode->i_mutex, ip->i_lock and ip->i_iolock.
 */
static inline void
xfs_rw_ilock(
	struct xfs_inode	*ip,
	int			type)
{
	if (type & XFS_IOLOCK_EXCL)
A
Al Viro 已提交
59
		inode_lock(VFS_I(ip));
60 61 62 63 64 65 66 67 68 69
	xfs_ilock(ip, type);
}

static inline void
xfs_rw_iunlock(
	struct xfs_inode	*ip,
	int			type)
{
	xfs_iunlock(ip, type);
	if (type & XFS_IOLOCK_EXCL)
A
Al Viro 已提交
70
		inode_unlock(VFS_I(ip));
71 72 73 74 75 76 77 78 79
}

static inline void
xfs_rw_ilock_demote(
	struct xfs_inode	*ip,
	int			type)
{
	xfs_ilock_demote(ip, type);
	if (type & XFS_IOLOCK_EXCL)
A
Al Viro 已提交
80
		inode_unlock(VFS_I(ip));
81 82
}

83
/*
84 85
 * Clear the specified ranges to zero through either the pagecache or DAX.
 * Holes and unwritten extents will be left as-is as they already are zeroed.
86
 */
87
int
88
xfs_zero_range(
89
	struct xfs_inode	*ip,
90 91 92
	xfs_off_t		pos,
	xfs_off_t		count,
	bool			*did_zero)
93
{
94
	return iomap_zero_range(VFS_I(ip), pos, count, NULL, &xfs_iomap_ops);
95 96
}

97 98 99 100 101 102 103 104
int
xfs_update_prealloc_flags(
	struct xfs_inode	*ip,
	enum xfs_prealloc_flags	flags)
{
	struct xfs_trans	*tp;
	int			error;

105 106 107
	error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_writeid,
			0, 0, 0, &tp);
	if (error)
108 109 110 111 112 113
		return error;

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);

	if (!(flags & XFS_PREALLOC_INVISIBLE)) {
D
Dave Chinner 已提交
114 115 116
		VFS_I(ip)->i_mode &= ~S_ISUID;
		if (VFS_I(ip)->i_mode & S_IXGRP)
			VFS_I(ip)->i_mode &= ~S_ISGID;
117 118 119 120 121 122 123 124 125 126 127
		xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
	}

	if (flags & XFS_PREALLOC_SET)
		ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
	if (flags & XFS_PREALLOC_CLEAR)
		ip->i_d.di_flags &= ~XFS_DIFLAG_PREALLOC;

	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
	if (flags & XFS_PREALLOC_SYNC)
		xfs_trans_set_sync(tp);
128
	return xfs_trans_commit(tp);
129 130
}

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
/*
 * Fsync operations on directories are much simpler than on regular files,
 * as there is no file data to flush, and thus also no need for explicit
 * cache flush operations, and there are no non-transaction metadata updates
 * on directories either.
 */
STATIC int
xfs_dir_fsync(
	struct file		*file,
	loff_t			start,
	loff_t			end,
	int			datasync)
{
	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_lsn_t		lsn = 0;

	trace_xfs_dir_fsync(ip);

	xfs_ilock(ip, XFS_ILOCK_SHARED);
	if (xfs_ipincount(ip))
		lsn = ip->i_itemp->ili_last_lsn;
	xfs_iunlock(ip, XFS_ILOCK_SHARED);

	if (!lsn)
		return 0;
D
Dave Chinner 已提交
157
	return _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, NULL);
158 159
}

160 161 162
STATIC int
xfs_file_fsync(
	struct file		*file,
163 164
	loff_t			start,
	loff_t			end,
165 166
	int			datasync)
{
167 168
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
169
	struct xfs_mount	*mp = ip->i_mount;
170 171
	int			error = 0;
	int			log_flushed = 0;
172
	xfs_lsn_t		lsn = 0;
173

C
Christoph Hellwig 已提交
174
	trace_xfs_file_fsync(ip);
175

176 177 178 179
	error = filemap_write_and_wait_range(inode->i_mapping, start, end);
	if (error)
		return error;

180
	if (XFS_FORCED_SHUTDOWN(mp))
E
Eric Sandeen 已提交
181
		return -EIO;
182 183 184

	xfs_iflags_clear(ip, XFS_ITRUNCATED);

185 186 187 188 189 190 191 192 193 194 195 196 197 198
	if (mp->m_flags & XFS_MOUNT_BARRIER) {
		/*
		 * If we have an RT and/or log subvolume we need to make sure
		 * to flush the write cache the device used for file data
		 * first.  This is to ensure newly written file data make
		 * it to disk before logging the new inode size in case of
		 * an extending write.
		 */
		if (XFS_IS_REALTIME_INODE(ip))
			xfs_blkdev_issue_flush(mp->m_rtdev_targp);
		else if (mp->m_logdev_targp != mp->m_ddev_targp)
			xfs_blkdev_issue_flush(mp->m_ddev_targp);
	}

199
	/*
200 201 202 203 204 205 206 207 208 209 210
	 * All metadata updates are logged, which means that we just have to
	 * flush the log up to the latest LSN that touched the inode. If we have
	 * concurrent fsync/fdatasync() calls, we need them to all block on the
	 * log force before we clear the ili_fsync_fields field. This ensures
	 * that we don't get a racing sync operation that does not wait for the
	 * metadata to hit the journal before returning. If we race with
	 * clearing the ili_fsync_fields, then all that will happen is the log
	 * force will do nothing as the lsn will already be on disk. We can't
	 * race with setting ili_fsync_fields because that is done under
	 * XFS_ILOCK_EXCL, and that can't happen because we hold the lock shared
	 * until after the ili_fsync_fields is cleared.
211 212
	 */
	xfs_ilock(ip, XFS_ILOCK_SHARED);
213 214
	if (xfs_ipincount(ip)) {
		if (!datasync ||
215
		    (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
216 217
			lsn = ip->i_itemp->ili_last_lsn;
	}
218

219
	if (lsn) {
220
		error = _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);
221 222 223
		ip->i_itemp->ili_fsync_fields = 0;
	}
	xfs_iunlock(ip, XFS_ILOCK_SHARED);
224

225 226 227 228 229 230 231 232 233 234 235 236
	/*
	 * If we only have a single device, and the log force about was
	 * a no-op we might have to flush the data device cache here.
	 * This can only happen for fdatasync/O_DSYNC if we were overwriting
	 * an already allocated file and thus do not have any metadata to
	 * commit.
	 */
	if ((mp->m_flags & XFS_MOUNT_BARRIER) &&
	    mp->m_logdev_targp == mp->m_ddev_targp &&
	    !XFS_IS_REALTIME_INODE(ip) &&
	    !log_flushed)
		xfs_blkdev_issue_flush(mp->m_ddev_targp);
237

D
Dave Chinner 已提交
238
	return error;
239 240
}

241
STATIC ssize_t
242
xfs_file_dio_aio_read(
243
	struct kiocb		*iocb,
A
Al Viro 已提交
244
	struct iov_iter		*to)
245
{
246 247
	struct address_space	*mapping = iocb->ki_filp->f_mapping;
	struct inode		*inode = mapping->host;
248
	struct xfs_inode	*ip = XFS_I(inode);
249
	loff_t			isize = i_size_read(inode);
250
	size_t			count = iov_iter_count(to);
251
	struct iov_iter		data;
252
	struct xfs_buftarg	*target;
253 254
	ssize_t			ret = 0;

255
	trace_xfs_file_direct_read(ip, count, iocb->ki_pos);
256

257 258
	if (!count)
		return 0; /* skip atime */
259

260 261 262 263
	if (XFS_IS_REALTIME_INODE(ip))
		target = ip->i_mount->m_rtdev_targp;
	else
		target = ip->i_mount->m_ddev_targp;
264

265 266 267 268 269
	/* DIO must be aligned to device logical sector size */
	if ((iocb->ki_pos | count) & target->bt_logical_sectormask) {
		if (iocb->ki_pos == isize)
			return 0;
		return -EINVAL;
270 271
	}

272
	/*
273 274 275 276 277 278 279 280
	 * Locking is a bit tricky here. If we take an exclusive lock for direct
	 * IO, we effectively serialise all new concurrent read IO to this file
	 * and block it behind IO that is currently in progress because IO in
	 * progress holds the IO lock shared. We only need to hold the lock
	 * exclusive to blow away the page cache, so only take lock exclusively
	 * if the page cache needs invalidation. This allows the normal direct
	 * IO case of no page cache pages to proceeed concurrently without
	 * serialisation.
281 282
	 */
	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
283
	if (mapping->nrpages) {
284
		xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
285 286
		xfs_rw_ilock(ip, XFS_IOLOCK_EXCL);

287 288 289 290 291 292 293 294 295 296 297
		/*
		 * The generic dio code only flushes the range of the particular
		 * I/O. Because we take an exclusive lock here, this whole
		 * sequence is considerably more expensive for us. This has a
		 * noticeable performance impact for any file with cached pages,
		 * even when outside of the range of the particular I/O.
		 *
		 * Hence, amortize the cost of the lock against a full file
		 * flush and reduce the chances of repeated iolock cycles going
		 * forward.
		 */
298 299
		if (mapping->nrpages) {
			ret = filemap_write_and_wait(mapping);
300 301 302 303
			if (ret) {
				xfs_rw_iunlock(ip, XFS_IOLOCK_EXCL);
				return ret;
			}
304 305 306 307 308 309

			/*
			 * Invalidate whole pages. This can return an error if
			 * we fail to invalidate a page, but this should never
			 * happen on XFS. Warn if it does fail.
			 */
310
			ret = invalidate_inode_pages2(mapping);
311 312
			WARN_ON_ONCE(ret);
			ret = 0;
313
		}
314
		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
315
	}
316

317
	data = *to;
318 319 320 321 322
	ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
			xfs_get_blocks_direct, NULL, NULL, 0);
	if (ret > 0) {
		iocb->ki_pos += ret;
		iov_iter_advance(to, ret);
323
	}
324
	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
325

326 327 328 329
	file_accessed(iocb->ki_filp);
	return ret;
}

330
static noinline ssize_t
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
xfs_file_dax_read(
	struct kiocb		*iocb,
	struct iov_iter		*to)
{
	struct address_space	*mapping = iocb->ki_filp->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct iov_iter		data = *to;
	size_t			count = iov_iter_count(to);
	ssize_t			ret = 0;

	trace_xfs_file_dax_read(ip, count, iocb->ki_pos);

	if (!count)
		return 0; /* skip atime */

	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
	ret = dax_do_io(iocb, inode, &data, xfs_get_blocks_direct, NULL, 0);
349 350 351 352
	if (ret > 0) {
		iocb->ki_pos += ret;
		iov_iter_advance(to, ret);
	}
353 354
	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);

355
	file_accessed(iocb->ki_filp);
356 357 358 359 360 361 362 363 364 365 366 367
	return ret;
}

STATIC ssize_t
xfs_file_buffered_aio_read(
	struct kiocb		*iocb,
	struct iov_iter		*to)
{
	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
	ssize_t			ret;

	trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos);
368

369
	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
A
Al Viro 已提交
370
	ret = generic_file_read_iter(iocb, to);
371 372 373 374 375 376 377 378 379 380
	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);

	return ret;
}

STATIC ssize_t
xfs_file_read_iter(
	struct kiocb		*iocb,
	struct iov_iter		*to)
{
381 382
	struct inode		*inode = file_inode(iocb->ki_filp);
	struct xfs_mount	*mp = XFS_I(inode)->i_mount;
383 384 385 386 387 388 389
	ssize_t			ret = 0;

	XFS_STATS_INC(mp, xs_read_calls);

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

390 391 392
	if (IS_DAX(inode))
		ret = xfs_file_dax_read(iocb, to);
	else if (iocb->ki_flags & IOCB_DIRECT)
393
		ret = xfs_file_dio_aio_read(iocb, to);
C
Christoph Hellwig 已提交
394
	else
395
		ret = xfs_file_buffered_aio_read(iocb, to);
396 397

	if (ret > 0)
398
		XFS_STATS_ADD(mp, xs_read_bytes, ret);
399 400 401 402
	return ret;
}

/*
403 404 405 406 407 408 409 410 411
 * Zero any on disk space between the current EOF and the new, larger EOF.
 *
 * This handles the normal case of zeroing the remainder of the last block in
 * the file and the unusual case of zeroing blocks out beyond the size of the
 * file.  This second case only happens with fixed size extents and when the
 * system crashes before the inode size was updated but after blocks were
 * allocated.
 *
 * Expects the iolock to be held exclusive, and will take the ilock internally.
412 413 414
 */
int					/* error (positive) */
xfs_zero_eof(
415 416
	struct xfs_inode	*ip,
	xfs_off_t		offset,		/* starting I/O offset */
417 418
	xfs_fsize_t		isize,		/* current inode size */
	bool			*did_zeroing)
419
{
420
	ASSERT(xfs_isilocked(ip, XFS_IOLOCK_EXCL));
421 422
	ASSERT(offset > isize);

423
	trace_xfs_zero_eof(ip, isize, offset - isize);
424
	return xfs_zero_range(ip, isize, offset - isize, did_zeroing);
425 426
}

427 428 429
/*
 * Common pre-write limit and setup checks.
 *
430 431 432
 * Called with the iolocked held either shared and exclusive according to
 * @iolock, and returns with it held.  Might upgrade the iolock to exclusive
 * if called for a direct write beyond i_size.
433 434 435
 */
STATIC ssize_t
xfs_file_aio_write_checks(
436 437
	struct kiocb		*iocb,
	struct iov_iter		*from,
438 439
	int			*iolock)
{
440
	struct file		*file = iocb->ki_filp;
441 442
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
443
	ssize_t			error = 0;
444
	size_t			count = iov_iter_count(from);
445
	bool			drained_dio = false;
446

447
restart:
448 449
	error = generic_write_checks(iocb, from);
	if (error <= 0)
450 451
		return error;

452
	error = xfs_break_layouts(inode, iolock, true);
453 454 455
	if (error)
		return error;

456 457 458 459 460 461 462
	/* For changing security info in file_remove_privs() we need i_mutex */
	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
		xfs_rw_iunlock(ip, *iolock);
		*iolock = XFS_IOLOCK_EXCL;
		xfs_rw_ilock(ip, *iolock);
		goto restart;
	}
463 464 465
	/*
	 * If the offset is beyond the size of the file, we need to zero any
	 * blocks that fall between the existing EOF and the start of this
466
	 * write.  If zeroing is needed and we are currently holding the
467 468
	 * iolock shared, we need to update it to exclusive which implies
	 * having to redo all checks before.
469 470 471 472 473 474 475 476
	 *
	 * We need to serialise against EOF updates that occur in IO
	 * completions here. We want to make sure that nobody is changing the
	 * size while we do this check until we have placed an IO barrier (i.e.
	 * hold the XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.
	 * The spinlock effectively forms a memory barrier once we have the
	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value
	 * and hence be able to correctly determine if we need to run zeroing.
477
	 */
478
	spin_lock(&ip->i_flags_lock);
479
	if (iocb->ki_pos > i_size_read(inode)) {
480 481
		bool	zero = false;

482
		spin_unlock(&ip->i_flags_lock);
483 484 485 486 487 488 489
		if (!drained_dio) {
			if (*iolock == XFS_IOLOCK_SHARED) {
				xfs_rw_iunlock(ip, *iolock);
				*iolock = XFS_IOLOCK_EXCL;
				xfs_rw_ilock(ip, *iolock);
				iov_iter_reexpand(from, count);
			}
490 491 492 493 494 495 496 497 498
			/*
			 * We now have an IO submission barrier in place, but
			 * AIO can do EOF updates during IO completion and hence
			 * we now need to wait for all of them to drain. Non-AIO
			 * DIO will have drained before we are given the
			 * XFS_IOLOCK_EXCL, and so for most cases this wait is a
			 * no-op.
			 */
			inode_dio_wait(inode);
499
			drained_dio = true;
500 501
			goto restart;
		}
502
		error = xfs_zero_eof(ip, iocb->ki_pos, i_size_read(inode), &zero);
503 504
		if (error)
			return error;
505 506
	} else
		spin_unlock(&ip->i_flags_lock);
507

C
Christoph Hellwig 已提交
508 509 510 511 512 513
	/*
	 * Updating the timestamps will grab the ilock again from
	 * xfs_fs_dirty_inode, so we have to call it after dropping the
	 * lock above.  Eventually we should look into a way to avoid
	 * the pointless lock roundtrip.
	 */
514 515 516 517 518
	if (likely(!(file->f_mode & FMODE_NOCMTIME))) {
		error = file_update_time(file);
		if (error)
			return error;
	}
C
Christoph Hellwig 已提交
519

520 521 522 523 524
	/*
	 * If we're writing the file then make sure to clear the setuid and
	 * setgid bits if the process is not being run by root.  This keeps
	 * people from modifying setuid and setgid binaries.
	 */
525 526 527
	if (!IS_NOSEC(inode))
		return file_remove_privs(file);
	return 0;
528 529
}

530 531 532 533
/*
 * xfs_file_dio_aio_write - handle direct IO writes
 *
 * Lock the inode appropriately to prepare for and issue a direct IO write.
534
 * By separating it from the buffered write path we remove all the tricky to
535 536
 * follow locking changes and looping.
 *
537 538 539 540 541 542 543 544 545 546 547 548 549
 * If there are cached pages or we're extending the file, we need IOLOCK_EXCL
 * until we're sure the bytes at the new EOF have been zeroed and/or the cached
 * pages are flushed out.
 *
 * In most cases the direct IO writes will be done holding IOLOCK_SHARED
 * allowing them to be done in parallel with reads and other direct IO writes.
 * However, if the IO is not aligned to filesystem blocks, the direct IO layer
 * needs to do sub-block zeroing and that requires serialisation against other
 * direct IOs to the same block. In this case we need to serialise the
 * submission of the unaligned IOs so that we don't get racing block zeroing in
 * the dio layer.  To avoid the problem with aio, we also need to wait for
 * outstanding IOs to complete so that unwritten extent conversion is completed
 * before we try to map the overlapping block. This is currently implemented by
C
Christoph Hellwig 已提交
550
 * hitting it with a big hammer (i.e. inode_dio_wait()).
551
 *
552 553 554 555 556 557
 * Returns with locks held indicated by @iolock and errors indicated by
 * negative return values.
 */
STATIC ssize_t
xfs_file_dio_aio_write(
	struct kiocb		*iocb,
558
	struct iov_iter		*from)
559 560 561 562 563 564 565
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	ssize_t			ret = 0;
566
	int			unaligned_io = 0;
567
	int			iolock;
568
	size_t			count = iov_iter_count(from);
569 570
	loff_t			end;
	struct iov_iter		data;
571 572 573
	struct xfs_buftarg	*target = XFS_IS_REALTIME_INODE(ip) ?
					mp->m_rtdev_targp : mp->m_ddev_targp;

574
	/* DIO must be aligned to device logical sector size */
575
	if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
E
Eric Sandeen 已提交
576
		return -EINVAL;
577

578
	/* "unaligned" here means not aligned to a filesystem block */
579 580
	if ((iocb->ki_pos & mp->m_blockmask) ||
	    ((iocb->ki_pos + count) & mp->m_blockmask))
581 582
		unaligned_io = 1;

583 584 585 586 587 588 589 590
	/*
	 * We don't need to take an exclusive lock unless there page cache needs
	 * to be invalidated or unaligned IO is being executed. We don't need to
	 * consider the EOF extension case here because
	 * xfs_file_aio_write_checks() will relock the inode as necessary for
	 * EOF zeroing cases and fill out the new inode size as appropriate.
	 */
	if (unaligned_io || mapping->nrpages)
591
		iolock = XFS_IOLOCK_EXCL;
592
	else
593 594
		iolock = XFS_IOLOCK_SHARED;
	xfs_rw_ilock(ip, iolock);
595 596 597 598 599 600

	/*
	 * Recheck if there are cached pages that need invalidate after we got
	 * the iolock to protect against other threads adding new pages while
	 * we were waiting for the iolock.
	 */
601 602 603 604
	if (mapping->nrpages && iolock == XFS_IOLOCK_SHARED) {
		xfs_rw_iunlock(ip, iolock);
		iolock = XFS_IOLOCK_EXCL;
		xfs_rw_ilock(ip, iolock);
605
	}
606

607
	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
608
	if (ret)
609
		goto out;
610
	count = iov_iter_count(from);
611
	end = iocb->ki_pos + count - 1;
612

613
	/*
614
	 * See xfs_file_dio_aio_read() for why we do a full-file flush here.
615
	 */
616
	if (mapping->nrpages) {
617
		ret = filemap_write_and_wait(VFS_I(ip)->i_mapping);
618
		if (ret)
619
			goto out;
620
		/*
621 622 623
		 * Invalidate whole pages. This can return an error if we fail
		 * to invalidate a page, but this should never happen on XFS.
		 * Warn if it does fail.
624
		 */
625
		ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping);
626 627
		WARN_ON_ONCE(ret);
		ret = 0;
628 629
	}

630 631 632 633 634
	/*
	 * If we are doing unaligned IO, wait for all other IO to drain,
	 * otherwise demote the lock if we had to flush cached pages
	 */
	if (unaligned_io)
C
Christoph Hellwig 已提交
635
		inode_dio_wait(inode);
636
	else if (iolock == XFS_IOLOCK_EXCL) {
637
		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
638
		iolock = XFS_IOLOCK_SHARED;
639 640
	}

C
Christoph Hellwig 已提交
641
	trace_xfs_file_direct_write(ip, count, iocb->ki_pos);
642

643
	data = *from;
644 645 646
	ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
			xfs_get_blocks_direct, xfs_end_io_direct_write,
			NULL, DIO_ASYNC_EXTEND);
647 648 649 650

	/* see generic_file_direct_write() for why this is necessary */
	if (mapping->nrpages) {
		invalidate_inode_pages2_range(mapping,
651
					      iocb->ki_pos >> PAGE_SHIFT,
652
					      end >> PAGE_SHIFT);
653 654 655
	}

	if (ret > 0) {
656
		iocb->ki_pos += ret;
657 658
		iov_iter_advance(from, ret);
	}
659 660 661
out:
	xfs_rw_iunlock(ip, iolock);

662
	/*
663 664
	 * No fallback to buffered IO on errors for XFS, direct IO will either
	 * complete fully or fail.
665
	 */
666 667 668 669
	ASSERT(ret < 0 || ret == count);
	return ret;
}

670
static noinline ssize_t
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
xfs_file_dax_write(
	struct kiocb		*iocb,
	struct iov_iter		*from)
{
	struct address_space	*mapping = iocb->ki_filp->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	ssize_t			ret = 0;
	int			unaligned_io = 0;
	int			iolock;
	struct iov_iter		data;

	/* "unaligned" here means not aligned to a filesystem block */
	if ((iocb->ki_pos & mp->m_blockmask) ||
	    ((iocb->ki_pos + iov_iter_count(from)) & mp->m_blockmask)) {
		unaligned_io = 1;
		iolock = XFS_IOLOCK_EXCL;
	} else if (mapping->nrpages) {
		iolock = XFS_IOLOCK_EXCL;
	} else {
		iolock = XFS_IOLOCK_SHARED;
	}
	xfs_rw_ilock(ip, iolock);

	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
	if (ret)
		goto out;

	/*
	 * Yes, even DAX files can have page cache attached to them:  A zeroed
	 * page is inserted into the pagecache when we have to serve a write
	 * fault on a hole.  It should never be dirtied and can simply be
	 * dropped from the pagecache once we get real data for the page.
705 706 707 708 709 710 711
	 *
	 * XXX: This is racy against mmap, and there's nothing we can do about
	 * it. dax_do_io() should really do this invalidation internally as
	 * it will know if we've allocated over a holei for this specific IO and
	 * if so it needs to update the mapping tree and invalidate existing
	 * PTEs over the newly allocated range. Remove this invalidation when
	 * dax_do_io() is fixed up.
712
	 */
713
	if (mapping->nrpages) {
714 715 716 717 718
		loff_t end = iocb->ki_pos + iov_iter_count(from) - 1;

		ret = invalidate_inode_pages2_range(mapping,
						    iocb->ki_pos >> PAGE_SHIFT,
						    end >> PAGE_SHIFT);
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
		WARN_ON_ONCE(ret);
	}

	if (iolock == XFS_IOLOCK_EXCL && !unaligned_io) {
		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
		iolock = XFS_IOLOCK_SHARED;
	}

	trace_xfs_file_dax_write(ip, iov_iter_count(from), iocb->ki_pos);

	data = *from;
	ret = dax_do_io(iocb, inode, &data, xfs_get_blocks_direct,
			xfs_end_io_direct_write, 0);
	if (ret > 0) {
		iocb->ki_pos += ret;
		iov_iter_advance(from, ret);
	}
out:
	xfs_rw_iunlock(ip, iolock);
738 739 740
	return ret;
}

741
STATIC ssize_t
742
xfs_file_buffered_aio_write(
743
	struct kiocb		*iocb,
744
	struct iov_iter		*from)
745 746 747 748
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
749
	struct xfs_inode	*ip = XFS_I(inode);
750 751
	ssize_t			ret;
	int			enospc = 0;
752
	int			iolock = XFS_IOLOCK_EXCL;
753

754
	xfs_rw_ilock(ip, iolock);
755

756
	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
757
	if (ret)
758
		goto out;
759 760

	/* We can write back this queue in page reclaim */
761
	current->backing_dev_info = inode_to_bdi(inode);
762 763

write_retry:
C
Christoph Hellwig 已提交
764
	trace_xfs_file_buffered_write(ip, iov_iter_count(from), iocb->ki_pos);
765
	ret = iomap_file_buffered_write(iocb, from, &xfs_iomap_ops);
766
	if (likely(ret >= 0))
767
		iocb->ki_pos += ret;
768

769
	/*
770 771 772 773 774 775 776
	 * If we hit a space limit, try to free up some lingering preallocated
	 * space before returning an error. In the case of ENOSPC, first try to
	 * write back all dirty inodes to free up some of the excess reserved
	 * metadata space. This reduces the chances that the eofblocks scan
	 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
	 * also behaves as a filter to prevent too many eofblocks scans from
	 * running at the same time.
777
	 */
778 779 780 781 782 783 784
	if (ret == -EDQUOT && !enospc) {
		enospc = xfs_inode_free_quota_eofblocks(ip);
		if (enospc)
			goto write_retry;
	} else if (ret == -ENOSPC && !enospc) {
		struct xfs_eofblocks eofb = {0};

785
		enospc = 1;
D
Dave Chinner 已提交
786
		xfs_flush_inodes(ip->i_mount);
787 788 789
		eofb.eof_scan_owner = ip->i_ino; /* for locking */
		eofb.eof_flags = XFS_EOF_FLAGS_SYNC;
		xfs_icache_free_eofblocks(ip->i_mount, &eofb);
D
Dave Chinner 已提交
790
		goto write_retry;
791
	}
792

793
	current->backing_dev_info = NULL;
794 795
out:
	xfs_rw_iunlock(ip, iolock);
796 797 798 799
	return ret;
}

STATIC ssize_t
A
Al Viro 已提交
800
xfs_file_write_iter(
801
	struct kiocb		*iocb,
A
Al Viro 已提交
802
	struct iov_iter		*from)
803 804 805 806 807 808
{
	struct file		*file = iocb->ki_filp;
	struct address_space	*mapping = file->f_mapping;
	struct inode		*inode = mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	ssize_t			ret;
A
Al Viro 已提交
809
	size_t			ocount = iov_iter_count(from);
810

811
	XFS_STATS_INC(ip->i_mount, xs_write_calls);
812 813 814 815

	if (ocount == 0)
		return 0;

A
Al Viro 已提交
816 817
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
		return -EIO;
818

819 820 821
	if (IS_DAX(inode))
		ret = xfs_file_dax_write(iocb, from);
	else if (iocb->ki_flags & IOCB_DIRECT)
A
Al Viro 已提交
822
		ret = xfs_file_dio_aio_write(iocb, from);
823
	else
A
Al Viro 已提交
824
		ret = xfs_file_buffered_aio_write(iocb, from);
825

826
	if (ret > 0) {
827
		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
828

829
		/* Handle various SYNC-type writes */
830
		ret = generic_write_sync(iocb, ret);
831
	}
832
	return ret;
833 834
}

835 836 837 838 839
#define	XFS_FALLOC_FL_SUPPORTED						\
		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
		 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |	\
		 FALLOC_FL_INSERT_RANGE)

840 841
STATIC long
xfs_file_fallocate(
842 843 844 845
	struct file		*file,
	int			mode,
	loff_t			offset,
	loff_t			len)
846
{
847 848 849
	struct inode		*inode = file_inode(file);
	struct xfs_inode	*ip = XFS_I(inode);
	long			error;
850
	enum xfs_prealloc_flags	flags = 0;
851
	uint			iolock = XFS_IOLOCK_EXCL;
852
	loff_t			new_size = 0;
853
	bool			do_file_insert = 0;
854

855 856
	if (!S_ISREG(inode->i_mode))
		return -EINVAL;
857
	if (mode & ~XFS_FALLOC_FL_SUPPORTED)
858 859
		return -EOPNOTSUPP;

860
	xfs_ilock(ip, iolock);
861
	error = xfs_break_layouts(inode, &iolock, false);
862 863 864
	if (error)
		goto out_unlock;

865 866 867
	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
	iolock |= XFS_MMAPLOCK_EXCL;

868 869 870 871
	if (mode & FALLOC_FL_PUNCH_HOLE) {
		error = xfs_free_file_space(ip, offset, len);
		if (error)
			goto out_unlock;
872 873 874 875
	} else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
		unsigned blksize_mask = (1 << inode->i_blkbits) - 1;

		if (offset & blksize_mask || len & blksize_mask) {
D
Dave Chinner 已提交
876
			error = -EINVAL;
877 878 879
			goto out_unlock;
		}

880 881 882 883 884
		/*
		 * There is no need to overlap collapse range with EOF,
		 * in which case it is effectively a truncate operation
		 */
		if (offset + len >= i_size_read(inode)) {
D
Dave Chinner 已提交
885
			error = -EINVAL;
886 887 888
			goto out_unlock;
		}

889 890 891 892 893
		new_size = i_size_read(inode) - len;

		error = xfs_collapse_file_space(ip, offset, len);
		if (error)
			goto out_unlock;
894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
	} else if (mode & FALLOC_FL_INSERT_RANGE) {
		unsigned blksize_mask = (1 << inode->i_blkbits) - 1;

		new_size = i_size_read(inode) + len;
		if (offset & blksize_mask || len & blksize_mask) {
			error = -EINVAL;
			goto out_unlock;
		}

		/* check the new inode size does not wrap through zero */
		if (new_size > inode->i_sb->s_maxbytes) {
			error = -EFBIG;
			goto out_unlock;
		}

		/* Offset should be less than i_size */
		if (offset >= i_size_read(inode)) {
			error = -EINVAL;
			goto out_unlock;
		}
		do_file_insert = 1;
915
	} else {
916 917
		flags |= XFS_PREALLOC_SET;

918 919 920
		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
		    offset + len > i_size_read(inode)) {
			new_size = offset + len;
D
Dave Chinner 已提交
921
			error = inode_newsize_ok(inode, new_size);
922 923 924
			if (error)
				goto out_unlock;
		}
925

926 927 928 929 930
		if (mode & FALLOC_FL_ZERO_RANGE)
			error = xfs_zero_file_space(ip, offset, len);
		else
			error = xfs_alloc_file_space(ip, offset, len,
						     XFS_BMAPI_PREALLOC);
931 932 933 934
		if (error)
			goto out_unlock;
	}

935
	if (file->f_flags & O_DSYNC)
936 937 938
		flags |= XFS_PREALLOC_SYNC;

	error = xfs_update_prealloc_flags(ip, flags);
939 940 941 942 943 944 945 946 947
	if (error)
		goto out_unlock;

	/* Change file size if needed */
	if (new_size) {
		struct iattr iattr;

		iattr.ia_valid = ATTR_SIZE;
		iattr.ia_size = new_size;
948
		error = xfs_setattr_size(ip, &iattr);
949 950
		if (error)
			goto out_unlock;
951 952
	}

953 954 955 956 957 958 959 960 961
	/*
	 * Perform hole insertion now that the file size has been
	 * updated so that if we crash during the operation we don't
	 * leave shifted extents past EOF and hence losing access to
	 * the data that is contained within them.
	 */
	if (do_file_insert)
		error = xfs_insert_file_space(ip, offset, len);

962
out_unlock:
963
	xfs_iunlock(ip, iolock);
D
Dave Chinner 已提交
964
	return error;
965 966 967
}


L
Linus Torvalds 已提交
968
STATIC int
969
xfs_file_open(
L
Linus Torvalds 已提交
970
	struct inode	*inode,
971
	struct file	*file)
L
Linus Torvalds 已提交
972
{
973
	if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
L
Linus Torvalds 已提交
974
		return -EFBIG;
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
	if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
		return -EIO;
	return 0;
}

STATIC int
xfs_dir_open(
	struct inode	*inode,
	struct file	*file)
{
	struct xfs_inode *ip = XFS_I(inode);
	int		mode;
	int		error;

	error = xfs_file_open(inode, file);
	if (error)
		return error;

	/*
	 * If there are any blocks, read-ahead block 0 as we're almost
	 * certain to have the next operation be a read there.
	 */
997
	mode = xfs_ilock_data_map_shared(ip);
998
	if (ip->i_d.di_nextents > 0)
999
		xfs_dir3_data_readahead(ip, 0, -1);
1000 1001
	xfs_iunlock(ip, mode);
	return 0;
L
Linus Torvalds 已提交
1002 1003 1004
}

STATIC int
1005
xfs_file_release(
L
Linus Torvalds 已提交
1006 1007 1008
	struct inode	*inode,
	struct file	*filp)
{
D
Dave Chinner 已提交
1009
	return xfs_release(XFS_I(inode));
L
Linus Torvalds 已提交
1010 1011 1012
}

STATIC int
1013
xfs_file_readdir(
A
Al Viro 已提交
1014 1015
	struct file	*file,
	struct dir_context *ctx)
L
Linus Torvalds 已提交
1016
{
A
Al Viro 已提交
1017
	struct inode	*inode = file_inode(file);
1018
	xfs_inode_t	*ip = XFS_I(inode);
C
Christoph Hellwig 已提交
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
	size_t		bufsize;

	/*
	 * The Linux API doesn't pass down the total size of the buffer
	 * we read into down to the filesystem.  With the filldir concept
	 * it's not needed for correct information, but the XFS dir2 leaf
	 * code wants an estimate of the buffer size to calculate it's
	 * readahead window and size the buffers used for mapping to
	 * physical blocks.
	 *
	 * Try to give it an estimate that's good enough, maybe at some
	 * point we can change the ->readdir prototype to include the
E
Eric Sandeen 已提交
1031
	 * buffer size.  For now we use the current glibc buffer size.
C
Christoph Hellwig 已提交
1032
	 */
E
Eric Sandeen 已提交
1033
	bufsize = (size_t)min_t(loff_t, 32768, ip->i_d.di_size);
C
Christoph Hellwig 已提交
1034

1035
	return xfs_readdir(ip, ctx, bufsize);
L
Linus Torvalds 已提交
1036 1037
}

1038 1039
/*
 * This type is designed to indicate the type of offset we would like
1040
 * to search from page cache for xfs_seek_hole_data().
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
 */
enum {
	HOLE_OFF = 0,
	DATA_OFF,
};

/*
 * Lookup the desired type of offset from the given page.
 *
 * On success, return true and the offset argument will point to the
 * start of the region that was found.  Otherwise this function will
 * return false and keep the offset argument unchanged.
 */
STATIC bool
xfs_lookup_buffer_offset(
	struct page		*page,
	loff_t			*offset,
	unsigned int		type)
{
	loff_t			lastoff = page_offset(page);
	bool			found = false;
	struct buffer_head	*bh, *head;

	bh = head = page_buffers(page);
	do {
		/*
		 * Unwritten extents that have data in the page
		 * cache covering them can be identified by the
		 * BH_Unwritten state flag.  Pages with multiple
		 * buffers might have a mix of holes, data and
		 * unwritten extents - any buffer with valid
		 * data in it should have BH_Uptodate flag set
		 * on it.
		 */
		if (buffer_unwritten(bh) ||
		    buffer_uptodate(bh)) {
			if (type == DATA_OFF)
				found = true;
		} else {
			if (type == HOLE_OFF)
				found = true;
		}

		if (found) {
			*offset = lastoff;
			break;
		}
		lastoff += bh->b_size;
	} while ((bh = bh->b_this_page) != head);

	return found;
}

/*
 * This routine is called to find out and return a data or hole offset
 * from the page cache for unwritten extents according to the desired
1097
 * type for xfs_seek_hole_data().
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
 *
 * The argument offset is used to tell where we start to search from the
 * page cache.  Map is used to figure out the end points of the range to
 * lookup pages.
 *
 * Return true if the desired type of offset was found, and the argument
 * offset is filled with that address.  Otherwise, return false and keep
 * offset unchanged.
 */
STATIC bool
xfs_find_get_desired_pgoff(
	struct inode		*inode,
	struct xfs_bmbt_irec	*map,
	unsigned int		type,
	loff_t			*offset)
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	struct pagevec		pvec;
	pgoff_t			index;
	pgoff_t			end;
	loff_t			endoff;
	loff_t			startoff = *offset;
	loff_t			lastoff = startoff;
	bool			found = false;

	pagevec_init(&pvec, 0);

1126
	index = startoff >> PAGE_SHIFT;
1127
	endoff = XFS_FSB_TO_B(mp, map->br_startoff + map->br_blockcount);
1128
	end = endoff >> PAGE_SHIFT;
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
	do {
		int		want;
		unsigned	nr_pages;
		unsigned int	i;

		want = min_t(pgoff_t, end - index, PAGEVEC_SIZE);
		nr_pages = pagevec_lookup(&pvec, inode->i_mapping, index,
					  want);
		/*
		 * No page mapped into given range.  If we are searching holes
		 * and if this is the first time we got into the loop, it means
		 * that the given offset is landed in a hole, return it.
		 *
		 * If we have already stepped through some block buffers to find
		 * holes but they all contains data.  In this case, the last
		 * offset is already updated and pointed to the end of the last
		 * mapped page, if it does not reach the endpoint to search,
		 * that means there should be a hole between them.
		 */
		if (nr_pages == 0) {
			/* Data search found nothing */
			if (type == DATA_OFF)
				break;

			ASSERT(type == HOLE_OFF);
			if (lastoff == startoff || lastoff < endoff) {
				found = true;
				*offset = lastoff;
			}
			break;
		}

		/*
		 * At lease we found one page.  If this is the first time we
		 * step into the loop, and if the first page index offset is
		 * greater than the given search offset, a hole was found.
		 */
		if (type == HOLE_OFF && lastoff == startoff &&
		    lastoff < page_offset(pvec.pages[0])) {
			found = true;
			break;
		}

		for (i = 0; i < nr_pages; i++) {
			struct page	*page = pvec.pages[i];
			loff_t		b_offset;

			/*
			 * At this point, the page may be truncated or
			 * invalidated (changing page->mapping to NULL),
			 * or even swizzled back from swapper_space to tmpfs
			 * file mapping. However, page->index will not change
			 * because we have a reference on the page.
			 *
			 * Searching done if the page index is out of range.
			 * If the current offset is not reaches the end of
			 * the specified search range, there should be a hole
			 * between them.
			 */
			if (page->index > end) {
				if (type == HOLE_OFF && lastoff < endoff) {
					*offset = lastoff;
					found = true;
				}
				goto out;
			}

			lock_page(page);
			/*
			 * Page truncated or invalidated(page->mapping == NULL).
			 * We can freely skip it and proceed to check the next
			 * page.
			 */
			if (unlikely(page->mapping != inode->i_mapping)) {
				unlock_page(page);
				continue;
			}

			if (!page_has_buffers(page)) {
				unlock_page(page);
				continue;
			}

			found = xfs_lookup_buffer_offset(page, &b_offset, type);
			if (found) {
				/*
				 * The found offset may be less than the start
				 * point to search if this is the first time to
				 * come here.
				 */
				*offset = max_t(loff_t, startoff, b_offset);
				unlock_page(page);
				goto out;
			}

			/*
			 * We either searching data but nothing was found, or
			 * searching hole but found a data buffer.  In either
			 * case, probably the next page contains the desired
			 * things, update the last offset to it so.
			 */
			lastoff = page_offset(page) + PAGE_SIZE;
			unlock_page(page);
		}

		/*
		 * The number of returned pages less than our desired, search
		 * done.  In this case, nothing was found for searching data,
		 * but we found a hole behind the last offset.
		 */
		if (nr_pages < want) {
			if (type == HOLE_OFF) {
				*offset = lastoff;
				found = true;
			}
			break;
		}

		index = pvec.pages[i - 1]->index + 1;
		pagevec_release(&pvec);
	} while (index <= end);

out:
	pagevec_release(&pvec);
	return found;
}

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
/*
 * caller must lock inode with xfs_ilock_data_map_shared,
 * can we craft an appropriate ASSERT?
 *
 * end is because the VFS-level lseek interface is defined such that any
 * offset past i_size shall return -ENXIO, but we use this for quota code
 * which does not maintain i_size, and we want to SEEK_DATA past i_size.
 */
loff_t
__xfs_seek_hole_data(
	struct inode		*inode,
1267
	loff_t			start,
1268
	loff_t			end,
1269
	int			whence)
1270 1271 1272 1273 1274
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	loff_t			uninitialized_var(offset);
	xfs_fileoff_t		fsbno;
1275
	xfs_filblks_t		lastbno;
1276 1277
	int			error;

1278
	if (start >= end) {
D
Dave Chinner 已提交
1279
		error = -ENXIO;
1280
		goto out_error;
1281 1282 1283 1284 1285 1286
	}

	/*
	 * Try to read extents from the first block indicated
	 * by fsbno to the end block of the file.
	 */
1287
	fsbno = XFS_B_TO_FSBT(mp, start);
1288
	lastbno = XFS_B_TO_FSB(mp, end);
1289

1290 1291 1292 1293
	for (;;) {
		struct xfs_bmbt_irec	map[2];
		int			nmap = 2;
		unsigned int		i;
1294

1295
		error = xfs_bmapi_read(ip, fsbno, lastbno - fsbno, map, &nmap,
1296 1297
				       XFS_BMAPI_ENTIRE);
		if (error)
1298
			goto out_error;
1299

1300 1301
		/* No extents at given offset, must be beyond EOF */
		if (nmap == 0) {
D
Dave Chinner 已提交
1302
			error = -ENXIO;
1303
			goto out_error;
1304 1305 1306 1307 1308 1309
		}

		for (i = 0; i < nmap; i++) {
			offset = max_t(loff_t, start,
				       XFS_FSB_TO_B(mp, map[i].br_startoff));

1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
			/* Landed in the hole we wanted? */
			if (whence == SEEK_HOLE &&
			    map[i].br_startblock == HOLESTARTBLOCK)
				goto out;

			/* Landed in the data extent we wanted? */
			if (whence == SEEK_DATA &&
			    (map[i].br_startblock == DELAYSTARTBLOCK ||
			     (map[i].br_state == XFS_EXT_NORM &&
			      !isnullstartblock(map[i].br_startblock))))
1320 1321 1322
				goto out;

			/*
1323 1324
			 * Landed in an unwritten extent, try to search
			 * for hole or data from page cache.
1325 1326 1327
			 */
			if (map[i].br_state == XFS_EXT_UNWRITTEN) {
				if (xfs_find_get_desired_pgoff(inode, &map[i],
1328 1329
				      whence == SEEK_HOLE ? HOLE_OFF : DATA_OFF,
							&offset))
1330 1331 1332 1333 1334
					goto out;
			}
		}

		/*
1335 1336
		 * We only received one extent out of the two requested. This
		 * means we've hit EOF and didn't find what we are looking for.
1337
		 */
1338
		if (nmap == 1) {
1339 1340 1341 1342 1343 1344
			/*
			 * If we were looking for a hole, set offset to
			 * the end of the file (i.e., there is an implicit
			 * hole at the end of any file).
		 	 */
			if (whence == SEEK_HOLE) {
1345
				offset = end;
1346 1347 1348 1349 1350 1351
				break;
			}
			/*
			 * If we were looking for data, it's nowhere to be found
			 */
			ASSERT(whence == SEEK_DATA);
D
Dave Chinner 已提交
1352
			error = -ENXIO;
1353
			goto out_error;
1354 1355
		}

1356 1357 1358 1359
		ASSERT(i > 1);

		/*
		 * Nothing was found, proceed to the next round of search
1360
		 * if the next reading offset is not at or beyond EOF.
1361 1362 1363
		 */
		fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
		start = XFS_FSB_TO_B(mp, fsbno);
1364
		if (start >= end) {
1365
			if (whence == SEEK_HOLE) {
1366
				offset = end;
1367 1368 1369
				break;
			}
			ASSERT(whence == SEEK_DATA);
D
Dave Chinner 已提交
1370
			error = -ENXIO;
1371
			goto out_error;
1372
		}
1373 1374
	}

1375 1376
out:
	/*
1377
	 * If at this point we have found the hole we wanted, the returned
1378
	 * offset may be bigger than the file size as it may be aligned to
1379
	 * page boundary for unwritten extents.  We need to deal with this
1380 1381
	 * situation in particular.
	 */
1382
	if (whence == SEEK_HOLE)
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
		offset = min_t(loff_t, offset, end);

	return offset;

out_error:
	return error;
}

STATIC loff_t
xfs_seek_hole_data(
	struct file		*file,
	loff_t			start,
	int			whence)
{
	struct inode		*inode = file->f_mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	uint			lock;
	loff_t			offset, end;
	int			error = 0;

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

	lock = xfs_ilock_data_map_shared(ip);

	end = i_size_read(inode);
	offset = __xfs_seek_hole_data(inode, start, end, whence);
	if (offset < 0) {
		error = offset;
		goto out_unlock;
	}

J
Jie Liu 已提交
1416
	offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1417 1418

out_unlock:
1419
	xfs_iunlock(ip, lock);
1420 1421

	if (error)
D
Dave Chinner 已提交
1422
		return error;
1423 1424 1425 1426 1427 1428 1429
	return offset;
}

STATIC loff_t
xfs_file_llseek(
	struct file	*file,
	loff_t		offset,
1430
	int		whence)
1431
{
1432
	switch (whence) {
1433 1434 1435
	case SEEK_END:
	case SEEK_CUR:
	case SEEK_SET:
1436
		return generic_file_llseek(file, offset, whence);
1437
	case SEEK_HOLE:
1438
	case SEEK_DATA:
1439
		return xfs_seek_hole_data(file, offset, whence);
1440 1441 1442 1443 1444
	default:
		return -EINVAL;
	}
}

1445 1446 1447 1448 1449
/*
 * Locking for serialisation of IO during page faults. This results in a lock
 * ordering of:
 *
 * mmap_sem (MM)
1450
 *   sb_start_pagefault(vfs, freeze)
1451
 *     i_mmaplock (XFS - truncate serialisation)
1452 1453
 *       page_lock (MM)
 *         i_lock (XFS - extent map serialisation)
1454 1455
 */

1456 1457 1458 1459 1460
/*
 * mmap()d file has taken write protection fault and is being made writable. We
 * can set the page state up correctly for a writable page, which means we can
 * do correct delalloc accounting (ENOSPC checking!) and unwritten extent
 * mapping.
1461 1462
 */
STATIC int
1463
xfs_filemap_page_mkwrite(
1464 1465 1466
	struct vm_area_struct	*vma,
	struct vm_fault		*vmf)
{
1467
	struct inode		*inode = file_inode(vma->vm_file);
1468
	int			ret;
1469

1470
	trace_xfs_filemap_page_mkwrite(XFS_I(inode));
1471

1472
	sb_start_pagefault(inode->i_sb);
1473
	file_update_time(vma->vm_file);
1474
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1475

1476
	if (IS_DAX(inode)) {
R
Ross Zwisler 已提交
1477
		ret = dax_mkwrite(vma, vmf, xfs_get_blocks_dax_fault);
1478
	} else {
1479
		ret = iomap_page_mkwrite(vma, vmf, &xfs_iomap_ops);
1480 1481 1482 1483 1484 1485 1486
		ret = block_page_mkwrite_return(ret);
	}

	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	sb_end_pagefault(inode->i_sb);

	return ret;
1487 1488
}

1489
STATIC int
1490
xfs_filemap_fault(
1491 1492 1493
	struct vm_area_struct	*vma,
	struct vm_fault		*vmf)
{
1494
	struct inode		*inode = file_inode(vma->vm_file);
1495
	int			ret;
1496

1497
	trace_xfs_filemap_fault(XFS_I(inode));
1498

1499
	/* DAX can shortcut the normal fault path on write faults! */
1500
	if ((vmf->flags & FAULT_FLAG_WRITE) && IS_DAX(inode))
1501
		return xfs_filemap_page_mkwrite(vma, vmf);
1502

1503 1504 1505 1506 1507 1508 1509 1510
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
	if (IS_DAX(inode)) {
		/*
		 * we do not want to trigger unwritten extent conversion on read
		 * faults - that is unnecessary overhead and would also require
		 * changes to xfs_get_blocks_direct() to map unwritten extent
		 * ioend for conversion on read-only mappings.
		 */
R
Ross Zwisler 已提交
1511
		ret = dax_fault(vma, vmf, xfs_get_blocks_dax_fault);
1512 1513 1514
	} else
		ret = filemap_fault(vma, vmf);
	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1515

1516 1517 1518
	return ret;
}

1519 1520 1521 1522 1523 1524 1525
/*
 * Similar to xfs_filemap_fault(), the DAX fault path can call into here on
 * both read and write faults. Hence we need to handle both cases. There is no
 * ->pmd_mkwrite callout for huge pages, so we have a single function here to
 * handle both cases here. @flags carries the information on the type of fault
 * occuring.
 */
M
Matthew Wilcox 已提交
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
STATIC int
xfs_filemap_pmd_fault(
	struct vm_area_struct	*vma,
	unsigned long		addr,
	pmd_t			*pmd,
	unsigned int		flags)
{
	struct inode		*inode = file_inode(vma->vm_file);
	struct xfs_inode	*ip = XFS_I(inode);
	int			ret;

	if (!IS_DAX(inode))
		return VM_FAULT_FALLBACK;

	trace_xfs_filemap_pmd_fault(ip);

1542 1543 1544 1545 1546
	if (flags & FAULT_FLAG_WRITE) {
		sb_start_pagefault(inode->i_sb);
		file_update_time(vma->vm_file);
	}

M
Matthew Wilcox 已提交
1547
	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
R
Ross Zwisler 已提交
1548
	ret = dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_dax_fault);
M
Matthew Wilcox 已提交
1549 1550
	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);

1551 1552
	if (flags & FAULT_FLAG_WRITE)
		sb_end_pagefault(inode->i_sb);
M
Matthew Wilcox 已提交
1553 1554 1555 1556

	return ret;
}

1557 1558 1559
/*
 * pfn_mkwrite was originally inteneded to ensure we capture time stamp
 * updates on write faults. In reality, it's need to serialise against
1560 1561
 * truncate similar to page_mkwrite. Hence we cycle the XFS_MMAPLOCK_SHARED
 * to ensure we serialise the fault barrier in place.
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
 */
static int
xfs_filemap_pfn_mkwrite(
	struct vm_area_struct	*vma,
	struct vm_fault		*vmf)
{

	struct inode		*inode = file_inode(vma->vm_file);
	struct xfs_inode	*ip = XFS_I(inode);
	int			ret = VM_FAULT_NOPAGE;
	loff_t			size;

	trace_xfs_filemap_pfn_mkwrite(ip);

	sb_start_pagefault(inode->i_sb);
	file_update_time(vma->vm_file);

	/* check if the faulting page hasn't raced with truncate */
	xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
	if (vmf->pgoff >= size)
		ret = VM_FAULT_SIGBUS;
1584 1585
	else if (IS_DAX(inode))
		ret = dax_pfn_mkwrite(vma, vmf);
1586 1587
	xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
	sb_end_pagefault(inode->i_sb);
M
Matthew Wilcox 已提交
1588
	return ret;
1589

M
Matthew Wilcox 已提交
1590 1591
}

1592 1593
static const struct vm_operations_struct xfs_file_vm_ops = {
	.fault		= xfs_filemap_fault,
M
Matthew Wilcox 已提交
1594
	.pmd_fault	= xfs_filemap_pmd_fault,
1595 1596
	.map_pages	= filemap_map_pages,
	.page_mkwrite	= xfs_filemap_page_mkwrite,
1597
	.pfn_mkwrite	= xfs_filemap_pfn_mkwrite,
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
};

STATIC int
xfs_file_mmap(
	struct file	*filp,
	struct vm_area_struct *vma)
{
	file_accessed(filp);
	vma->vm_ops = &xfs_file_vm_ops;
	if (IS_DAX(file_inode(filp)))
M
Matthew Wilcox 已提交
1608
		vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
1609
	return 0;
1610 1611
}

1612
const struct file_operations xfs_file_operations = {
1613
	.llseek		= xfs_file_llseek,
A
Al Viro 已提交
1614
	.read_iter	= xfs_file_read_iter,
A
Al Viro 已提交
1615
	.write_iter	= xfs_file_write_iter,
1616
	.splice_read	= generic_file_splice_read,
A
Al Viro 已提交
1617
	.splice_write	= iter_file_splice_write,
1618
	.unlocked_ioctl	= xfs_file_ioctl,
L
Linus Torvalds 已提交
1619
#ifdef CONFIG_COMPAT
1620
	.compat_ioctl	= xfs_file_compat_ioctl,
L
Linus Torvalds 已提交
1621
#endif
1622 1623 1624 1625
	.mmap		= xfs_file_mmap,
	.open		= xfs_file_open,
	.release	= xfs_file_release,
	.fsync		= xfs_file_fsync,
1626
	.fallocate	= xfs_file_fallocate,
L
Linus Torvalds 已提交
1627 1628
};

1629
const struct file_operations xfs_dir_file_operations = {
1630
	.open		= xfs_dir_open,
L
Linus Torvalds 已提交
1631
	.read		= generic_read_dir,
1632
	.iterate_shared	= xfs_file_readdir,
1633
	.llseek		= generic_file_llseek,
1634
	.unlocked_ioctl	= xfs_file_ioctl,
1635
#ifdef CONFIG_COMPAT
1636
	.compat_ioctl	= xfs_file_compat_ioctl,
1637
#endif
1638
	.fsync		= xfs_dir_fsync,
L
Linus Torvalds 已提交
1639
};