xfs_aops.c 43.3 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_shared.h"
20 21 22
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
L
Linus Torvalds 已提交
23 24
#include "xfs_mount.h"
#include "xfs_inode.h"
25
#include "xfs_trans.h"
26
#include "xfs_inode_item.h"
27
#include "xfs_alloc.h"
L
Linus Torvalds 已提交
28 29
#include "xfs_error.h"
#include "xfs_iomap.h"
C
Christoph Hellwig 已提交
30
#include "xfs_trace.h"
31
#include "xfs_bmap.h"
D
Dave Chinner 已提交
32
#include "xfs_bmap_util.h"
33
#include "xfs_bmap_btree.h"
34
#include "xfs_reflink.h"
35
#include <linux/gfp.h>
L
Linus Torvalds 已提交
36
#include <linux/mpage.h>
37
#include <linux/pagevec.h>
L
Linus Torvalds 已提交
38 39
#include <linux/writeback.h>

40 41 42 43
/* flags for direct write completions */
#define XFS_DIO_FLAG_UNWRITTEN	(1 << 0)
#define XFS_DIO_FLAG_APPEND	(1 << 1)

44 45 46 47 48 49 50 51 52 53 54
/*
 * structure owned by writepages passed to individual writepage calls
 */
struct xfs_writepage_ctx {
	struct xfs_bmbt_irec    imap;
	bool			imap_valid;
	unsigned int		io_type;
	struct xfs_ioend	*ioend;
	sector_t		last_block;
};

C
Christoph Hellwig 已提交
55
void
56 57 58 59 60 61 62
xfs_count_page_state(
	struct page		*page,
	int			*delalloc,
	int			*unwritten)
{
	struct buffer_head	*bh, *head;

63
	*delalloc = *unwritten = 0;
64 65 66

	bh = head = page_buffers(page);
	do {
67
		if (buffer_unwritten(bh))
68 69 70 71 72 73
			(*unwritten) = 1;
		else if (buffer_delay(bh))
			(*delalloc) = 1;
	} while ((bh = bh->b_this_page) != head);
}

74
struct block_device *
C
Christoph Hellwig 已提交
75
xfs_find_bdev_for_inode(
C
Christoph Hellwig 已提交
76
	struct inode		*inode)
C
Christoph Hellwig 已提交
77
{
C
Christoph Hellwig 已提交
78
	struct xfs_inode	*ip = XFS_I(inode);
C
Christoph Hellwig 已提交
79 80
	struct xfs_mount	*mp = ip->i_mount;

81
	if (XFS_IS_REALTIME_INODE(ip))
C
Christoph Hellwig 已提交
82 83 84 85 86
		return mp->m_rtdev_targp->bt_bdev;
	else
		return mp->m_ddev_targp->bt_bdev;
}

87
/*
88 89 90
 * We're now finished for good with this page.  Update the page state via the
 * associated buffer_heads, paying attention to the start and end offsets that
 * we need to process on the page.
91 92 93 94 95 96
 *
 * Landmine Warning: bh->b_end_io() will call end_page_writeback() on the last
 * buffer in the IO. Once it does this, it is unsafe to access the bufferhead or
 * the page at all, as we may be racing with memory reclaim and it can free both
 * the bufferhead chain and the page as it will see the page as clean and
 * unused.
97 98 99 100 101 102 103 104
 */
static void
xfs_finish_page_writeback(
	struct inode		*inode,
	struct bio_vec		*bvec,
	int			error)
{
	unsigned int		end = bvec->bv_offset + bvec->bv_len - 1;
105
	struct buffer_head	*head, *bh, *next;
106
	unsigned int		off = 0;
107
	unsigned int		bsize;
108 109

	ASSERT(bvec->bv_offset < PAGE_SIZE);
110
	ASSERT((bvec->bv_offset & ((1 << inode->i_blkbits) - 1)) == 0);
111
	ASSERT(end < PAGE_SIZE);
112
	ASSERT((bvec->bv_len & ((1 << inode->i_blkbits) - 1)) == 0);
113 114 115

	bh = head = page_buffers(bvec->bv_page);

116
	bsize = bh->b_size;
117
	do {
118
		next = bh->b_this_page;
119 120 121 122 123 124
		if (off < bvec->bv_offset)
			goto next_bh;
		if (off > end)
			break;
		bh->b_end_io(bh, !error);
next_bh:
125 126
		off += bsize;
	} while ((bh = next) != head);
127 128 129 130 131 132
}

/*
 * We're now finished for good with this ioend structure.  Update the page
 * state, release holds on bios, and finally free up memory.  Do not use the
 * ioend after this.
133
 */
134 135
STATIC void
xfs_destroy_ioend(
136 137
	struct xfs_ioend	*ioend,
	int			error)
138
{
139
	struct inode		*inode = ioend->io_inode;
140
	struct bio		*last = ioend->io_bio;
141
	struct bio		*bio, *next;
142

143
	for (bio = &ioend->io_inline_bio; bio; bio = next) {
144 145 146
		struct bio_vec	*bvec;
		int		i;

147 148 149 150 151 152 153 154
		/*
		 * For the last bio, bi_private points to the ioend, so we
		 * need to explicitly end the iteration here.
		 */
		if (bio == last)
			next = NULL;
		else
			next = bio->bi_private;
C
Christoph Hellwig 已提交
155

156 157 158 159 160
		/* walk each page on bio, ending page IO on them */
		bio_for_each_segment_all(bvec, bio, i)
			xfs_finish_page_writeback(inode, bvec, error);

		bio_put(bio);
161
	}
162 163
}

C
Christoph Hellwig 已提交
164 165 166 167 168 169 170 171 172
/*
 * Fast and loose check if this write could update the on-disk inode size.
 */
static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)
{
	return ioend->io_offset + ioend->io_size >
		XFS_I(ioend->io_inode)->i_d.di_size;
}

173 174 175 176 177 178 179 180
STATIC int
xfs_setfilesize_trans_alloc(
	struct xfs_ioend	*ioend)
{
	struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;
	struct xfs_trans	*tp;
	int			error;

181 182
	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
	if (error)
183 184 185 186
		return error;

	ioend->io_append_trans = tp;

J
Jan Kara 已提交
187
	/*
188
	 * We may pass freeze protection with a transaction.  So tell lockdep
J
Jan Kara 已提交
189 190
	 * we released it.
	 */
191
	__sb_writers_release(ioend->io_inode->i_sb, SB_FREEZE_FS);
192 193 194 195 196 197 198 199
	/*
	 * We hand off the transaction to the completion thread now, so
	 * clear the flag here.
	 */
	current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS);
	return 0;
}

200
/*
201
 * Update on-disk file size now that data has been written to disk.
202
 */
203
STATIC int
204
__xfs_setfilesize(
205 206 207 208
	struct xfs_inode	*ip,
	struct xfs_trans	*tp,
	xfs_off_t		offset,
	size_t			size)
209 210 211
{
	xfs_fsize_t		isize;

212
	xfs_ilock(ip, XFS_ILOCK_EXCL);
213
	isize = xfs_new_eof(ip, offset + size);
214 215
	if (!isize) {
		xfs_iunlock(ip, XFS_ILOCK_EXCL);
216
		xfs_trans_cancel(tp);
217
		return 0;
218 219
	}

220
	trace_xfs_setfilesize(ip, offset, size);
221 222 223 224 225

	ip->i_d.di_size = isize;
	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);

226
	return xfs_trans_commit(tp);
227 228
}

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
int
xfs_setfilesize(
	struct xfs_inode	*ip,
	xfs_off_t		offset,
	size_t			size)
{
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_trans	*tp;
	int			error;

	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
	if (error)
		return error;

	return __xfs_setfilesize(ip, tp, offset, size);
}

246 247
STATIC int
xfs_setfilesize_ioend(
248 249
	struct xfs_ioend	*ioend,
	int			error)
250 251 252 253 254 255 256 257 258 259
{
	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
	struct xfs_trans	*tp = ioend->io_append_trans;

	/*
	 * The transaction may have been allocated in the I/O submission thread,
	 * thus we need to mark ourselves as being in a transaction manually.
	 * Similarly for freeze protection.
	 */
	current_set_flags_nested(&tp->t_pflags, PF_FSTRANS);
260
	__sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS);
261

262
	/* we abort the update if there was an IO error */
263
	if (error) {
264
		xfs_trans_cancel(tp);
265
		return error;
266 267
	}

268
	return __xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);
269 270
}

271
/*
272
 * IO write completion.
273 274
 */
STATIC void
275
xfs_end_io(
276
	struct work_struct *work)
277
{
278 279 280 281
	struct xfs_ioend	*ioend =
		container_of(work, struct xfs_ioend, io_work);
	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
	int			error = ioend->io_bio->bi_error;
282

283 284 285 286 287
	/*
	 * Set an error if the mount has shut down and proceed with end I/O
	 * processing so it can perform whatever cleanups are necessary.
	 */
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
288
		error = -EIO;
289

290 291 292
	/*
	 * For unwritten extents we need to issue transactions to convert a
	 * range to normal written extens after the data I/O has finished.
293 294 295
	 * Detecting and handling completion IO errors is done individually
	 * for each case as different cleanup operations need to be performed
	 * on error.
296
	 */
297
	if (ioend->io_type == XFS_IO_UNWRITTEN) {
298
		if (error)
299
			goto done;
300 301
		error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
						  ioend->io_size);
302
	} else if (ioend->io_append_trans) {
303
		error = xfs_setfilesize_ioend(ioend, error);
304
	} else {
305
		ASSERT(!xfs_ioend_is_append(ioend));
306
	}
307

308
done:
309
	xfs_destroy_ioend(ioend, error);
310 311
}

312 313 314
STATIC void
xfs_end_bio(
	struct bio		*bio)
315
{
316 317
	struct xfs_ioend	*ioend = bio->bi_private;
	struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;
318

319 320 321 322 323 324
	if (ioend->io_type == XFS_IO_UNWRITTEN)
		queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
	else if (ioend->io_append_trans)
		queue_work(mp->m_data_workqueue, &ioend->io_work);
	else
		xfs_destroy_ioend(ioend, bio->bi_error);
325 326
}

L
Linus Torvalds 已提交
327 328 329 330
STATIC int
xfs_map_blocks(
	struct inode		*inode,
	loff_t			offset,
C
Christoph Hellwig 已提交
331
	struct xfs_bmbt_irec	*imap,
332
	int			type)
L
Linus Torvalds 已提交
333
{
C
Christoph Hellwig 已提交
334 335
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
C
Christoph Hellwig 已提交
336
	ssize_t			count = 1 << inode->i_blkbits;
C
Christoph Hellwig 已提交
337 338 339 340 341 342
	xfs_fileoff_t		offset_fsb, end_fsb;
	int			error = 0;
	int			bmapi_flags = XFS_BMAPI_ENTIRE;
	int			nimaps = 1;

	if (XFS_FORCED_SHUTDOWN(mp))
E
Eric Sandeen 已提交
343
		return -EIO;
C
Christoph Hellwig 已提交
344

345
	ASSERT(type != XFS_IO_COW);
346
	if (type == XFS_IO_UNWRITTEN)
C
Christoph Hellwig 已提交
347
		bmapi_flags |= XFS_BMAPI_IGSTATE;
C
Christoph Hellwig 已提交
348

349
	xfs_ilock(ip, XFS_ILOCK_SHARED);
C
Christoph Hellwig 已提交
350 351
	ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE ||
	       (ip->i_df.if_flags & XFS_IFEXTENTS));
D
Dave Chinner 已提交
352
	ASSERT(offset <= mp->m_super->s_maxbytes);
C
Christoph Hellwig 已提交
353

D
Dave Chinner 已提交
354 355
	if (offset + count > mp->m_super->s_maxbytes)
		count = mp->m_super->s_maxbytes - offset;
C
Christoph Hellwig 已提交
356 357
	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
	offset_fsb = XFS_B_TO_FSBT(mp, offset);
D
Dave Chinner 已提交
358 359
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
				imap, &nimaps, bmapi_flags);
360 361 362 363 364 365 366
	/*
	 * Truncate an overwrite extent if there's a pending CoW
	 * reservation before the end of this extent.  This forces us
	 * to come back to writepage to take care of the CoW.
	 */
	if (nimaps && type == XFS_IO_OVERWRITE)
		xfs_reflink_trim_irec_to_next_cow(ip, offset_fsb, imap);
C
Christoph Hellwig 已提交
367
	xfs_iunlock(ip, XFS_ILOCK_SHARED);
C
Christoph Hellwig 已提交
368

C
Christoph Hellwig 已提交
369
	if (error)
D
Dave Chinner 已提交
370
		return error;
C
Christoph Hellwig 已提交
371

372
	if (type == XFS_IO_DELALLOC &&
C
Christoph Hellwig 已提交
373
	    (!nimaps || isnullstartblock(imap->br_startblock))) {
374 375
		error = xfs_iomap_write_allocate(ip, XFS_DATA_FORK, offset,
				imap);
C
Christoph Hellwig 已提交
376
		if (!error)
377
			trace_xfs_map_blocks_alloc(ip, offset, count, type, imap);
D
Dave Chinner 已提交
378
		return error;
C
Christoph Hellwig 已提交
379 380
	}

C
Christoph Hellwig 已提交
381
#ifdef DEBUG
382
	if (type == XFS_IO_UNWRITTEN) {
C
Christoph Hellwig 已提交
383 384 385 386 387 388 389 390
		ASSERT(nimaps);
		ASSERT(imap->br_startblock != HOLESTARTBLOCK);
		ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
	}
#endif
	if (nimaps)
		trace_xfs_map_blocks_found(ip, offset, count, type, imap);
	return 0;
L
Linus Torvalds 已提交
391 392
}

393
STATIC bool
394
xfs_imap_valid(
395
	struct inode		*inode,
C
Christoph Hellwig 已提交
396
	struct xfs_bmbt_irec	*imap,
397
	xfs_off_t		offset)
L
Linus Torvalds 已提交
398
{
399
	offset >>= inode->i_blkbits;
400

401 402
	return offset >= imap->br_startoff &&
		offset < imap->br_startoff + imap->br_blockcount;
L
Linus Torvalds 已提交
403 404
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
STATIC void
xfs_start_buffer_writeback(
	struct buffer_head	*bh)
{
	ASSERT(buffer_mapped(bh));
	ASSERT(buffer_locked(bh));
	ASSERT(!buffer_delay(bh));
	ASSERT(!buffer_unwritten(bh));

	mark_buffer_async_write(bh);
	set_buffer_uptodate(bh);
	clear_buffer_dirty(bh);
}

STATIC void
xfs_start_page_writeback(
	struct page		*page,
422
	int			clear_dirty)
423 424 425
{
	ASSERT(PageLocked(page));
	ASSERT(!PageWriteback(page));
426 427 428 429 430 431 432 433 434

	/*
	 * if the page was not fully cleaned, we need to ensure that the higher
	 * layers come back to it correctly. That means we need to keep the page
	 * dirty, and for WB_SYNC_ALL writeback we need to ensure the
	 * PAGECACHE_TAG_TOWRITE index mark is not removed so another attempt to
	 * write this page in this writeback sweep will be made.
	 */
	if (clear_dirty) {
435
		clear_page_dirty_for_io(page);
436 437 438 439
		set_page_writeback(page);
	} else
		set_page_writeback_keepwrite(page);

440 441 442
	unlock_page(page);
}

443
static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh)
444 445 446 447 448
{
	return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
}

/*
449 450 451 452 453 454
 * Submit the bio for an ioend. We are passed an ioend with a bio attached to
 * it, and we submit that bio. The ioend may be used for multiple bio
 * submissions, so we only want to allocate an append transaction for the ioend
 * once. In the case of multiple bio submission, each bio will take an IO
 * reference to the ioend to ensure that the ioend completion is only done once
 * all bios have been submitted and the ioend is really done.
455 456 457
 *
 * If @fail is non-zero, it means that we have a situation where some part of
 * the submission process has failed after we have marked paged for writeback
458 459 460
 * and unlocked them. In this situation, we need to fail the bio and ioend
 * rather than submit it to IO. This typically only happens on a filesystem
 * shutdown.
461
 */
462
STATIC int
463
xfs_submit_ioend(
464
	struct writeback_control *wbc,
465
	struct xfs_ioend	*ioend,
466
	int			status)
467
{
468 469
	/* Reserve log space if we might write beyond the on-disk inode size. */
	if (!status &&
470
	    ioend->io_type != XFS_IO_UNWRITTEN &&
471 472
	    xfs_ioend_is_append(ioend) &&
	    !ioend->io_append_trans)
473
		status = xfs_setfilesize_trans_alloc(ioend);
474

475 476
	ioend->io_bio->bi_private = ioend;
	ioend->io_bio->bi_end_io = xfs_end_bio;
M
Mike Christie 已提交
477 478
	bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE,
			 (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0);
479 480 481 482 483 484 485
	/*
	 * If we are failing the IO now, just mark the ioend with an
	 * error and finish it. This will run IO completion immediately
	 * as there is only one reference to the ioend at this point in
	 * time.
	 */
	if (status) {
486 487
		ioend->io_bio->bi_error = status;
		bio_endio(ioend->io_bio);
488 489
		return status;
	}
490

491
	submit_bio(ioend->io_bio);
492
	return 0;
493 494
}

495 496 497 498 499 500 501 502
static void
xfs_init_bio_from_bh(
	struct bio		*bio,
	struct buffer_head	*bh)
{
	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
	bio->bi_bdev = bh->b_bdev;
}
503

504 505 506 507 508 509 510 511 512
static struct xfs_ioend *
xfs_alloc_ioend(
	struct inode		*inode,
	unsigned int		type,
	xfs_off_t		offset,
	struct buffer_head	*bh)
{
	struct xfs_ioend	*ioend;
	struct bio		*bio;
513

514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
	bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, xfs_ioend_bioset);
	xfs_init_bio_from_bh(bio, bh);

	ioend = container_of(bio, struct xfs_ioend, io_inline_bio);
	INIT_LIST_HEAD(&ioend->io_list);
	ioend->io_type = type;
	ioend->io_inode = inode;
	ioend->io_size = 0;
	ioend->io_offset = offset;
	INIT_WORK(&ioend->io_work, xfs_end_io);
	ioend->io_append_trans = NULL;
	ioend->io_bio = bio;
	return ioend;
}

/*
 * Allocate a new bio, and chain the old bio to the new one.
 *
 * Note that we have to do perform the chaining in this unintuitive order
 * so that the bi_private linkage is set up in the right direction for the
 * traversal in xfs_destroy_ioend().
 */
static void
xfs_chain_bio(
	struct xfs_ioend	*ioend,
	struct writeback_control *wbc,
	struct buffer_head	*bh)
{
	struct bio *new;

	new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
	xfs_init_bio_from_bh(new, bh);

	bio_chain(ioend->io_bio, new);
	bio_get(ioend->io_bio);		/* for xfs_destroy_ioend */
M
Mike Christie 已提交
549 550
	bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE,
			  (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0);
551
	submit_bio(ioend->io_bio);
552
	ioend->io_bio = new;
553 554 555 556 557 558
}

/*
 * Test to see if we've been building up a completion structure for
 * earlier buffers -- if so, we try to append to this ioend if we
 * can, otherwise we finish off any current ioend and start another.
559 560
 * Return the ioend we finished off so that the caller can submit it
 * once it has finished processing the dirty page.
561 562 563 564 565
 */
STATIC void
xfs_add_to_ioend(
	struct inode		*inode,
	struct buffer_head	*bh,
566
	xfs_off_t		offset,
567
	struct xfs_writepage_ctx *wpc,
568
	struct writeback_control *wbc,
569
	struct list_head	*iolist)
570
{
571
	if (!wpc->ioend || wpc->io_type != wpc->ioend->io_type ||
572 573
	    bh->b_blocknr != wpc->last_block + 1 ||
	    offset != wpc->ioend->io_offset + wpc->ioend->io_size) {
574 575
		if (wpc->ioend)
			list_add(&wpc->ioend->io_list, iolist);
576
		wpc->ioend = xfs_alloc_ioend(inode, wpc->io_type, offset, bh);
577 578
	}

579 580 581 582 583 584
	/*
	 * If the buffer doesn't fit into the bio we need to allocate a new
	 * one.  This shouldn't happen more than once for a given buffer.
	 */
	while (xfs_bio_add_buffer(wpc->ioend->io_bio, bh) != bh->b_size)
		xfs_chain_bio(wpc->ioend, wbc, bh);
585

586 587
	wpc->ioend->io_size += bh->b_size;
	wpc->last_block = bh->b_blocknr;
588
	xfs_start_buffer_writeback(bh);
589 590
}

591 592
STATIC void
xfs_map_buffer(
C
Christoph Hellwig 已提交
593
	struct inode		*inode,
594
	struct buffer_head	*bh,
C
Christoph Hellwig 已提交
595
	struct xfs_bmbt_irec	*imap,
C
Christoph Hellwig 已提交
596
	xfs_off_t		offset)
597 598
{
	sector_t		bn;
599
	struct xfs_mount	*m = XFS_I(inode)->i_mount;
C
Christoph Hellwig 已提交
600 601
	xfs_off_t		iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
	xfs_daddr_t		iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
602

C
Christoph Hellwig 已提交
603 604
	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
605

606
	bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
607
	      ((offset - iomap_offset) >> inode->i_blkbits);
608

C
Christoph Hellwig 已提交
609
	ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
610 611 612 613 614

	bh->b_blocknr = bn;
	set_buffer_mapped(bh);
}

L
Linus Torvalds 已提交
615 616
STATIC void
xfs_map_at_offset(
C
Christoph Hellwig 已提交
617
	struct inode		*inode,
L
Linus Torvalds 已提交
618
	struct buffer_head	*bh,
C
Christoph Hellwig 已提交
619
	struct xfs_bmbt_irec	*imap,
C
Christoph Hellwig 已提交
620
	xfs_off_t		offset)
L
Linus Torvalds 已提交
621
{
C
Christoph Hellwig 已提交
622 623
	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
L
Linus Torvalds 已提交
624

C
Christoph Hellwig 已提交
625
	xfs_map_buffer(inode, bh, imap, offset);
L
Linus Torvalds 已提交
626 627
	set_buffer_mapped(bh);
	clear_buffer_delay(bh);
628
	clear_buffer_unwritten(bh);
L
Linus Torvalds 已提交
629 630 631
}

/*
632 633 634 635
 * Test if a given page contains at least one buffer of a given @type.
 * If @check_all_buffers is true, then we walk all the buffers in the page to
 * try to find one of the type passed in. If it is not set, then the caller only
 * needs to check the first buffer on the page for a match.
L
Linus Torvalds 已提交
636
 */
637
STATIC bool
638
xfs_check_page_type(
639
	struct page		*page,
640 641
	unsigned int		type,
	bool			check_all_buffers)
L
Linus Torvalds 已提交
642
{
643 644
	struct buffer_head	*bh;
	struct buffer_head	*head;
L
Linus Torvalds 已提交
645

646 647 648 649 650 651
	if (PageWriteback(page))
		return false;
	if (!page->mapping)
		return false;
	if (!page_has_buffers(page))
		return false;
L
Linus Torvalds 已提交
652

653 654 655 656 657 658
	bh = head = page_buffers(page);
	do {
		if (buffer_unwritten(bh)) {
			if (type == XFS_IO_UNWRITTEN)
				return true;
		} else if (buffer_delay(bh)) {
659
			if (type == XFS_IO_DELALLOC)
660 661
				return true;
		} else if (buffer_dirty(bh) && buffer_mapped(bh)) {
662
			if (type == XFS_IO_OVERWRITE)
663 664
				return true;
		}
L
Linus Torvalds 已提交
665

666 667 668 669
		/* If we are only checking the first buffer, we are done now. */
		if (!check_all_buffers)
			break;
	} while ((bh = bh->b_this_page) != head);
L
Linus Torvalds 已提交
670

671
	return false;
L
Linus Torvalds 已提交
672 673
}

674 675 676
STATIC void
xfs_vm_invalidatepage(
	struct page		*page,
677 678
	unsigned int		offset,
	unsigned int		length)
679
{
680 681 682
	trace_xfs_invalidatepage(page->mapping->host, page, offset,
				 length);
	block_invalidatepage(page, offset, length);
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
}

/*
 * If the page has delalloc buffers on it, we need to punch them out before we
 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
 * is done on that same region - the delalloc extent is returned when none is
 * supposed to be there.
 *
 * We prevent this by truncating away the delalloc regions on the page before
 * invalidating it. Because they are delalloc, we can do this without needing a
 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
 * truncation without a transaction as there is no space left for block
 * reservation (typically why we see a ENOSPC in writeback).
 *
 * This is not a performance critical path, so for now just do the punching a
 * buffer head at a time.
 */
STATIC void
xfs_aops_discard_page(
	struct page		*page)
{
	struct inode		*inode = page->mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct buffer_head	*bh, *head;
	loff_t			offset = page_offset(page);

710
	if (!xfs_check_page_type(page, XFS_IO_DELALLOC, true))
711 712
		goto out_invalidate;

713 714 715
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
		goto out_invalidate;

716
	xfs_alert(ip->i_mount,
717 718 719 720 721 722 723
		"page discard on page %p, inode 0x%llx, offset %llu.",
			page, ip->i_ino, offset);

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	bh = head = page_buffers(page);
	do {
		int		error;
724
		xfs_fileoff_t	start_fsb;
725 726 727 728

		if (!buffer_delay(bh))
			goto next_buffer;

729 730
		start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
		error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
731 732
		if (error) {
			/* something screwed, just bail */
733
			if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
734
				xfs_alert(ip->i_mount,
735
			"page discard unable to remove delalloc mapping.");
736
			}
737 738 739
			break;
		}
next_buffer:
740
		offset += 1 << inode->i_blkbits;
741 742 743 744 745

	} while ((bh = bh->b_this_page) != head);

	xfs_iunlock(ip, XFS_ILOCK_EXCL);
out_invalidate:
746
	xfs_vm_invalidatepage(page, 0, PAGE_SIZE);
747 748 749
	return;
}

750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
static int
xfs_map_cow(
	struct xfs_writepage_ctx *wpc,
	struct inode		*inode,
	loff_t			offset,
	unsigned int		*new_type)
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_bmbt_irec	imap;
	bool			is_cow = false, need_alloc = false;
	int			error;

	/*
	 * If we already have a valid COW mapping keep using it.
	 */
	if (wpc->io_type == XFS_IO_COW) {
		wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap, offset);
		if (wpc->imap_valid) {
			*new_type = XFS_IO_COW;
			return 0;
		}
	}

	/*
	 * Else we need to check if there is a COW mapping at this offset.
	 */
	xfs_ilock(ip, XFS_ILOCK_SHARED);
	is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap, &need_alloc);
	xfs_iunlock(ip, XFS_ILOCK_SHARED);

	if (!is_cow)
		return 0;

	/*
	 * And if the COW mapping has a delayed extent here we need to
	 * allocate real space for it now.
	 */
	if (need_alloc) {
		error = xfs_iomap_write_allocate(ip, XFS_COW_FORK, offset,
				&imap);
		if (error)
			return error;
	}

	wpc->io_type = *new_type = XFS_IO_COW;
	wpc->imap_valid = true;
	wpc->imap = imap;
	return 0;
}

800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
/*
 * We implement an immediate ioend submission policy here to avoid needing to
 * chain multiple ioends and hence nest mempool allocations which can violate
 * forward progress guarantees we need to provide. The current ioend we are
 * adding buffers to is cached on the writepage context, and if the new buffer
 * does not append to the cached ioend it will create a new ioend and cache that
 * instead.
 *
 * If a new ioend is created and cached, the old ioend is returned and queued
 * locally for submission once the entire page is processed or an error has been
 * detected.  While ioends are submitted immediately after they are completed,
 * batching optimisations are provided by higher level block plugging.
 *
 * At the end of a writeback pass, there will be a cached ioend remaining on the
 * writepage context that the caller will need to submit.
 */
816 817 818
static int
xfs_writepage_map(
	struct xfs_writepage_ctx *wpc,
819
	struct writeback_control *wbc,
820 821 822 823 824
	struct inode		*inode,
	struct page		*page,
	loff_t			offset,
	__uint64_t              end_offset)
{
825 826
	LIST_HEAD(submit_list);
	struct xfs_ioend	*ioend, *next;
827 828 829 830
	struct buffer_head	*bh, *head;
	ssize_t			len = 1 << inode->i_blkbits;
	int			error = 0;
	int			count = 0;
831
	int			uptodate = 1;
832
	unsigned int		new_type;
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852

	bh = head = page_buffers(page);
	offset = page_offset(page);
	do {
		if (offset >= end_offset)
			break;
		if (!buffer_uptodate(bh))
			uptodate = 0;

		/*
		 * set_page_dirty dirties all buffers in a page, independent
		 * of their state.  The dirty state however is entirely
		 * meaningless for holes (!mapped && uptodate), so skip
		 * buffers covering holes here.
		 */
		if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
			wpc->imap_valid = false;
			continue;
		}

853 854 855 856 857 858 859
		if (buffer_unwritten(bh))
			new_type = XFS_IO_UNWRITTEN;
		else if (buffer_delay(bh))
			new_type = XFS_IO_DELALLOC;
		else if (buffer_uptodate(bh))
			new_type = XFS_IO_OVERWRITE;
		else {
860 861 862 863 864 865 866 867 868 869 870 871
			if (PageUptodate(page))
				ASSERT(buffer_mapped(bh));
			/*
			 * This buffer is not uptodate and will not be
			 * written to disk.  Ensure that we will put any
			 * subsequent writeable buffers into a new
			 * ioend.
			 */
			wpc->imap_valid = false;
			continue;
		}

872 873 874 875 876 877 878 879 880 881 882
		if (xfs_is_reflink_inode(XFS_I(inode))) {
			error = xfs_map_cow(wpc, inode, offset, &new_type);
			if (error)
				goto out;
		}

		if (wpc->io_type != new_type) {
			wpc->io_type = new_type;
			wpc->imap_valid = false;
		}

883 884 885 886 887 888 889
		if (wpc->imap_valid)
			wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap,
							 offset);
		if (!wpc->imap_valid) {
			error = xfs_map_blocks(inode, offset, &wpc->imap,
					     wpc->io_type);
			if (error)
890
				goto out;
891 892 893 894 895 896 897
			wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap,
							 offset);
		}
		if (wpc->imap_valid) {
			lock_buffer(bh);
			if (wpc->io_type != XFS_IO_OVERWRITE)
				xfs_map_at_offset(inode, bh, &wpc->imap, offset);
898
			xfs_add_to_ioend(inode, bh, offset, wpc, wbc, &submit_list);
899 900 901 902 903 904 905 906
			count++;
		}

	} while (offset += len, ((bh = bh->b_this_page) != head));

	if (uptodate && bh == head)
		SetPageUptodate(page);

907
	ASSERT(wpc->ioend || list_empty(&submit_list));
908

909
out:
910
	/*
911 912 913 914 915 916 917 918 919
	 * On error, we have to fail the ioend here because we have locked
	 * buffers in the ioend. If we don't do this, we'll deadlock
	 * invalidating the page as that tries to lock the buffers on the page.
	 * Also, because we may have set pages under writeback, we have to make
	 * sure we run IO completion to mark the error state of the IO
	 * appropriately, so we can't cancel the ioend directly here. That means
	 * we have to mark this page as under writeback if we included any
	 * buffers from it in the ioend chain so that completion treats it
	 * correctly.
920
	 *
921 922 923 924 925
	 * If we didn't include the page in the ioend, the on error we can
	 * simply discard and unlock it as there are no other users of the page
	 * or it's buffers right now. The caller will still need to trigger
	 * submission of outstanding ioends on the writepage context so they are
	 * treated correctly on error.
926
	 */
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
	if (count) {
		xfs_start_page_writeback(page, !error);

		/*
		 * Preserve the original error if there was one, otherwise catch
		 * submission errors here and propagate into subsequent ioend
		 * submissions.
		 */
		list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
			int error2;

			list_del_init(&ioend->io_list);
			error2 = xfs_submit_ioend(wbc, ioend, error);
			if (error2 && !error)
				error = error2;
		}
	} else if (error) {
944 945 946
		xfs_aops_discard_page(page);
		ClearPageUptodate(page);
		unlock_page(page);
947 948 949 950 951 952 953 954
	} else {
		/*
		 * We can end up here with no error and nothing to write if we
		 * race with a partial page truncate on a sub-page block sized
		 * filesystem. In that case we need to mark the page clean.
		 */
		xfs_start_page_writeback(page, 1);
		end_page_writeback(page);
955
	}
956

957 958 959 960
	mapping_set_error(page->mapping, error);
	return error;
}

L
Linus Torvalds 已提交
961
/*
962 963 964 965 966 967
 * Write out a dirty page.
 *
 * For delalloc space on the page we need to allocate space and flush it.
 * For unwritten space on the page we need to start the conversion to
 * regular allocated space.
 * For any other dirty buffer heads on the page we should flush them.
L
Linus Torvalds 已提交
968 969
 */
STATIC int
970
xfs_do_writepage(
971
	struct page		*page,
972 973
	struct writeback_control *wbc,
	void			*data)
L
Linus Torvalds 已提交
974
{
975
	struct xfs_writepage_ctx *wpc = data;
976
	struct inode		*inode = page->mapping->host;
L
Linus Torvalds 已提交
977 978
	loff_t			offset;
	__uint64_t              end_offset;
979
	pgoff_t                 end_index;
980

981
	trace_xfs_writepage(inode, page, 0, 0);
982

983 984
	ASSERT(page_has_buffers(page));

985 986 987
	/*
	 * Refuse to write the page out if we are called from reclaim context.
	 *
988 989 990
	 * This avoids stack overflows when called from deeply used stacks in
	 * random callers for direct reclaim or memcg reclaim.  We explicitly
	 * allow reclaim from kswapd as the stack usage there is relatively low.
991
	 *
992 993
	 * This should never happen except in the case of a VM regression so
	 * warn about it.
994
	 */
995 996
	if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
			PF_MEMALLOC))
997
		goto redirty;
L
Linus Torvalds 已提交
998

999
	/*
1000 1001
	 * Given that we do not allow direct reclaim to call us, we should
	 * never be called while in a filesystem transaction.
1002
	 */
1003
	if (WARN_ON_ONCE(current->flags & PF_FSTRANS))
1004
		goto redirty;
1005

1006
	/*
1007 1008
	 * Is this page beyond the end of the file?
	 *
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
	 * The page index is less than the end_index, adjust the end_offset
	 * to the highest offset that this page should represent.
	 * -----------------------------------------------------
	 * |			file mapping	       | <EOF> |
	 * -----------------------------------------------------
	 * | Page ... | Page N-2 | Page N-1 |  Page N  |       |
	 * ^--------------------------------^----------|--------
	 * |     desired writeback range    |      see else    |
	 * ---------------------------------^------------------|
	 */
1019
	offset = i_size_read(inode);
1020
	end_index = offset >> PAGE_SHIFT;
1021
	if (page->index < end_index)
1022
		end_offset = (xfs_off_t)(page->index + 1) << PAGE_SHIFT;
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
	else {
		/*
		 * Check whether the page to write out is beyond or straddles
		 * i_size or not.
		 * -------------------------------------------------------
		 * |		file mapping		        | <EOF>  |
		 * -------------------------------------------------------
		 * | Page ... | Page N-2 | Page N-1 |  Page N   | Beyond |
		 * ^--------------------------------^-----------|---------
		 * |				    |      Straddles     |
		 * ---------------------------------^-----------|--------|
		 */
1035
		unsigned offset_into_page = offset & (PAGE_SIZE - 1);
1036 1037

		/*
1038 1039 1040 1041
		 * Skip the page if it is fully outside i_size, e.g. due to a
		 * truncate operation that is in progress. We must redirty the
		 * page so that reclaim stops reclaiming it. Otherwise
		 * xfs_vm_releasepage() is called on it and gets confused.
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
		 *
		 * Note that the end_index is unsigned long, it would overflow
		 * if the given offset is greater than 16TB on 32-bit system
		 * and if we do check the page is fully outside i_size or not
		 * via "if (page->index >= end_index + 1)" as "end_index + 1"
		 * will be evaluated to 0.  Hence this page will be redirtied
		 * and be written out repeatedly which would result in an
		 * infinite loop, the user program that perform this operation
		 * will hang.  Instead, we can verify this situation by checking
		 * if the page to write is totally beyond the i_size or if it's
		 * offset is just equal to the EOF.
1053
		 */
1054 1055
		if (page->index > end_index ||
		    (page->index == end_index && offset_into_page == 0))
1056
			goto redirty;
1057 1058 1059 1060 1061

		/*
		 * The page straddles i_size.  It must be zeroed out on each
		 * and every writepage invocation because it may be mmapped.
		 * "A file is mapped in multiples of the page size.  For a file
1062
		 * that is not a multiple of the page size, the remaining
1063 1064 1065
		 * memory is zeroed when mapped, and writes to that region are
		 * not written out to the file."
		 */
1066
		zero_user_segment(page, offset_into_page, PAGE_SIZE);
1067 1068 1069

		/* Adjust the end_offset to the end of file */
		end_offset = offset;
L
Linus Torvalds 已提交
1070 1071
	}

1072
	return xfs_writepage_map(wpc, wbc, inode, page, offset, end_offset);
1073

1074
redirty:
1075 1076 1077 1078 1079
	redirty_page_for_writepage(wbc, page);
	unlock_page(page);
	return 0;
}

1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
STATIC int
xfs_vm_writepage(
	struct page		*page,
	struct writeback_control *wbc)
{
	struct xfs_writepage_ctx wpc = {
		.io_type = XFS_IO_INVALID,
	};
	int			ret;

	ret = xfs_do_writepage(page, wbc, &wpc);
1091 1092 1093
	if (wpc.ioend)
		ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
	return ret;
1094 1095
}

1096 1097 1098 1099 1100
STATIC int
xfs_vm_writepages(
	struct address_space	*mapping,
	struct writeback_control *wbc)
{
1101 1102 1103 1104 1105
	struct xfs_writepage_ctx wpc = {
		.io_type = XFS_IO_INVALID,
	};
	int			ret;

1106
	xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1107 1108 1109 1110
	if (dax_mapping(mapping))
		return dax_writeback_mapping_range(mapping,
				xfs_find_bdev_for_inode(mapping->host), wbc);

1111
	ret = write_cache_pages(mapping, wbc, xfs_do_writepage, &wpc);
1112 1113 1114
	if (wpc.ioend)
		ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
	return ret;
1115 1116
}

1117 1118
/*
 * Called to move a page into cleanable state - and from there
1119
 * to be released. The page should already be clean. We always
1120 1121
 * have buffer heads in this call.
 *
1122
 * Returns 1 if the page is ok to release, 0 otherwise.
1123 1124
 */
STATIC int
1125
xfs_vm_releasepage(
1126 1127 1128
	struct page		*page,
	gfp_t			gfp_mask)
{
1129
	int			delalloc, unwritten;
1130

1131
	trace_xfs_releasepage(page->mapping->host, page, 0, 0);
1132

1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	/*
	 * mm accommodates an old ext3 case where clean pages might not have had
	 * the dirty bit cleared. Thus, it can send actual dirty pages to
	 * ->releasepage() via shrink_active_list(). Conversely,
	 * block_invalidatepage() can send pages that are still marked dirty
	 * but otherwise have invalidated buffers.
	 *
	 * We've historically freed buffers on the latter. Instead, quietly
	 * filter out all dirty pages to avoid spurious buffer state warnings.
	 * This can likely be removed once shrink_active_list() is fixed.
	 */
	if (PageDirty(page))
		return 0;

1147
	xfs_count_page_state(page, &delalloc, &unwritten);
1148

1149
	if (WARN_ON_ONCE(delalloc))
1150
		return 0;
1151
	if (WARN_ON_ONCE(unwritten))
1152 1153 1154 1155 1156
		return 0;

	return try_to_free_buffers(page);
}

1157
/*
1158 1159
 * When we map a DIO buffer, we may need to pass flags to
 * xfs_end_io_direct_write to tell it what kind of write IO we are doing.
1160 1161 1162 1163 1164 1165 1166
 *
 * Note that for DIO, an IO to the highest supported file block offset (i.e.
 * 2^63 - 1FSB bytes) will result in the offset + count overflowing a signed 64
 * bit variable. Hence if we see this overflow, we have to assume that the IO is
 * extending the file size. We won't know for sure until IO completion is run
 * and the actual max write offset is communicated to the IO completion
 * routine.
1167 1168 1169 1170 1171 1172
 */
static void
xfs_map_direct(
	struct inode		*inode,
	struct buffer_head	*bh_result,
	struct xfs_bmbt_irec	*imap,
1173
	xfs_off_t		offset)
1174
{
1175
	uintptr_t		*flags = (uintptr_t *)&bh_result->b_private;
1176 1177
	xfs_off_t		size = bh_result->b_size;

1178 1179
	trace_xfs_get_blocks_map_direct(XFS_I(inode), offset, size,
		ISUNWRITTEN(imap) ? XFS_IO_UNWRITTEN : XFS_IO_OVERWRITE, imap);
1180

1181 1182 1183 1184 1185
	if (ISUNWRITTEN(imap)) {
		*flags |= XFS_DIO_FLAG_UNWRITTEN;
		set_buffer_defer_completion(bh_result);
	} else if (offset + size > i_size_read(inode) || offset + size < 0) {
		*flags |= XFS_DIO_FLAG_APPEND;
1186
		set_buffer_defer_completion(bh_result);
1187 1188 1189
	}
}

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
/*
 * If this is O_DIRECT or the mpage code calling tell them how large the mapping
 * is, so that we can avoid repeated get_blocks calls.
 *
 * If the mapping spans EOF, then we have to break the mapping up as the mapping
 * for blocks beyond EOF must be marked new so that sub block regions can be
 * correctly zeroed. We can't do this for mappings within EOF unless the mapping
 * was just allocated or is unwritten, otherwise the callers would overwrite
 * existing data with zeros. Hence we have to split the mapping into a range up
 * to and including EOF, and a second mapping for beyond EOF.
 */
static void
xfs_map_trim_size(
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	struct xfs_bmbt_irec	*imap,
	xfs_off_t		offset,
	ssize_t			size)
{
	xfs_off_t		mapping_size;

	mapping_size = imap->br_startoff + imap->br_blockcount - iblock;
	mapping_size <<= inode->i_blkbits;

	ASSERT(mapping_size > 0);
	if (mapping_size > size)
		mapping_size = size;
	if (offset < i_size_read(inode) &&
	    offset + mapping_size >= i_size_read(inode)) {
		/* limit mapping to block that spans EOF */
		mapping_size = roundup_64(i_size_read(inode) - offset,
					  1 << inode->i_blkbits);
	}
	if (mapping_size > LONG_MAX)
		mapping_size = LONG_MAX;

	bh_result->b_size = mapping_size;
}

L
Linus Torvalds 已提交
1230
STATIC int
1231
__xfs_get_blocks(
L
Linus Torvalds 已提交
1232 1233 1234 1235
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create,
1236 1237
	bool			direct,
	bool			dax_fault)
L
Linus Torvalds 已提交
1238
{
C
Christoph Hellwig 已提交
1239 1240 1241 1242 1243
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb, end_fsb;
	int			error = 0;
	int			lockmode = 0;
C
Christoph Hellwig 已提交
1244
	struct xfs_bmbt_irec	imap;
C
Christoph Hellwig 已提交
1245
	int			nimaps = 1;
1246 1247
	xfs_off_t		offset;
	ssize_t			size;
C
Christoph Hellwig 已提交
1248
	int			new = 0;
C
Christoph Hellwig 已提交
1249

1250 1251
	BUG_ON(create && !direct);

C
Christoph Hellwig 已提交
1252
	if (XFS_FORCED_SHUTDOWN(mp))
E
Eric Sandeen 已提交
1253
		return -EIO;
L
Linus Torvalds 已提交
1254

1255
	offset = (xfs_off_t)iblock << inode->i_blkbits;
1256 1257
	ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
	size = bh_result->b_size;
1258

1259
	if (!create && offset >= i_size_read(inode))
1260 1261
		return 0;

1262 1263
	/*
	 * Direct I/O is usually done on preallocated files, so try getting
1264
	 * a block mapping without an exclusive lock first.
1265
	 */
1266
	lockmode = xfs_ilock_data_map_shared(ip);
1267

D
Dave Chinner 已提交
1268 1269 1270
	ASSERT(offset <= mp->m_super->s_maxbytes);
	if (offset + size > mp->m_super->s_maxbytes)
		size = mp->m_super->s_maxbytes - offset;
C
Christoph Hellwig 已提交
1271 1272 1273
	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
	offset_fsb = XFS_B_TO_FSBT(mp, offset);

D
Dave Chinner 已提交
1274 1275
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
				&imap, &nimaps, XFS_BMAPI_ENTIRE);
L
Linus Torvalds 已提交
1276
	if (error)
C
Christoph Hellwig 已提交
1277 1278
		goto out_unlock;

1279
	/* for DAX, we convert unwritten extents directly */
C
Christoph Hellwig 已提交
1280 1281 1282
	if (create &&
	    (!nimaps ||
	     (imap.br_startblock == HOLESTARTBLOCK ||
1283 1284
	      imap.br_startblock == DELAYSTARTBLOCK) ||
	     (IS_DAX(inode) && ISUNWRITTEN(&imap)))) {
1285 1286 1287 1288 1289 1290
		/*
		 * xfs_iomap_write_direct() expects the shared lock. It
		 * is unlocked on return.
		 */
		if (lockmode == XFS_ILOCK_EXCL)
			xfs_ilock_demote(ip, lockmode);
1291

1292 1293 1294 1295 1296
		error = xfs_iomap_write_direct(ip, offset, size,
					       &imap, nimaps);
		if (error)
			return error;
		new = 1;
1297

1298 1299 1300
		trace_xfs_get_blocks_alloc(ip, offset, size,
				ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN
						   : XFS_IO_DELALLOC, &imap);
C
Christoph Hellwig 已提交
1301
	} else if (nimaps) {
1302 1303 1304
		trace_xfs_get_blocks_found(ip, offset, size,
				ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN
						   : XFS_IO_OVERWRITE, &imap);
1305
		xfs_iunlock(ip, lockmode);
C
Christoph Hellwig 已提交
1306 1307 1308 1309
	} else {
		trace_xfs_get_blocks_notfound(ip, offset, size);
		goto out_unlock;
	}
L
Linus Torvalds 已提交
1310

1311 1312 1313 1314 1315 1316
	if (IS_DAX(inode) && create) {
		ASSERT(!ISUNWRITTEN(&imap));
		/* zeroing is not needed at a higher layer */
		new = 0;
	}

1317
	/* trim mapping down to size requested */
1318
	xfs_map_trim_size(inode, iblock, bh_result, &imap, offset, size);
1319

1320 1321 1322 1323
	/*
	 * For unwritten extents do not report a disk address in the buffered
	 * read case (treat as if we're reading into a hole).
	 */
C
Christoph Hellwig 已提交
1324
	if (imap.br_startblock != HOLESTARTBLOCK &&
1325 1326 1327 1328
	    imap.br_startblock != DELAYSTARTBLOCK &&
	    (create || !ISUNWRITTEN(&imap))) {
		xfs_map_buffer(inode, bh_result, &imap, offset);
		if (ISUNWRITTEN(&imap))
L
Linus Torvalds 已提交
1329
			set_buffer_unwritten(bh_result);
1330
		/* direct IO needs special help */
1331
		if (create) {
1332 1333 1334 1335 1336
			if (dax_fault)
				ASSERT(!ISUNWRITTEN(&imap));
			else
				xfs_map_direct(inode, bh_result, &imap, offset);
		}
L
Linus Torvalds 已提交
1337 1338
	}

1339 1340 1341 1342
	/*
	 * If this is a realtime file, data may be on a different device.
	 * to that pointed to from the buffer_head b_bdev currently.
	 */
C
Christoph Hellwig 已提交
1343
	bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
L
Linus Torvalds 已提交
1344

1345
	/*
1346 1347 1348 1349 1350 1351 1352
	 * If we previously allocated a block out beyond eof and we are now
	 * coming back to use it then we will need to flag it as new even if it
	 * has a disk address.
	 *
	 * With sub-block writes into unwritten extents we also need to mark
	 * the buffer as new so that the unwritten parts of the buffer gets
	 * correctly zeroed.
L
Linus Torvalds 已提交
1353 1354 1355
	 */
	if (create &&
	    ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1356
	     (offset >= i_size_read(inode)) ||
C
Christoph Hellwig 已提交
1357
	     (new || ISUNWRITTEN(&imap))))
L
Linus Torvalds 已提交
1358 1359
		set_buffer_new(bh_result);

1360
	BUG_ON(direct && imap.br_startblock == DELAYSTARTBLOCK);
L
Linus Torvalds 已提交
1361 1362

	return 0;
C
Christoph Hellwig 已提交
1363 1364 1365

out_unlock:
	xfs_iunlock(ip, lockmode);
D
Dave Chinner 已提交
1366
	return error;
L
Linus Torvalds 已提交
1367 1368 1369
}

int
1370
xfs_get_blocks(
L
Linus Torvalds 已提交
1371 1372 1373 1374 1375
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create)
{
1376
	return __xfs_get_blocks(inode, iblock, bh_result, create, false, false);
L
Linus Torvalds 已提交
1377 1378
}

1379
int
1380
xfs_get_blocks_direct(
L
Linus Torvalds 已提交
1381 1382 1383 1384 1385
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create)
{
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
	return __xfs_get_blocks(inode, iblock, bh_result, create, true, false);
}

int
xfs_get_blocks_dax_fault(
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create)
{
	return __xfs_get_blocks(inode, iblock, bh_result, create, true, true);
L
Linus Torvalds 已提交
1397 1398
}

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
/*
 * Complete a direct I/O write request.
 *
 * xfs_map_direct passes us some flags in the private data to tell us what to
 * do.  If no flags are set, then the write IO is an overwrite wholly within
 * the existing allocated file size and so there is nothing for us to do.
 *
 * Note that in this case the completion can be called in interrupt context,
 * whereas if we have flags set we will always be called in task context
 * (i.e. from a workqueue).
 */
1410
int
1411 1412
xfs_end_io_direct_write(
	struct kiocb		*iocb,
1413
	loff_t			offset,
1414 1415
	ssize_t			size,
	void			*private)
1416
{
1417 1418 1419 1420
	struct inode		*inode = file_inode(iocb->ki_filp);
	struct xfs_inode	*ip = XFS_I(inode);
	uintptr_t		flags = (uintptr_t)private;
	int			error = 0;
1421

1422
	trace_xfs_end_io_direct_write(ip, offset, size);
1423

1424
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
1425
		return -EIO;
1426

1427 1428
	if (size <= 0)
		return size;
1429

1430
	/*
1431
	 * The flags tell us whether we are doing unwritten extent conversions
1432 1433
	 * or an append transaction that updates the on-disk file size. These
	 * cases are the only cases where we should *potentially* be needing
1434
	 * to update the VFS inode size.
1435 1436 1437 1438 1439 1440 1441
	 */
	if (flags == 0) {
		ASSERT(offset + size <= i_size_read(inode));
		return 0;
	}

	/*
1442
	 * We need to update the in-core inode size here so that we don't end up
1443 1444 1445
	 * with the on-disk inode size being outside the in-core inode size. We
	 * have no other method of updating EOF for AIO, so always do it here
	 * if necessary.
1446 1447 1448 1449 1450
	 *
	 * We need to lock the test/set EOF update as we can be racing with
	 * other IO completions here to update the EOF. Failing to serialise
	 * here can result in EOF moving backwards and Bad Things Happen when
	 * that occurs.
1451
	 */
1452
	spin_lock(&ip->i_flags_lock);
1453 1454
	if (offset + size > i_size_read(inode))
		i_size_write(inode, offset + size);
1455
	spin_unlock(&ip->i_flags_lock);
1456

1457 1458
	if (flags & XFS_DIO_FLAG_UNWRITTEN) {
		trace_xfs_end_io_direct_write_unwritten(ip, offset, size);
1459

1460 1461 1462
		error = xfs_iomap_write_unwritten(ip, offset, size);
	} else if (flags & XFS_DIO_FLAG_APPEND) {
		trace_xfs_end_io_direct_write_append(ip, offset, size);
1463

1464
		error = xfs_setfilesize(ip, offset, size);
1465 1466
	}

1467
	return error;
1468 1469
}

1470 1471
STATIC ssize_t
xfs_vm_direct_IO(
D
Dave Chinner 已提交
1472
	struct kiocb		*iocb,
1473
	struct iov_iter		*iter)
D
Dave Chinner 已提交
1474
{
1475
	/*
1476
	 * We just need the method present so that open/fcntl allow direct I/O.
1477
	 */
1478
	return -EINVAL;
1479
}
L
Linus Torvalds 已提交
1480 1481

STATIC sector_t
1482
xfs_vm_bmap(
L
Linus Torvalds 已提交
1483 1484 1485 1486
	struct address_space	*mapping,
	sector_t		block)
{
	struct inode		*inode = (struct inode *)mapping->host;
1487
	struct xfs_inode	*ip = XFS_I(inode);
L
Linus Torvalds 已提交
1488

C
Christoph Hellwig 已提交
1489
	trace_xfs_vm_bmap(XFS_I(inode));
1490
	xfs_ilock(ip, XFS_IOLOCK_SHARED);
D
Dave Chinner 已提交
1491
	filemap_write_and_wait(mapping);
1492
	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
1493
	return generic_block_bmap(mapping, block, xfs_get_blocks);
L
Linus Torvalds 已提交
1494 1495 1496
}

STATIC int
1497
xfs_vm_readpage(
L
Linus Torvalds 已提交
1498 1499 1500
	struct file		*unused,
	struct page		*page)
{
1501
	trace_xfs_vm_readpage(page->mapping->host, 1);
1502
	return mpage_readpage(page, xfs_get_blocks);
L
Linus Torvalds 已提交
1503 1504 1505
}

STATIC int
1506
xfs_vm_readpages(
L
Linus Torvalds 已提交
1507 1508 1509 1510 1511
	struct file		*unused,
	struct address_space	*mapping,
	struct list_head	*pages,
	unsigned		nr_pages)
{
1512
	trace_xfs_vm_readpages(mapping->host, nr_pages);
1513
	return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
L
Linus Torvalds 已提交
1514 1515
}

1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
/*
 * This is basically a copy of __set_page_dirty_buffers() with one
 * small tweak: buffers beyond EOF do not get marked dirty. If we mark them
 * dirty, we'll never be able to clean them because we don't write buffers
 * beyond EOF, and that means we can't invalidate pages that span EOF
 * that have been marked dirty. Further, the dirty state can leak into
 * the file interior if the file is extended, resulting in all sorts of
 * bad things happening as the state does not match the underlying data.
 *
 * XXX: this really indicates that bufferheads in XFS need to die. Warts like
 * this only exist because of bufferheads and how the generic code manages them.
 */
STATIC int
xfs_vm_set_page_dirty(
	struct page		*page)
{
	struct address_space	*mapping = page->mapping;
	struct inode		*inode = mapping->host;
	loff_t			end_offset;
	loff_t			offset;
	int			newly_dirty;

	if (unlikely(!mapping))
		return !TestSetPageDirty(page);

	end_offset = i_size_read(inode);
	offset = page_offset(page);

	spin_lock(&mapping->private_lock);
	if (page_has_buffers(page)) {
		struct buffer_head *head = page_buffers(page);
		struct buffer_head *bh = head;

		do {
			if (offset < end_offset)
				set_buffer_dirty(bh);
			bh = bh->b_this_page;
			offset += 1 << inode->i_blkbits;
		} while (bh != head);
	}
1556
	/*
1557 1558
	 * Lock out page->mem_cgroup migration to keep PageDirty
	 * synchronized with per-memcg dirty page counters.
1559
	 */
J
Johannes Weiner 已提交
1560
	lock_page_memcg(page);
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	newly_dirty = !TestSetPageDirty(page);
	spin_unlock(&mapping->private_lock);

	if (newly_dirty) {
		/* sigh - __set_page_dirty() is static, so copy it here, too */
		unsigned long flags;

		spin_lock_irqsave(&mapping->tree_lock, flags);
		if (page->mapping) {	/* Race with truncate? */
			WARN_ON_ONCE(!PageUptodate(page));
J
Johannes Weiner 已提交
1571
			account_page_dirtied(page, mapping);
1572 1573 1574 1575 1576
			radix_tree_tag_set(&mapping->page_tree,
					page_index(page), PAGECACHE_TAG_DIRTY);
		}
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
	}
J
Johannes Weiner 已提交
1577
	unlock_page_memcg(page);
1578 1579
	if (newly_dirty)
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1580 1581 1582
	return newly_dirty;
}

1583
const struct address_space_operations xfs_address_space_operations = {
1584 1585 1586
	.readpage		= xfs_vm_readpage,
	.readpages		= xfs_vm_readpages,
	.writepage		= xfs_vm_writepage,
1587
	.writepages		= xfs_vm_writepages,
1588
	.set_page_dirty		= xfs_vm_set_page_dirty,
1589 1590
	.releasepage		= xfs_vm_releasepage,
	.invalidatepage		= xfs_vm_invalidatepage,
1591 1592
	.bmap			= xfs_vm_bmap,
	.direct_IO		= xfs_vm_direct_IO,
1593
	.migratepage		= buffer_migrate_page,
1594
	.is_partially_uptodate  = block_is_partially_uptodate,
1595
	.error_remove_page	= generic_error_remove_page,
L
Linus Torvalds 已提交
1596
};