xfs_aops.c 37.3 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_bit.h"
L
Linus Torvalds 已提交
20
#include "xfs_log.h"
21
#include "xfs_inum.h"
L
Linus Torvalds 已提交
22
#include "xfs_sb.h"
23
#include "xfs_ag.h"
L
Linus Torvalds 已提交
24 25 26 27 28
#include "xfs_trans.h"
#include "xfs_mount.h"
#include "xfs_bmap_btree.h"
#include "xfs_dinode.h"
#include "xfs_inode.h"
29
#include "xfs_inode_item.h"
30
#include "xfs_alloc.h"
L
Linus Torvalds 已提交
31 32 33
#include "xfs_error.h"
#include "xfs_rw.h"
#include "xfs_iomap.h"
34
#include "xfs_vnodeops.h"
C
Christoph Hellwig 已提交
35
#include "xfs_trace.h"
36
#include "xfs_bmap.h"
37
#include <linux/gfp.h>
L
Linus Torvalds 已提交
38
#include <linux/mpage.h>
39
#include <linux/pagevec.h>
L
Linus Torvalds 已提交
40 41
#include <linux/writeback.h>

C
Christoph Hellwig 已提交
42
void
43 44 45 46 47 48 49
xfs_count_page_state(
	struct page		*page,
	int			*delalloc,
	int			*unwritten)
{
	struct buffer_head	*bh, *head;

50
	*delalloc = *unwritten = 0;
51 52 53

	bh = head = page_buffers(page);
	do {
54
		if (buffer_unwritten(bh))
55 56 57 58 59 60
			(*unwritten) = 1;
		else if (buffer_delay(bh))
			(*delalloc) = 1;
	} while ((bh = bh->b_this_page) != head);
}

C
Christoph Hellwig 已提交
61 62
STATIC struct block_device *
xfs_find_bdev_for_inode(
C
Christoph Hellwig 已提交
63
	struct inode		*inode)
C
Christoph Hellwig 已提交
64
{
C
Christoph Hellwig 已提交
65
	struct xfs_inode	*ip = XFS_I(inode);
C
Christoph Hellwig 已提交
66 67
	struct xfs_mount	*mp = ip->i_mount;

68
	if (XFS_IS_REALTIME_INODE(ip))
C
Christoph Hellwig 已提交
69 70 71 72 73
		return mp->m_rtdev_targp->bt_bdev;
	else
		return mp->m_ddev_targp->bt_bdev;
}

74 75 76 77 78 79
/*
 * We're now finished for good with this ioend structure.
 * Update the page state via the associated buffer_heads,
 * release holds on the inode and bio, and finally free
 * up memory.  Do not use the ioend after this.
 */
80 81 82 83
STATIC void
xfs_destroy_ioend(
	xfs_ioend_t		*ioend)
{
84 85 86 87
	struct buffer_head	*bh, *next;

	for (bh = ioend->io_buffer_head; bh; bh = next) {
		next = bh->b_private;
88
		bh->b_end_io(bh, !ioend->io_error);
89
	}
C
Christoph Hellwig 已提交
90

C
Christoph Hellwig 已提交
91
	if (ioend->io_iocb) {
92 93 94 95
		if (ioend->io_isasync) {
			aio_complete(ioend->io_iocb, ioend->io_error ?
					ioend->io_error : ioend->io_result, 0);
		}
C
Christoph Hellwig 已提交
96 97
		inode_dio_done(ioend->io_inode);
	}
C
Christoph Hellwig 已提交
98

99 100 101
	mempool_free(ioend, xfs_ioend_pool);
}

C
Christoph Hellwig 已提交
102 103 104 105 106 107 108 109 110
/*
 * Fast and loose check if this write could update the on-disk inode size.
 */
static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)
{
	return ioend->io_offset + ioend->io_size >
		XFS_I(ioend->io_inode)->i_d.di_size;
}

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
STATIC int
xfs_setfilesize_trans_alloc(
	struct xfs_ioend	*ioend)
{
	struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;
	struct xfs_trans	*tp;
	int			error;

	tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS);

	error = xfs_trans_reserve(tp, 0, XFS_FSYNC_TS_LOG_RES(mp), 0, 0, 0);
	if (error) {
		xfs_trans_cancel(tp, 0);
		return error;
	}

	ioend->io_append_trans = tp;

	/*
	 * We hand off the transaction to the completion thread now, so
	 * clear the flag here.
	 */
	current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS);
	return 0;
}

137
/*
138
 * Update on-disk file size now that data has been written to disk.
139
 */
140
STATIC int
141
xfs_setfilesize(
142
	struct xfs_ioend	*ioend)
143
{
144
	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
145
	struct xfs_trans	*tp = ioend->io_append_trans;
146 147
	xfs_fsize_t		isize;

148 149 150 151 152 153 154
	/*
	 * The transaction was allocated in the I/O submission thread,
	 * thus we need to mark ourselves as beeing in a transaction
	 * manually.
	 */
	current_set_flags_nested(&tp->t_pflags, PF_FSTRANS);

155
	xfs_ilock(ip, XFS_ILOCK_EXCL);
156
	isize = xfs_new_eof(ip, ioend->io_offset + ioend->io_size);
157 158 159 160
	if (!isize) {
		xfs_iunlock(ip, XFS_ILOCK_EXCL);
		xfs_trans_cancel(tp, 0);
		return 0;
161 162
	}

163 164 165 166 167 168 169
	trace_xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);

	ip->i_d.di_size = isize;
	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);

	return xfs_trans_commit(tp, 0);
170 171 172
}

/*
173
 * Schedule IO completion handling on the final put of an ioend.
C
Christoph Hellwig 已提交
174 175 176
 *
 * If there is no work to do we might as well call it a day and free the
 * ioend right now.
177 178 179
 */
STATIC void
xfs_finish_ioend(
180
	struct xfs_ioend	*ioend)
181 182
{
	if (atomic_dec_and_test(&ioend->io_remaining)) {
183 184
		struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;

185
		if (ioend->io_type == IO_UNWRITTEN)
186
			queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
187
		else if (ioend->io_append_trans)
188
			queue_work(mp->m_data_workqueue, &ioend->io_work);
C
Christoph Hellwig 已提交
189 190
		else
			xfs_destroy_ioend(ioend);
191
	}
192 193
}

194
/*
195
 * IO write completion.
196 197
 */
STATIC void
198
xfs_end_io(
199
	struct work_struct *work)
200
{
201 202
	xfs_ioend_t	*ioend = container_of(work, xfs_ioend_t, io_work);
	struct xfs_inode *ip = XFS_I(ioend->io_inode);
203
	int		error = 0;
204

205
	if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
206
		ioend->io_error = -EIO;
207 208 209 210 211
		goto done;
	}
	if (ioend->io_error)
		goto done;

212 213 214 215
	/*
	 * For unwritten extents we need to issue transactions to convert a
	 * range to normal written extens after the data I/O has finished.
	 */
216
	if (ioend->io_type == IO_UNWRITTEN) {
217 218 219 220 221 222 223 224 225 226 227 228 229 230
		/*
		 * For buffered I/O we never preallocate a transaction when
		 * doing the unwritten extent conversion, but for direct I/O
		 * we do not know if we are converting an unwritten extent
		 * or not at the point where we preallocate the transaction.
		 */
		if (ioend->io_append_trans) {
			ASSERT(ioend->io_isdirect);

			current_set_flags_nested(
				&ioend->io_append_trans->t_pflags, PF_FSTRANS);
			xfs_trans_cancel(ioend->io_append_trans, 0);
		}

231 232
		error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
						 ioend->io_size);
233 234 235 236
		if (error) {
			ioend->io_error = -error;
			goto done;
		}
237 238 239 240
	} else if (ioend->io_append_trans) {
		error = xfs_setfilesize(ioend);
		if (error)
			ioend->io_error = -error;
241
	} else {
242
		ASSERT(!xfs_ioend_is_append(ioend));
243
	}
244

245
done:
246
	xfs_destroy_ioend(ioend);
247 248
}

249 250 251 252 253 254 255 256 257 258 259
/*
 * Call IO completion handling in caller context on the final put of an ioend.
 */
STATIC void
xfs_finish_ioend_sync(
	struct xfs_ioend	*ioend)
{
	if (atomic_dec_and_test(&ioend->io_remaining))
		xfs_end_io(&ioend->io_work);
}

260 261 262 263 264 265 266 267
/*
 * Allocate and initialise an IO completion structure.
 * We need to track unwritten extent write completion here initially.
 * We'll need to extend this for updating the ondisk inode size later
 * (vs. incore size).
 */
STATIC xfs_ioend_t *
xfs_alloc_ioend(
268 269
	struct inode		*inode,
	unsigned int		type)
270 271 272 273 274 275 276 277 278 279 280
{
	xfs_ioend_t		*ioend;

	ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);

	/*
	 * Set the count to 1 initially, which will prevent an I/O
	 * completion callback from happening before we have started
	 * all the I/O from calling the completion routine too early.
	 */
	atomic_set(&ioend->io_remaining, 1);
C
Christoph Hellwig 已提交
281
	ioend->io_isasync = 0;
282
	ioend->io_isdirect = 0;
283
	ioend->io_error = 0;
284 285
	ioend->io_list = NULL;
	ioend->io_type = type;
286
	ioend->io_inode = inode;
287
	ioend->io_buffer_head = NULL;
288
	ioend->io_buffer_tail = NULL;
289 290
	ioend->io_offset = 0;
	ioend->io_size = 0;
291 292
	ioend->io_iocb = NULL;
	ioend->io_result = 0;
293
	ioend->io_append_trans = NULL;
294

295
	INIT_WORK(&ioend->io_work, xfs_end_io);
296 297 298
	return ioend;
}

L
Linus Torvalds 已提交
299 300 301 302
STATIC int
xfs_map_blocks(
	struct inode		*inode,
	loff_t			offset,
C
Christoph Hellwig 已提交
303
	struct xfs_bmbt_irec	*imap,
C
Christoph Hellwig 已提交
304 305
	int			type,
	int			nonblocking)
L
Linus Torvalds 已提交
306
{
C
Christoph Hellwig 已提交
307 308
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
C
Christoph Hellwig 已提交
309
	ssize_t			count = 1 << inode->i_blkbits;
C
Christoph Hellwig 已提交
310 311 312 313 314 315 316 317
	xfs_fileoff_t		offset_fsb, end_fsb;
	int			error = 0;
	int			bmapi_flags = XFS_BMAPI_ENTIRE;
	int			nimaps = 1;

	if (XFS_FORCED_SHUTDOWN(mp))
		return -XFS_ERROR(EIO);

C
Christoph Hellwig 已提交
318
	if (type == IO_UNWRITTEN)
C
Christoph Hellwig 已提交
319
		bmapi_flags |= XFS_BMAPI_IGSTATE;
C
Christoph Hellwig 已提交
320 321 322 323 324

	if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) {
		if (nonblocking)
			return -XFS_ERROR(EAGAIN);
		xfs_ilock(ip, XFS_ILOCK_SHARED);
C
Christoph Hellwig 已提交
325 326
	}

C
Christoph Hellwig 已提交
327 328
	ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE ||
	       (ip->i_df.if_flags & XFS_IFEXTENTS));
C
Christoph Hellwig 已提交
329
	ASSERT(offset <= mp->m_maxioffset);
C
Christoph Hellwig 已提交
330

C
Christoph Hellwig 已提交
331 332 333 334
	if (offset + count > mp->m_maxioffset)
		count = mp->m_maxioffset - offset;
	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
	offset_fsb = XFS_B_TO_FSBT(mp, offset);
D
Dave Chinner 已提交
335 336
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
				imap, &nimaps, bmapi_flags);
C
Christoph Hellwig 已提交
337
	xfs_iunlock(ip, XFS_ILOCK_SHARED);
C
Christoph Hellwig 已提交
338

C
Christoph Hellwig 已提交
339 340
	if (error)
		return -XFS_ERROR(error);
C
Christoph Hellwig 已提交
341

C
Christoph Hellwig 已提交
342 343
	if (type == IO_DELALLOC &&
	    (!nimaps || isnullstartblock(imap->br_startblock))) {
C
Christoph Hellwig 已提交
344 345 346
		error = xfs_iomap_write_allocate(ip, offset, count, imap);
		if (!error)
			trace_xfs_map_blocks_alloc(ip, offset, count, type, imap);
C
Christoph Hellwig 已提交
347
		return -XFS_ERROR(error);
C
Christoph Hellwig 已提交
348 349
	}

C
Christoph Hellwig 已提交
350 351 352 353 354 355 356 357 358 359
#ifdef DEBUG
	if (type == IO_UNWRITTEN) {
		ASSERT(nimaps);
		ASSERT(imap->br_startblock != HOLESTARTBLOCK);
		ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
	}
#endif
	if (nimaps)
		trace_xfs_map_blocks_found(ip, offset, count, type, imap);
	return 0;
L
Linus Torvalds 已提交
360 361
}

362
STATIC int
363
xfs_imap_valid(
364
	struct inode		*inode,
C
Christoph Hellwig 已提交
365
	struct xfs_bmbt_irec	*imap,
366
	xfs_off_t		offset)
L
Linus Torvalds 已提交
367
{
368
	offset >>= inode->i_blkbits;
369

370 371
	return offset >= imap->br_startoff &&
		offset < imap->br_startoff + imap->br_blockcount;
L
Linus Torvalds 已提交
372 373
}

374 375 376
/*
 * BIO completion handler for buffered IO.
 */
A
Al Viro 已提交
377
STATIC void
378 379 380 381 382 383 384
xfs_end_bio(
	struct bio		*bio,
	int			error)
{
	xfs_ioend_t		*ioend = bio->bi_private;

	ASSERT(atomic_read(&bio->bi_cnt) >= 1);
385
	ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
386 387 388 389 390

	/* Toss bio and pass work off to an xfsdatad thread */
	bio->bi_private = NULL;
	bio->bi_end_io = NULL;
	bio_put(bio);
391

392
	xfs_finish_ioend(ioend);
393 394 395 396
}

STATIC void
xfs_submit_ioend_bio(
397 398 399
	struct writeback_control *wbc,
	xfs_ioend_t		*ioend,
	struct bio		*bio)
400 401 402 403
{
	atomic_inc(&ioend->io_remaining);
	bio->bi_private = ioend;
	bio->bi_end_io = xfs_end_bio;
J
Jens Axboe 已提交
404
	submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio);
405 406 407 408 409 410 411
}

STATIC struct bio *
xfs_alloc_ioend_bio(
	struct buffer_head	*bh)
{
	int			nvecs = bio_get_nr_vecs(bh->b_bdev);
412
	struct bio		*bio = bio_alloc(GFP_NOIO, nvecs);
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442

	ASSERT(bio->bi_private == NULL);
	bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
	bio->bi_bdev = bh->b_bdev;
	return bio;
}

STATIC void
xfs_start_buffer_writeback(
	struct buffer_head	*bh)
{
	ASSERT(buffer_mapped(bh));
	ASSERT(buffer_locked(bh));
	ASSERT(!buffer_delay(bh));
	ASSERT(!buffer_unwritten(bh));

	mark_buffer_async_write(bh);
	set_buffer_uptodate(bh);
	clear_buffer_dirty(bh);
}

STATIC void
xfs_start_page_writeback(
	struct page		*page,
	int			clear_dirty,
	int			buffers)
{
	ASSERT(PageLocked(page));
	ASSERT(!PageWriteback(page));
	if (clear_dirty)
443 444
		clear_page_dirty_for_io(page);
	set_page_writeback(page);
445
	unlock_page(page);
446 447
	/* If no buffers on the page are to be written, finish it here */
	if (!buffers)
448 449 450 451 452 453 454 455 456
		end_page_writeback(page);
}

static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
{
	return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
}

/*
457 458 459 460 461 462 463 464 465 466 467 468 469 470
 * Submit all of the bios for all of the ioends we have saved up, covering the
 * initial writepage page and also any probed pages.
 *
 * Because we may have multiple ioends spanning a page, we need to start
 * writeback on all the buffers before we submit them for I/O. If we mark the
 * buffers as we got, then we can end up with a page that only has buffers
 * marked async write and I/O complete on can occur before we mark the other
 * buffers async write.
 *
 * The end result of this is that we trip a bug in end_page_writeback() because
 * we call it twice for the one page as the code in end_buffer_async_write()
 * assumes that all buffers on the page are started at the same time.
 *
 * The fix is two passes across the ioend list - one to start writeback on the
471
 * buffer_heads, and then submit them for I/O on the second pass.
472 473 474
 */
STATIC void
xfs_submit_ioend(
475
	struct writeback_control *wbc,
476 477
	xfs_ioend_t		*ioend)
{
478
	xfs_ioend_t		*head = ioend;
479 480 481 482 483
	xfs_ioend_t		*next;
	struct buffer_head	*bh;
	struct bio		*bio;
	sector_t		lastblock = 0;

484 485 486
	/* Pass 1 - start writeback */
	do {
		next = ioend->io_list;
487
		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)
488 489 490 491 492
			xfs_start_buffer_writeback(bh);
	} while ((ioend = next) != NULL);

	/* Pass 2 - submit I/O */
	ioend = head;
493 494 495 496 497 498 499 500 501 502
	do {
		next = ioend->io_list;
		bio = NULL;

		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {

			if (!bio) {
 retry:
				bio = xfs_alloc_ioend_bio(bh);
			} else if (bh->b_blocknr != lastblock + 1) {
503
				xfs_submit_ioend_bio(wbc, ioend, bio);
504 505 506 507
				goto retry;
			}

			if (bio_add_buffer(bio, bh) != bh->b_size) {
508
				xfs_submit_ioend_bio(wbc, ioend, bio);
509 510 511 512 513 514
				goto retry;
			}

			lastblock = bh->b_blocknr;
		}
		if (bio)
515
			xfs_submit_ioend_bio(wbc, ioend, bio);
516
		xfs_finish_ioend(ioend);
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
	} while ((ioend = next) != NULL);
}

/*
 * Cancel submission of all buffer_heads so far in this endio.
 * Toss the endio too.  Only ever called for the initial page
 * in a writepage request, so only ever one page.
 */
STATIC void
xfs_cancel_ioend(
	xfs_ioend_t		*ioend)
{
	xfs_ioend_t		*next;
	struct buffer_head	*bh, *next_bh;

	do {
		next = ioend->io_list;
		bh = ioend->io_buffer_head;
		do {
			next_bh = bh->b_private;
			clear_buffer_async_write(bh);
			unlock_buffer(bh);
		} while ((bh = next_bh) != NULL);

		mempool_free(ioend, xfs_ioend_pool);
	} while ((ioend = next) != NULL);
}

/*
 * Test to see if we've been building up a completion structure for
 * earlier buffers -- if so, we try to append to this ioend if we
 * can, otherwise we finish off any current ioend and start another.
 * Return true if we've finished the given ioend.
 */
STATIC void
xfs_add_to_ioend(
	struct inode		*inode,
	struct buffer_head	*bh,
555
	xfs_off_t		offset,
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
	unsigned int		type,
	xfs_ioend_t		**result,
	int			need_ioend)
{
	xfs_ioend_t		*ioend = *result;

	if (!ioend || need_ioend || type != ioend->io_type) {
		xfs_ioend_t	*previous = *result;

		ioend = xfs_alloc_ioend(inode, type);
		ioend->io_offset = offset;
		ioend->io_buffer_head = bh;
		ioend->io_buffer_tail = bh;
		if (previous)
			previous->io_list = ioend;
		*result = ioend;
	} else {
		ioend->io_buffer_tail->b_private = bh;
		ioend->io_buffer_tail = bh;
	}

	bh->b_private = NULL;
	ioend->io_size += bh->b_size;
}

581 582
STATIC void
xfs_map_buffer(
C
Christoph Hellwig 已提交
583
	struct inode		*inode,
584
	struct buffer_head	*bh,
C
Christoph Hellwig 已提交
585
	struct xfs_bmbt_irec	*imap,
C
Christoph Hellwig 已提交
586
	xfs_off_t		offset)
587 588
{
	sector_t		bn;
589
	struct xfs_mount	*m = XFS_I(inode)->i_mount;
C
Christoph Hellwig 已提交
590 591
	xfs_off_t		iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
	xfs_daddr_t		iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
592

C
Christoph Hellwig 已提交
593 594
	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
595

596
	bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
597
	      ((offset - iomap_offset) >> inode->i_blkbits);
598

C
Christoph Hellwig 已提交
599
	ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
600 601 602 603 604

	bh->b_blocknr = bn;
	set_buffer_mapped(bh);
}

L
Linus Torvalds 已提交
605 606
STATIC void
xfs_map_at_offset(
C
Christoph Hellwig 已提交
607
	struct inode		*inode,
L
Linus Torvalds 已提交
608
	struct buffer_head	*bh,
C
Christoph Hellwig 已提交
609
	struct xfs_bmbt_irec	*imap,
C
Christoph Hellwig 已提交
610
	xfs_off_t		offset)
L
Linus Torvalds 已提交
611
{
C
Christoph Hellwig 已提交
612 613
	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
L
Linus Torvalds 已提交
614

C
Christoph Hellwig 已提交
615
	xfs_map_buffer(inode, bh, imap, offset);
L
Linus Torvalds 已提交
616 617
	set_buffer_mapped(bh);
	clear_buffer_delay(bh);
618
	clear_buffer_unwritten(bh);
L
Linus Torvalds 已提交
619 620 621
}

/*
622 623
 * Test if a given page is suitable for writing as part of an unwritten
 * or delayed allocate extent.
L
Linus Torvalds 已提交
624
 */
625 626 627
STATIC int
xfs_is_delayed_page(
	struct page		*page,
628
	unsigned int		type)
L
Linus Torvalds 已提交
629 630
{
	if (PageWriteback(page))
631
		return 0;
L
Linus Torvalds 已提交
632 633 634 635 636 637 638

	if (page->mapping && page_has_buffers(page)) {
		struct buffer_head	*bh, *head;
		int			acceptable = 0;

		bh = head = page_buffers(page);
		do {
639
			if (buffer_unwritten(bh))
640
				acceptable = (type == IO_UNWRITTEN);
641
			else if (buffer_delay(bh))
C
Christoph Hellwig 已提交
642
				acceptable = (type == IO_DELALLOC);
643
			else if (buffer_dirty(bh) && buffer_mapped(bh))
C
Christoph Hellwig 已提交
644
				acceptable = (type == IO_OVERWRITE);
645
			else
L
Linus Torvalds 已提交
646 647 648 649
				break;
		} while ((bh = bh->b_this_page) != head);

		if (acceptable)
650
			return 1;
L
Linus Torvalds 已提交
651 652
	}

653
	return 0;
L
Linus Torvalds 已提交
654 655 656 657 658 659 660 661
}

/*
 * Allocate & map buffers for page given the extent map. Write it out.
 * except for the original page of a writepage, this is called on
 * delalloc/unwritten pages only, for the original page it is possible
 * that the page has no mapping at all.
 */
662
STATIC int
L
Linus Torvalds 已提交
663 664 665
xfs_convert_page(
	struct inode		*inode,
	struct page		*page,
666
	loff_t			tindex,
C
Christoph Hellwig 已提交
667
	struct xfs_bmbt_irec	*imap,
668
	xfs_ioend_t		**ioendp,
669
	struct writeback_control *wbc)
L
Linus Torvalds 已提交
670
{
671
	struct buffer_head	*bh, *head;
672 673
	xfs_off_t		end_offset;
	unsigned long		p_offset;
674
	unsigned int		type;
675
	int			len, page_dirty;
676
	int			count = 0, done = 0, uptodate = 1;
677
 	xfs_off_t		offset = page_offset(page);
L
Linus Torvalds 已提交
678

679 680
	if (page->index != tindex)
		goto fail;
N
Nick Piggin 已提交
681
	if (!trylock_page(page))
682 683 684 685 686 687 688 689
		goto fail;
	if (PageWriteback(page))
		goto fail_unlock_page;
	if (page->mapping != inode->i_mapping)
		goto fail_unlock_page;
	if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
		goto fail_unlock_page;

690 691
	/*
	 * page_dirty is initially a count of buffers on the page before
692
	 * EOF and is decremented as we move each into a cleanable state.
693 694 695 696 697 698 699 700 701
	 *
	 * Derivation:
	 *
	 * End offset is the highest offset that this page should represent.
	 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
	 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
	 * hence give us the correct page_dirty count. On any other page,
	 * it will be zero and in that case we need page_dirty to be the
	 * count of buffers on the page.
702
	 */
703 704 705 706
	end_offset = min_t(unsigned long long,
			(xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
			i_size_read(inode));

707
	len = 1 << inode->i_blkbits;
708 709 710 711
	p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
					PAGE_CACHE_SIZE);
	p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
	page_dirty = p_offset / len;
712

L
Linus Torvalds 已提交
713 714
	bh = head = page_buffers(page);
	do {
715
		if (offset >= end_offset)
L
Linus Torvalds 已提交
716
			break;
717 718 719 720
		if (!buffer_uptodate(bh))
			uptodate = 0;
		if (!(PageUptodate(page) || buffer_uptodate(bh))) {
			done = 1;
L
Linus Torvalds 已提交
721
			continue;
722 723
		}

724 725
		if (buffer_unwritten(bh) || buffer_delay(bh) ||
		    buffer_mapped(bh)) {
726
			if (buffer_unwritten(bh))
727
				type = IO_UNWRITTEN;
728
			else if (buffer_delay(bh))
C
Christoph Hellwig 已提交
729
				type = IO_DELALLOC;
730 731
			else
				type = IO_OVERWRITE;
732

733
			if (!xfs_imap_valid(inode, imap, offset)) {
734
				done = 1;
735 736 737
				continue;
			}

738 739
			lock_buffer(bh);
			if (type != IO_OVERWRITE)
740
				xfs_map_at_offset(inode, bh, imap, offset);
741 742 743
			xfs_add_to_ioend(inode, bh, offset, type,
					 ioendp, done);

744 745 746
			page_dirty--;
			count++;
		} else {
747
			done = 1;
L
Linus Torvalds 已提交
748
		}
749
	} while (offset += len, (bh = bh->b_this_page) != head);
L
Linus Torvalds 已提交
750

751 752 753
	if (uptodate && bh == head)
		SetPageUptodate(page);

754
	if (count) {
755 756
		if (--wbc->nr_to_write <= 0 &&
		    wbc->sync_mode == WB_SYNC_NONE)
757
			done = 1;
L
Linus Torvalds 已提交
758
	}
759
	xfs_start_page_writeback(page, !page_dirty, count);
760 761

	return done;
762 763 764 765
 fail_unlock_page:
	unlock_page(page);
 fail:
	return 1;
L
Linus Torvalds 已提交
766 767 768 769 770 771 772 773 774 775
}

/*
 * Convert & write out a cluster of pages in the same extent as defined
 * by mp and following the start page.
 */
STATIC void
xfs_cluster_write(
	struct inode		*inode,
	pgoff_t			tindex,
C
Christoph Hellwig 已提交
776
	struct xfs_bmbt_irec	*imap,
777
	xfs_ioend_t		**ioendp,
L
Linus Torvalds 已提交
778 779 780
	struct writeback_control *wbc,
	pgoff_t			tlast)
{
781 782
	struct pagevec		pvec;
	int			done = 0, i;
L
Linus Torvalds 已提交
783

784 785 786 787 788
	pagevec_init(&pvec, 0);
	while (!done && tindex <= tlast) {
		unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);

		if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
L
Linus Torvalds 已提交
789
			break;
790 791 792

		for (i = 0; i < pagevec_count(&pvec); i++) {
			done = xfs_convert_page(inode, pvec.pages[i], tindex++,
793
					imap, ioendp, wbc);
794 795 796 797 798 799
			if (done)
				break;
		}

		pagevec_release(&pvec);
		cond_resched();
L
Linus Torvalds 已提交
800 801 802
	}
}

803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
STATIC void
xfs_vm_invalidatepage(
	struct page		*page,
	unsigned long		offset)
{
	trace_xfs_invalidatepage(page->mapping->host, page, offset);
	block_invalidatepage(page, offset);
}

/*
 * If the page has delalloc buffers on it, we need to punch them out before we
 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
 * is done on that same region - the delalloc extent is returned when none is
 * supposed to be there.
 *
 * We prevent this by truncating away the delalloc regions on the page before
 * invalidating it. Because they are delalloc, we can do this without needing a
 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
 * truncation without a transaction as there is no space left for block
 * reservation (typically why we see a ENOSPC in writeback).
 *
 * This is not a performance critical path, so for now just do the punching a
 * buffer head at a time.
 */
STATIC void
xfs_aops_discard_page(
	struct page		*page)
{
	struct inode		*inode = page->mapping->host;
	struct xfs_inode	*ip = XFS_I(inode);
	struct buffer_head	*bh, *head;
	loff_t			offset = page_offset(page);

C
Christoph Hellwig 已提交
837
	if (!xfs_is_delayed_page(page, IO_DELALLOC))
838 839
		goto out_invalidate;

840 841 842
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
		goto out_invalidate;

843
	xfs_alert(ip->i_mount,
844 845 846 847 848 849 850
		"page discard on page %p, inode 0x%llx, offset %llu.",
			page, ip->i_ino, offset);

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	bh = head = page_buffers(page);
	do {
		int		error;
851
		xfs_fileoff_t	start_fsb;
852 853 854 855

		if (!buffer_delay(bh))
			goto next_buffer;

856 857
		start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
		error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
858 859
		if (error) {
			/* something screwed, just bail */
860
			if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
861
				xfs_alert(ip->i_mount,
862
			"page discard unable to remove delalloc mapping.");
863
			}
864 865 866
			break;
		}
next_buffer:
867
		offset += 1 << inode->i_blkbits;
868 869 870 871 872 873 874 875 876

	} while ((bh = bh->b_this_page) != head);

	xfs_iunlock(ip, XFS_ILOCK_EXCL);
out_invalidate:
	xfs_vm_invalidatepage(page, 0);
	return;
}

L
Linus Torvalds 已提交
877
/*
878 879 880 881 882 883
 * Write out a dirty page.
 *
 * For delalloc space on the page we need to allocate space and flush it.
 * For unwritten space on the page we need to start the conversion to
 * regular allocated space.
 * For any other dirty buffer heads on the page we should flush them.
L
Linus Torvalds 已提交
884 885
 */
STATIC int
886 887 888
xfs_vm_writepage(
	struct page		*page,
	struct writeback_control *wbc)
L
Linus Torvalds 已提交
889
{
890
	struct inode		*inode = page->mapping->host;
891
	struct buffer_head	*bh, *head;
C
Christoph Hellwig 已提交
892
	struct xfs_bmbt_irec	imap;
893
	xfs_ioend_t		*ioend = NULL, *iohead = NULL;
L
Linus Torvalds 已提交
894
	loff_t			offset;
895
	unsigned int		type;
L
Linus Torvalds 已提交
896
	__uint64_t              end_offset;
897
	pgoff_t                 end_index, last_index;
C
Christoph Hellwig 已提交
898
	ssize_t			len;
C
Christoph Hellwig 已提交
899
	int			err, imap_valid = 0, uptodate = 1;
900
	int			count = 0;
C
Christoph Hellwig 已提交
901
	int			nonblocking = 0;
902 903 904

	trace_xfs_writepage(inode, page, 0);

905 906
	ASSERT(page_has_buffers(page));

907 908 909
	/*
	 * Refuse to write the page out if we are called from reclaim context.
	 *
910 911 912
	 * This avoids stack overflows when called from deeply used stacks in
	 * random callers for direct reclaim or memcg reclaim.  We explicitly
	 * allow reclaim from kswapd as the stack usage there is relatively low.
913
	 *
914 915
	 * This should never happen except in the case of a VM regression so
	 * warn about it.
916
	 */
917 918
	if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
			PF_MEMALLOC))
919
		goto redirty;
L
Linus Torvalds 已提交
920

921
	/*
922 923
	 * Given that we do not allow direct reclaim to call us, we should
	 * never be called while in a filesystem transaction.
924
	 */
925
	if (WARN_ON(current->flags & PF_FSTRANS))
926
		goto redirty;
927

L
Linus Torvalds 已提交
928 929 930 931 932 933 934
	/* Is this page beyond the end of the file? */
	offset = i_size_read(inode);
	end_index = offset >> PAGE_CACHE_SHIFT;
	last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
	if (page->index >= end_index) {
		if ((page->index >= end_index + 1) ||
		    !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
935
			unlock_page(page);
936
			return 0;
L
Linus Torvalds 已提交
937 938 939
		}
	}

940
	end_offset = min_t(unsigned long long,
941 942
			(xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
			offset);
943 944 945
	len = 1 << inode->i_blkbits;

	bh = head = page_buffers(page);
946
	offset = page_offset(page);
C
Christoph Hellwig 已提交
947 948
	type = IO_OVERWRITE;

949
	if (wbc->sync_mode == WB_SYNC_NONE)
C
Christoph Hellwig 已提交
950
		nonblocking = 1;
951

L
Linus Torvalds 已提交
952
	do {
953 954
		int new_ioend = 0;

L
Linus Torvalds 已提交
955 956 957 958 959
		if (offset >= end_offset)
			break;
		if (!buffer_uptodate(bh))
			uptodate = 0;

960
		/*
961 962 963 964
		 * set_page_dirty dirties all buffers in a page, independent
		 * of their state.  The dirty state however is entirely
		 * meaningless for holes (!mapped && uptodate), so skip
		 * buffers covering holes here.
965 966 967 968 969 970
		 */
		if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
			imap_valid = 0;
			continue;
		}

C
Christoph Hellwig 已提交
971 972 973 974
		if (buffer_unwritten(bh)) {
			if (type != IO_UNWRITTEN) {
				type = IO_UNWRITTEN;
				imap_valid = 0;
L
Linus Torvalds 已提交
975
			}
C
Christoph Hellwig 已提交
976 977 978 979
		} else if (buffer_delay(bh)) {
			if (type != IO_DELALLOC) {
				type = IO_DELALLOC;
				imap_valid = 0;
L
Linus Torvalds 已提交
980
			}
981
		} else if (buffer_uptodate(bh)) {
C
Christoph Hellwig 已提交
982 983
			if (type != IO_OVERWRITE) {
				type = IO_OVERWRITE;
984 985
				imap_valid = 0;
			}
C
Christoph Hellwig 已提交
986 987 988 989
		} else {
			if (PageUptodate(page)) {
				ASSERT(buffer_mapped(bh));
				imap_valid = 0;
990
			}
C
Christoph Hellwig 已提交
991 992
			continue;
		}
993

C
Christoph Hellwig 已提交
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
		if (imap_valid)
			imap_valid = xfs_imap_valid(inode, &imap, offset);
		if (!imap_valid) {
			/*
			 * If we didn't have a valid mapping then we need to
			 * put the new mapping into a separate ioend structure.
			 * This ensures non-contiguous extents always have
			 * separate ioends, which is particularly important
			 * for unwritten extent conversion at I/O completion
			 * time.
			 */
			new_ioend = 1;
			err = xfs_map_blocks(inode, offset, &imap, type,
					     nonblocking);
			if (err)
				goto error;
			imap_valid = xfs_imap_valid(inode, &imap, offset);
		}
		if (imap_valid) {
1013 1014
			lock_buffer(bh);
			if (type != IO_OVERWRITE)
C
Christoph Hellwig 已提交
1015 1016 1017 1018
				xfs_map_at_offset(inode, bh, &imap, offset);
			xfs_add_to_ioend(inode, bh, offset, type, &ioend,
					 new_ioend);
			count++;
L
Linus Torvalds 已提交
1019
		}
1020 1021 1022 1023 1024

		if (!iohead)
			iohead = ioend;

	} while (offset += len, ((bh = bh->b_this_page) != head));
L
Linus Torvalds 已提交
1025 1026 1027 1028

	if (uptodate && bh == head)
		SetPageUptodate(page);

1029
	xfs_start_page_writeback(page, 1, count);
L
Linus Torvalds 已提交
1030

1031
	if (ioend && imap_valid) {
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
		xfs_off_t		end_index;

		end_index = imap.br_startoff + imap.br_blockcount;

		/* to bytes */
		end_index <<= inode->i_blkbits;

		/* to pages */
		end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;

		/* check against file size */
		if (end_index > last_index)
			end_index = last_index;
1045

C
Christoph Hellwig 已提交
1046
		xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
1047
				  wbc, end_index);
L
Linus Torvalds 已提交
1048 1049
	}

1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
	if (iohead) {
		/*
		 * Reserve log space if we might write beyond the on-disk
		 * inode size.
		 */
		if (ioend->io_type != IO_UNWRITTEN &&
		    xfs_ioend_is_append(ioend)) {
			err = xfs_setfilesize_trans_alloc(ioend);
			if (err)
				goto error;
		}

1062
		xfs_submit_ioend(wbc, iohead);
1063
	}
1064

1065
	return 0;
L
Linus Torvalds 已提交
1066 1067

error:
1068 1069
	if (iohead)
		xfs_cancel_ioend(iohead);
L
Linus Torvalds 已提交
1070

1071 1072 1073
	if (err == -EAGAIN)
		goto redirty;

1074
	xfs_aops_discard_page(page);
1075 1076
	ClearPageUptodate(page);
	unlock_page(page);
L
Linus Torvalds 已提交
1077
	return err;
1078

1079
redirty:
1080 1081 1082 1083 1084
	redirty_page_for_writepage(wbc, page);
	unlock_page(page);
	return 0;
}

1085 1086 1087 1088 1089
STATIC int
xfs_vm_writepages(
	struct address_space	*mapping,
	struct writeback_control *wbc)
{
1090
	xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1091 1092 1093
	return generic_writepages(mapping, wbc);
}

1094 1095
/*
 * Called to move a page into cleanable state - and from there
1096
 * to be released. The page should already be clean. We always
1097 1098
 * have buffer heads in this call.
 *
1099
 * Returns 1 if the page is ok to release, 0 otherwise.
1100 1101
 */
STATIC int
1102
xfs_vm_releasepage(
1103 1104 1105
	struct page		*page,
	gfp_t			gfp_mask)
{
1106
	int			delalloc, unwritten;
1107

1108
	trace_xfs_releasepage(page->mapping->host, page, 0);
1109

1110
	xfs_count_page_state(page, &delalloc, &unwritten);
1111

1112
	if (WARN_ON(delalloc))
1113
		return 0;
1114
	if (WARN_ON(unwritten))
1115 1116 1117 1118 1119
		return 0;

	return try_to_free_buffers(page);
}

L
Linus Torvalds 已提交
1120
STATIC int
1121
__xfs_get_blocks(
L
Linus Torvalds 已提交
1122 1123 1124 1125
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create,
1126
	int			direct)
L
Linus Torvalds 已提交
1127
{
C
Christoph Hellwig 已提交
1128 1129 1130 1131 1132
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb, end_fsb;
	int			error = 0;
	int			lockmode = 0;
C
Christoph Hellwig 已提交
1133
	struct xfs_bmbt_irec	imap;
C
Christoph Hellwig 已提交
1134
	int			nimaps = 1;
1135 1136
	xfs_off_t		offset;
	ssize_t			size;
C
Christoph Hellwig 已提交
1137
	int			new = 0;
C
Christoph Hellwig 已提交
1138 1139 1140

	if (XFS_FORCED_SHUTDOWN(mp))
		return -XFS_ERROR(EIO);
L
Linus Torvalds 已提交
1141

1142
	offset = (xfs_off_t)iblock << inode->i_blkbits;
1143 1144
	ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
	size = bh_result->b_size;
1145 1146 1147 1148

	if (!create && direct && offset >= i_size_read(inode))
		return 0;

1149 1150 1151 1152 1153 1154 1155 1156
	/*
	 * Direct I/O is usually done on preallocated files, so try getting
	 * a block mapping without an exclusive lock first.  For buffered
	 * writes we already have the exclusive iolock anyway, so avoiding
	 * a lock roundtrip here by taking the ilock exclusive from the
	 * beginning is a useful micro optimization.
	 */
	if (create && !direct) {
C
Christoph Hellwig 已提交
1157 1158 1159 1160 1161
		lockmode = XFS_ILOCK_EXCL;
		xfs_ilock(ip, lockmode);
	} else {
		lockmode = xfs_ilock_map_shared(ip);
	}
1162

C
Christoph Hellwig 已提交
1163 1164 1165 1166 1167 1168
	ASSERT(offset <= mp->m_maxioffset);
	if (offset + size > mp->m_maxioffset)
		size = mp->m_maxioffset - offset;
	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
	offset_fsb = XFS_B_TO_FSBT(mp, offset);

D
Dave Chinner 已提交
1169 1170
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
				&imap, &nimaps, XFS_BMAPI_ENTIRE);
L
Linus Torvalds 已提交
1171
	if (error)
C
Christoph Hellwig 已提交
1172 1173 1174 1175 1176 1177 1178
		goto out_unlock;

	if (create &&
	    (!nimaps ||
	     (imap.br_startblock == HOLESTARTBLOCK ||
	      imap.br_startblock == DELAYSTARTBLOCK))) {
		if (direct) {
1179 1180 1181 1182 1183 1184 1185
			/*
			 * Drop the ilock in preparation for starting the block
			 * allocation transaction.  It will be retaken
			 * exclusively inside xfs_iomap_write_direct for the
			 * actual allocation.
			 */
			xfs_iunlock(ip, lockmode);
C
Christoph Hellwig 已提交
1186 1187
			error = xfs_iomap_write_direct(ip, offset, size,
						       &imap, nimaps);
1188 1189
			if (error)
				return -error;
C
Christoph Hellwig 已提交
1190
		} else {
1191 1192 1193 1194
			/*
			 * Delalloc reservations do not require a transaction,
			 * we can go on without dropping the lock here.
			 */
C
Christoph Hellwig 已提交
1195
			error = xfs_iomap_write_delay(ip, offset, size, &imap);
1196 1197 1198 1199
			if (error)
				goto out_unlock;

			xfs_iunlock(ip, lockmode);
C
Christoph Hellwig 已提交
1200 1201 1202 1203 1204
		}

		trace_xfs_get_blocks_alloc(ip, offset, size, 0, &imap);
	} else if (nimaps) {
		trace_xfs_get_blocks_found(ip, offset, size, 0, &imap);
1205
		xfs_iunlock(ip, lockmode);
C
Christoph Hellwig 已提交
1206 1207 1208 1209
	} else {
		trace_xfs_get_blocks_notfound(ip, offset, size);
		goto out_unlock;
	}
L
Linus Torvalds 已提交
1210

C
Christoph Hellwig 已提交
1211 1212
	if (imap.br_startblock != HOLESTARTBLOCK &&
	    imap.br_startblock != DELAYSTARTBLOCK) {
1213 1214
		/*
		 * For unwritten extents do not report a disk address on
L
Linus Torvalds 已提交
1215 1216
		 * the read case (treat as if we're reading into a hole).
		 */
C
Christoph Hellwig 已提交
1217 1218 1219
		if (create || !ISUNWRITTEN(&imap))
			xfs_map_buffer(inode, bh_result, &imap, offset);
		if (create && ISUNWRITTEN(&imap)) {
L
Linus Torvalds 已提交
1220 1221 1222 1223 1224 1225
			if (direct)
				bh_result->b_private = inode;
			set_buffer_unwritten(bh_result);
		}
	}

1226 1227 1228 1229
	/*
	 * If this is a realtime file, data may be on a different device.
	 * to that pointed to from the buffer_head b_bdev currently.
	 */
C
Christoph Hellwig 已提交
1230
	bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
L
Linus Torvalds 已提交
1231

1232
	/*
1233 1234 1235 1236 1237 1238 1239
	 * If we previously allocated a block out beyond eof and we are now
	 * coming back to use it then we will need to flag it as new even if it
	 * has a disk address.
	 *
	 * With sub-block writes into unwritten extents we also need to mark
	 * the buffer as new so that the unwritten parts of the buffer gets
	 * correctly zeroed.
L
Linus Torvalds 已提交
1240 1241 1242
	 */
	if (create &&
	    ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1243
	     (offset >= i_size_read(inode)) ||
C
Christoph Hellwig 已提交
1244
	     (new || ISUNWRITTEN(&imap))))
L
Linus Torvalds 已提交
1245 1246
		set_buffer_new(bh_result);

C
Christoph Hellwig 已提交
1247
	if (imap.br_startblock == DELAYSTARTBLOCK) {
L
Linus Torvalds 已提交
1248 1249 1250 1251 1252 1253 1254 1255
		BUG_ON(direct);
		if (create) {
			set_buffer_uptodate(bh_result);
			set_buffer_mapped(bh_result);
			set_buffer_delay(bh_result);
		}
	}

1256 1257 1258 1259
	/*
	 * If this is O_DIRECT or the mpage code calling tell them how large
	 * the mapping is, so that we can avoid repeated get_blocks calls.
	 */
1260
	if (direct || size > (1 << inode->i_blkbits)) {
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
		xfs_off_t		mapping_size;

		mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
		mapping_size <<= inode->i_blkbits;

		ASSERT(mapping_size > 0);
		if (mapping_size > size)
			mapping_size = size;
		if (mapping_size > LONG_MAX)
			mapping_size = LONG_MAX;

		bh_result->b_size = mapping_size;
L
Linus Torvalds 已提交
1273 1274 1275
	}

	return 0;
C
Christoph Hellwig 已提交
1276 1277 1278 1279

out_unlock:
	xfs_iunlock(ip, lockmode);
	return -error;
L
Linus Torvalds 已提交
1280 1281 1282
}

int
1283
xfs_get_blocks(
L
Linus Torvalds 已提交
1284 1285 1286 1287 1288
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create)
{
1289
	return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
L
Linus Torvalds 已提交
1290 1291 1292
}

STATIC int
1293
xfs_get_blocks_direct(
L
Linus Torvalds 已提交
1294 1295 1296 1297 1298
	struct inode		*inode,
	sector_t		iblock,
	struct buffer_head	*bh_result,
	int			create)
{
1299
	return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
L
Linus Torvalds 已提交
1300 1301
}

1302 1303 1304 1305 1306 1307
/*
 * Complete a direct I/O write request.
 *
 * If the private argument is non-NULL __xfs_get_blocks signals us that we
 * need to issue a transaction to convert the range from unwritten to written
 * extents.  In case this is regular synchronous I/O we just call xfs_end_io
L
Lucas De Marchi 已提交
1308
 * to do this and we are done.  But in case this was a successful AIO
1309 1310 1311 1312
 * request this handler is called from interrupt context, from which we
 * can't start transactions.  In that case offload the I/O completion to
 * the workqueues we also use for buffered I/O completion.
 */
1313
STATIC void
1314 1315 1316 1317 1318 1319 1320
xfs_end_io_direct_write(
	struct kiocb		*iocb,
	loff_t			offset,
	ssize_t			size,
	void			*private,
	int			ret,
	bool			is_async)
1321
{
1322
	struct xfs_ioend	*ioend = iocb->private;
1323

1324 1325 1326 1327 1328 1329 1330 1331 1332
	/*
	 * While the generic direct I/O code updates the inode size, it does
	 * so only after the end_io handler is called, which means our
	 * end_io handler thinks the on-disk size is outside the in-core
	 * size.  To prevent this just update it a little bit earlier here.
	 */
	if (offset + size > i_size_read(ioend->io_inode))
		i_size_write(ioend->io_inode, offset + size);

1333
	/*
1334 1335 1336
	 * blockdev_direct_IO can return an error even after the I/O
	 * completion handler was called.  Thus we need to protect
	 * against double-freeing.
1337
	 */
1338 1339
	iocb->private = NULL;

1340 1341
	ioend->io_offset = offset;
	ioend->io_size = size;
C
Christoph Hellwig 已提交
1342 1343
	ioend->io_iocb = iocb;
	ioend->io_result = ret;
1344 1345 1346 1347
	if (private && size > 0)
		ioend->io_type = IO_UNWRITTEN;

	if (is_async) {
C
Christoph Hellwig 已提交
1348
		ioend->io_isasync = 1;
1349
		xfs_finish_ioend(ioend);
1350
	} else {
1351
		xfs_finish_ioend_sync(ioend);
1352 1353 1354
	}
}

L
Linus Torvalds 已提交
1355
STATIC ssize_t
1356
xfs_vm_direct_IO(
L
Linus Torvalds 已提交
1357 1358 1359 1360 1361 1362
	int			rw,
	struct kiocb		*iocb,
	const struct iovec	*iov,
	loff_t			offset,
	unsigned long		nr_segs)
{
1363 1364
	struct inode		*inode = iocb->ki_filp->f_mapping->host;
	struct block_device	*bdev = xfs_find_bdev_for_inode(inode);
1365
	struct xfs_ioend	*ioend = NULL;
1366 1367 1368
	ssize_t			ret;

	if (rw & WRITE) {
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
		size_t size = iov_length(iov, nr_segs);

		/*
		 * We need to preallocate a transaction for a size update
		 * here.  In the case that this write both updates the size
		 * and converts at least on unwritten extent we will cancel
		 * the still clean transaction after the I/O has finished.
		 */
		iocb->private = ioend = xfs_alloc_ioend(inode, IO_DIRECT);
		if (offset + size > XFS_I(inode)->i_d.di_size) {
			ret = xfs_setfilesize_trans_alloc(ioend);
			if (ret)
				goto out_destroy_ioend;
			ioend->io_isdirect = 1;
		}
1384

1385 1386 1387 1388
		ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
					    offset, nr_segs,
					    xfs_get_blocks_direct,
					    xfs_end_io_direct_write, NULL, 0);
1389
		if (ret != -EIOCBQUEUED && iocb->private)
1390
			goto out_trans_cancel;
1391
	} else {
1392 1393 1394 1395
		ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
					    offset, nr_segs,
					    xfs_get_blocks_direct,
					    NULL, NULL, 0);
1396
	}
1397 1398

	return ret;
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408

out_trans_cancel:
	if (ioend->io_append_trans) {
		current_set_flags_nested(&ioend->io_append_trans->t_pflags,
					 PF_FSTRANS);
		xfs_trans_cancel(ioend->io_append_trans, 0);
	}
out_destroy_ioend:
	xfs_destroy_ioend(ioend);
	return ret;
L
Linus Torvalds 已提交
1409 1410
}

C
Christoph Hellwig 已提交
1411 1412 1413 1414 1415 1416 1417 1418
STATIC void
xfs_vm_write_failed(
	struct address_space	*mapping,
	loff_t			to)
{
	struct inode		*inode = mapping->host;

	if (to > inode->i_size) {
1419
		/*
1420 1421 1422 1423 1424
		 * Punch out the delalloc blocks we have already allocated.
		 *
		 * Don't bother with xfs_setattr given that nothing can have
		 * made it to disk yet as the page is still locked at this
		 * point.
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
		 */
		struct xfs_inode	*ip = XFS_I(inode);
		xfs_fileoff_t		start_fsb;
		xfs_fileoff_t		end_fsb;
		int			error;

		truncate_pagecache(inode, to, inode->i_size);

		/*
		 * Check if there are any blocks that are outside of i_size
		 * that need to be trimmed back.
		 */
1437
		start_fsb = XFS_B_TO_FSB(ip->i_mount, inode->i_size);
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
		end_fsb = XFS_B_TO_FSB(ip->i_mount, to);
		if (end_fsb <= start_fsb)
			return;

		xfs_ilock(ip, XFS_ILOCK_EXCL);
		error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
							end_fsb - start_fsb);
		if (error) {
			/* something screwed, just bail */
			if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
1448
				xfs_alert(ip->i_mount,
1449 1450 1451 1452 1453
			"xfs_vm_write_failed: unable to clean up ino %lld",
						ip->i_ino);
			}
		}
		xfs_iunlock(ip, XFS_ILOCK_EXCL);
C
Christoph Hellwig 已提交
1454 1455 1456
	}
}

1457
STATIC int
N
Nick Piggin 已提交
1458
xfs_vm_write_begin(
1459
	struct file		*file,
N
Nick Piggin 已提交
1460 1461 1462 1463 1464 1465
	struct address_space	*mapping,
	loff_t			pos,
	unsigned		len,
	unsigned		flags,
	struct page		**pagep,
	void			**fsdata)
1466
{
1467 1468 1469 1470
	int			ret;

	ret = block_write_begin(mapping, pos, len, flags | AOP_FLAG_NOFS,
				pagep, xfs_get_blocks);
C
Christoph Hellwig 已提交
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
	if (unlikely(ret))
		xfs_vm_write_failed(mapping, pos + len);
	return ret;
}

STATIC int
xfs_vm_write_end(
	struct file		*file,
	struct address_space	*mapping,
	loff_t			pos,
	unsigned		len,
	unsigned		copied,
	struct page		*page,
	void			*fsdata)
{
	int			ret;
1487

C
Christoph Hellwig 已提交
1488 1489 1490
	ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
	if (unlikely(ret < len))
		xfs_vm_write_failed(mapping, pos + len);
1491
	return ret;
1492
}
L
Linus Torvalds 已提交
1493 1494

STATIC sector_t
1495
xfs_vm_bmap(
L
Linus Torvalds 已提交
1496 1497 1498 1499
	struct address_space	*mapping,
	sector_t		block)
{
	struct inode		*inode = (struct inode *)mapping->host;
1500
	struct xfs_inode	*ip = XFS_I(inode);
L
Linus Torvalds 已提交
1501

C
Christoph Hellwig 已提交
1502
	trace_xfs_vm_bmap(XFS_I(inode));
1503
	xfs_ilock(ip, XFS_IOLOCK_SHARED);
1504
	xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
1505
	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
1506
	return generic_block_bmap(mapping, block, xfs_get_blocks);
L
Linus Torvalds 已提交
1507 1508 1509
}

STATIC int
1510
xfs_vm_readpage(
L
Linus Torvalds 已提交
1511 1512 1513
	struct file		*unused,
	struct page		*page)
{
1514
	return mpage_readpage(page, xfs_get_blocks);
L
Linus Torvalds 已提交
1515 1516 1517
}

STATIC int
1518
xfs_vm_readpages(
L
Linus Torvalds 已提交
1519 1520 1521 1522 1523
	struct file		*unused,
	struct address_space	*mapping,
	struct list_head	*pages,
	unsigned		nr_pages)
{
1524
	return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
L
Linus Torvalds 已提交
1525 1526
}

1527
const struct address_space_operations xfs_address_space_operations = {
1528 1529 1530
	.readpage		= xfs_vm_readpage,
	.readpages		= xfs_vm_readpages,
	.writepage		= xfs_vm_writepage,
1531
	.writepages		= xfs_vm_writepages,
1532 1533
	.releasepage		= xfs_vm_releasepage,
	.invalidatepage		= xfs_vm_invalidatepage,
N
Nick Piggin 已提交
1534
	.write_begin		= xfs_vm_write_begin,
C
Christoph Hellwig 已提交
1535
	.write_end		= xfs_vm_write_end,
1536 1537
	.bmap			= xfs_vm_bmap,
	.direct_IO		= xfs_vm_direct_IO,
1538
	.migratepage		= buffer_migrate_page,
1539
	.is_partially_uptodate  = block_is_partially_uptodate,
1540
	.error_remove_page	= generic_error_remove_page,
L
Linus Torvalds 已提交
1541
};