data.c 26.9 KB
Newer Older
J
Jaegeuk Kim 已提交
1
/*
2 3 4 5 6 7 8 9 10 11 12 13 14
 * fs/f2fs/data.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/buffer_head.h>
#include <linux/mpage.h>
15
#include <linux/aio.h>
16 17 18 19
#include <linux/writeback.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/bio.h>
20
#include <linux/prefetch.h>
21 22 23 24

#include "f2fs.h"
#include "node.h"
#include "segment.h"
25
#include <trace/events/f2fs.h>
26

27 28
static void f2fs_read_end_io(struct bio *bio, int err)
{
29 30
	struct bio_vec *bvec;
	int i;
31

32
	bio_for_each_segment_all(bvec, bio, i) {
33 34
		struct page *page = bvec->bv_page;

35 36 37
		if (!err) {
			SetPageUptodate(page);
		} else {
38 39 40 41
			ClearPageUptodate(page);
			SetPageError(page);
		}
		unlock_page(page);
42
	}
43 44 45 46 47
	bio_put(bio);
}

static void f2fs_write_end_io(struct bio *bio, int err)
{
48
	struct f2fs_sb_info *sbi = bio->bi_private;
49 50
	struct bio_vec *bvec;
	int i;
51

52
	bio_for_each_segment_all(bvec, bio, i) {
53 54
		struct page *page = bvec->bv_page;

55
		if (unlikely(err)) {
56 57
			SetPageError(page);
			set_bit(AS_EIO, &page->mapping->flags);
58
			f2fs_stop_checkpoint(sbi);
59 60 61
		}
		end_page_writeback(page);
		dec_page_count(sbi, F2FS_WRITEBACK);
62
	}
63

64 65 66 67
	if (sbi->wait_io) {
		complete(sbi->wait_io);
		sbi->wait_io = NULL;
	}
68 69 70 71 72 73 74 75

	if (!get_pages(sbi, F2FS_WRITEBACK) &&
			!list_empty(&sbi->cp_wait.task_list))
		wake_up(&sbi->cp_wait);

	bio_put(bio);
}

76 77 78 79 80 81 82 83 84 85 86 87
/*
 * Low-level block read/write IO operations.
 */
static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
				int npages, bool is_read)
{
	struct bio *bio;

	/* No failure on bio allocation */
	bio = bio_alloc(GFP_NOIO, npages);

	bio->bi_bdev = sbi->sb->s_bdev;
88
	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
89
	bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
90
	bio->bi_private = sbi;
91 92 93 94

	return bio;
}

J
Jaegeuk Kim 已提交
95
static void __submit_merged_bio(struct f2fs_bio_info *io)
96
{
J
Jaegeuk Kim 已提交
97 98
	struct f2fs_io_info *fio = &io->fio;
	int rw;
99 100 101 102

	if (!io->bio)
		return;

103
	rw = fio->rw;
104 105

	if (is_read_io(rw)) {
106 107
		trace_f2fs_submit_read_bio(io->sbi->sb, rw,
						fio->type, io->bio);
J
Jaegeuk Kim 已提交
108
		submit_bio(rw, io->bio);
109
	} else {
110 111 112 113 114 115 116 117
		trace_f2fs_submit_write_bio(io->sbi->sb, rw,
						fio->type, io->bio);
		/*
		 * META_FLUSH is only from the checkpoint procedure, and we
		 * should wait this metadata bio for FS consistency.
		 */
		if (fio->type == META_FLUSH) {
			DECLARE_COMPLETION_ONSTACK(wait);
118
			io->sbi->wait_io = &wait;
119 120 121 122 123
			submit_bio(rw, io->bio);
			wait_for_completion(&wait);
		} else {
			submit_bio(rw, io->bio);
		}
124
	}
125

126 127 128 129
	io->bio = NULL;
}

void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
J
Jaegeuk Kim 已提交
130
				enum page_type type, int rw)
131 132 133 134 135 136
{
	enum page_type btype = PAGE_TYPE_OF_BIO(type);
	struct f2fs_bio_info *io;

	io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];

137
	down_write(&io->io_rwsem);
J
Jaegeuk Kim 已提交
138 139 140 141

	/* change META to META_FLUSH in the checkpoint procedure */
	if (type >= META_FLUSH) {
		io->fio.type = META_FLUSH;
J
Jaegeuk Kim 已提交
142 143 144 145
		if (test_opt(sbi, NOBARRIER))
			io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
		else
			io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
J
Jaegeuk Kim 已提交
146 147
	}
	__submit_merged_bio(io);
148
	up_write(&io->io_rwsem);
149 150 151 152 153 154 155 156 157 158 159 160 161 162
}

/*
 * Fill the locked page with data located in the block address.
 * Return unlocked page.
 */
int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
					block_t blk_addr, int rw)
{
	struct bio *bio;

	trace_f2fs_submit_page_bio(page, blk_addr, rw);

	/* Allocate a new bio */
163
	bio = __bio_alloc(sbi, blk_addr, 1, is_read_io(rw));
164 165 166 167 168 169 170 171 172 173 174 175

	if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
		bio_put(bio);
		f2fs_put_page(page, 1);
		return -EFAULT;
	}

	submit_bio(rw, bio);
	return 0;
}

void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
J
Jaegeuk Kim 已提交
176
			block_t blk_addr, struct f2fs_io_info *fio)
177
{
J
Jaegeuk Kim 已提交
178
	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
179
	struct f2fs_bio_info *io;
180
	bool is_read = is_read_io(fio->rw);
181

182
	io = is_read ? &sbi->read_io : &sbi->write_io[btype];
183 184 185

	verify_block_addr(sbi, blk_addr);

186
	down_write(&io->io_rwsem);
187

188
	if (!is_read)
189 190
		inc_page_count(sbi, F2FS_WRITEBACK);

191
	if (io->bio && (io->last_block_in_bio != blk_addr - 1 ||
J
Jaegeuk Kim 已提交
192 193
						io->fio.rw != fio->rw))
		__submit_merged_bio(io);
194 195
alloc_new:
	if (io->bio == NULL) {
196 197 198
		int bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));

		io->bio = __bio_alloc(sbi, blk_addr, bio_blocks, is_read);
J
Jaegeuk Kim 已提交
199
		io->fio = *fio;
200 201 202 203
	}

	if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) <
							PAGE_CACHE_SIZE) {
J
Jaegeuk Kim 已提交
204
		__submit_merged_bio(io);
205 206 207 208 209
		goto alloc_new;
	}

	io->last_block_in_bio = blk_addr;

210
	up_write(&io->io_rwsem);
J
Jaegeuk Kim 已提交
211
	trace_f2fs_submit_page_mbio(page, fio->rw, fio->type, blk_addr);
212 213
}

J
Jaegeuk Kim 已提交
214
/*
215 216 217 218 219 220 221 222 223 224 225 226
 * Lock ordering for the change of data block address:
 * ->data_page
 *  ->node_page
 *    update block addresses in the node page
 */
static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
{
	struct f2fs_node *rn;
	__le32 *addr_array;
	struct page *node_page = dn->node_page;
	unsigned int ofs_in_node = dn->ofs_in_node;

227
	f2fs_wait_on_page_writeback(node_page, NODE);
228

229
	rn = F2FS_NODE(node_page);
230 231 232 233 234 235 236 237 238 239 240

	/* Get physical address of data block */
	addr_array = blkaddr_in_node(rn);
	addr_array[ofs_in_node] = cpu_to_le32(new_addr);
	set_page_dirty(node_page);
}

int reserve_new_block(struct dnode_of_data *dn)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);

241
	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
242
		return -EPERM;
243
	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
244 245
		return -ENOSPC;

246 247
	trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);

248 249
	__set_data_blkaddr(dn, NEW_ADDR);
	dn->data_blkaddr = NEW_ADDR;
250
	mark_inode_dirty(dn->inode);
251 252 253 254
	sync_inode_page(dn);
	return 0;
}

255 256 257 258 259
int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
{
	bool need_put = dn->inode_page ? false : true;
	int err;

260 261 262
	/* if inode_page exists, index should be zero */
	f2fs_bug_on(!need_put && index);

263 264 265
	err = get_dnode_of_data(dn, index, ALLOC_NODE);
	if (err)
		return err;
266

267 268
	if (dn->data_blkaddr == NULL_ADDR)
		err = reserve_new_block(dn);
269
	if (err || need_put)
270 271 272 273
		f2fs_put_dnode(dn);
	return err;
}

274 275 276 277 278 279 280
static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
					struct buffer_head *bh_result)
{
	struct f2fs_inode_info *fi = F2FS_I(inode);
	pgoff_t start_fofs, end_fofs;
	block_t start_blkaddr;

281 282 283
	if (is_inode_flag_set(fi, FI_NO_EXTENT))
		return 0;

284 285 286 287 288 289
	read_lock(&fi->ext.ext_lock);
	if (fi->ext.len == 0) {
		read_unlock(&fi->ext.ext_lock);
		return 0;
	}

290 291
	stat_inc_total_hit(inode->i_sb);

292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
	start_fofs = fi->ext.fofs;
	end_fofs = fi->ext.fofs + fi->ext.len - 1;
	start_blkaddr = fi->ext.blk_addr;

	if (pgofs >= start_fofs && pgofs <= end_fofs) {
		unsigned int blkbits = inode->i_sb->s_blocksize_bits;
		size_t count;

		clear_buffer_new(bh_result);
		map_bh(bh_result, inode->i_sb,
				start_blkaddr + pgofs - start_fofs);
		count = end_fofs - pgofs + 1;
		if (count < (UINT_MAX >> blkbits))
			bh_result->b_size = (count << blkbits);
		else
			bh_result->b_size = UINT_MAX;

309
		stat_inc_read_hit(inode->i_sb);
310 311 312 313 314 315 316 317 318 319 320 321
		read_unlock(&fi->ext.ext_lock);
		return 1;
	}
	read_unlock(&fi->ext.ext_lock);
	return 0;
}

void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
{
	struct f2fs_inode_info *fi = F2FS_I(dn->inode);
	pgoff_t fofs, start_fofs, end_fofs;
	block_t start_blkaddr, end_blkaddr;
322
	int need_update = true;
323

324
	f2fs_bug_on(blk_addr == NEW_ADDR);
325 326
	fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
							dn->ofs_in_node;
327 328 329 330

	/* Update the page address in the parent node */
	__set_data_blkaddr(dn, blk_addr);

331 332 333
	if (is_inode_flag_set(fi, FI_NO_EXTENT))
		return;

334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
	write_lock(&fi->ext.ext_lock);

	start_fofs = fi->ext.fofs;
	end_fofs = fi->ext.fofs + fi->ext.len - 1;
	start_blkaddr = fi->ext.blk_addr;
	end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;

	/* Drop and initialize the matched extent */
	if (fi->ext.len == 1 && fofs == start_fofs)
		fi->ext.len = 0;

	/* Initial extent */
	if (fi->ext.len == 0) {
		if (blk_addr != NULL_ADDR) {
			fi->ext.fofs = fofs;
			fi->ext.blk_addr = blk_addr;
			fi->ext.len = 1;
		}
		goto end_update;
	}

N
Namjae Jeon 已提交
355
	/* Front merge */
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
	if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
		fi->ext.fofs--;
		fi->ext.blk_addr--;
		fi->ext.len++;
		goto end_update;
	}

	/* Back merge */
	if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
		fi->ext.len++;
		goto end_update;
	}

	/* Split the existing extent */
	if (fi->ext.len > 1 &&
		fofs >= start_fofs && fofs <= end_fofs) {
		if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
			fi->ext.len = fofs - start_fofs;
		} else {
			fi->ext.fofs = fofs + 1;
			fi->ext.blk_addr = start_blkaddr +
					fofs - start_fofs + 1;
			fi->ext.len -= fofs - start_fofs + 1;
		}
380 381
	} else {
		need_update = false;
382 383
	}

384 385 386 387 388 389
	/* Finally, if the extent is very fragmented, let's drop the cache. */
	if (fi->ext.len < F2FS_MIN_EXTENT_LEN) {
		fi->ext.len = 0;
		set_inode_flag(fi, FI_NO_EXTENT);
		need_update = true;
	}
390 391
end_update:
	write_unlock(&fi->ext.ext_lock);
392 393 394
	if (need_update)
		sync_inode_page(dn);
	return;
395 396
}

397
struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
398 399 400 401 402 403 404 405 406 407 408 409 410
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct address_space *mapping = inode->i_mapping;
	struct dnode_of_data dn;
	struct page *page;
	int err;

	page = find_get_page(mapping, index);
	if (page && PageUptodate(page))
		return page;
	f2fs_put_page(page, 0);

	set_new_dnode(&dn, inode, NULL, NULL, 0);
411
	err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
412 413 414 415 416 417 418 419
	if (err)
		return ERR_PTR(err);
	f2fs_put_dnode(&dn);

	if (dn.data_blkaddr == NULL_ADDR)
		return ERR_PTR(-ENOENT);

	/* By fallocate(), there is no cached page, but with NEW_ADDR */
420
	if (unlikely(dn.data_blkaddr == NEW_ADDR))
421 422
		return ERR_PTR(-EINVAL);

423
	page = grab_cache_page(mapping, index);
424 425 426
	if (!page)
		return ERR_PTR(-ENOMEM);

427 428 429 430 431
	if (PageUptodate(page)) {
		unlock_page(page);
		return page;
	}

432
	err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
433
					sync ? READ_SYNC : READA);
434 435 436
	if (err)
		return ERR_PTR(err);

437 438
	if (sync) {
		wait_on_page_locked(page);
439
		if (unlikely(!PageUptodate(page))) {
440 441 442
			f2fs_put_page(page, 0);
			return ERR_PTR(-EIO);
		}
443 444 445 446
	}
	return page;
}

J
Jaegeuk Kim 已提交
447
/*
448 449 450 451 452 453 454 455 456 457 458 459
 * If it tries to access a hole, return an error.
 * Because, the callers, functions in dir.c and GC, should be able to know
 * whether this page exists or not.
 */
struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct address_space *mapping = inode->i_mapping;
	struct dnode_of_data dn;
	struct page *page;
	int err;

460
repeat:
461
	page = grab_cache_page(mapping, index);
462 463 464
	if (!page)
		return ERR_PTR(-ENOMEM);

465
	set_new_dnode(&dn, inode, NULL, NULL, 0);
466
	err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
467 468
	if (err) {
		f2fs_put_page(page, 1);
469
		return ERR_PTR(err);
470
	}
471 472
	f2fs_put_dnode(&dn);

473
	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
474
		f2fs_put_page(page, 1);
475
		return ERR_PTR(-ENOENT);
476
	}
477 478 479 480

	if (PageUptodate(page))
		return page;

J
Jaegeuk Kim 已提交
481 482 483 484 485 486 487 488 489 490 491
	/*
	 * A new dentry page is allocated but not able to be written, since its
	 * new inode page couldn't be allocated due to -ENOSPC.
	 * In such the case, its blkaddr can be remained as NEW_ADDR.
	 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
	 */
	if (dn.data_blkaddr == NEW_ADDR) {
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
		SetPageUptodate(page);
		return page;
	}
492

493
	err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr, READ_SYNC);
494
	if (err)
495
		return ERR_PTR(err);
496 497

	lock_page(page);
498
	if (unlikely(!PageUptodate(page))) {
499 500
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
501
	}
502
	if (unlikely(page->mapping != mapping)) {
503 504
		f2fs_put_page(page, 1);
		goto repeat;
505 506 507 508
	}
	return page;
}

J
Jaegeuk Kim 已提交
509
/*
510 511
 * Caller ensures that this data page is never allocated.
 * A new zero-filled data page is allocated in the page cache.
512
 *
C
Chao Yu 已提交
513 514
 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
515
 * Note that, ipage is set only by make_empty_dir.
516
 */
517
struct page *get_new_data_page(struct inode *inode,
518
		struct page *ipage, pgoff_t index, bool new_i_size)
519 520 521 522 523 524 525
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
	struct dnode_of_data dn;
	int err;

526
	set_new_dnode(&dn, inode, ipage, NULL, 0);
527
	err = f2fs_reserve_block(&dn, index);
528 529
	if (err)
		return ERR_PTR(err);
530
repeat:
531
	page = grab_cache_page(mapping, index);
532 533 534 535
	if (!page) {
		err = -ENOMEM;
		goto put_err;
	}
536 537 538 539 540 541

	if (PageUptodate(page))
		return page;

	if (dn.data_blkaddr == NEW_ADDR) {
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
542
		SetPageUptodate(page);
543
	} else {
544 545
		err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
								READ_SYNC);
546
		if (err)
547 548
			goto put_err;

549
		lock_page(page);
550
		if (unlikely(!PageUptodate(page))) {
551
			f2fs_put_page(page, 1);
552 553
			err = -EIO;
			goto put_err;
554
		}
555
		if (unlikely(page->mapping != mapping)) {
556 557
			f2fs_put_page(page, 1);
			goto repeat;
558 559 560 561 562 563
		}
	}

	if (new_i_size &&
		i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
		i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
564 565
		/* Only the directory inode sets new_i_size */
		set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
566 567
	}
	return page;
568 569 570 571

put_err:
	f2fs_put_dnode(&dn);
	return ERR_PTR(err);
572 573
}

574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
static int __allocate_data_block(struct dnode_of_data *dn)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct f2fs_summary sum;
	block_t new_blkaddr;
	struct node_info ni;
	int type;

	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
		return -EPERM;
	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
		return -ENOSPC;

	__set_data_blkaddr(dn, NEW_ADDR);
	dn->data_blkaddr = NEW_ADDR;

	get_node_info(sbi, dn->nid, &ni);
	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);

	type = CURSEG_WARM_DATA;

	allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type);

	/* direct IO doesn't use extent cache to maximize the performance */
	set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
	update_extent_cache(new_blkaddr, dn);
	clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);

	dn->data_blkaddr = new_blkaddr;
	return 0;
}

J
Jaegeuk Kim 已提交
606
/*
C
Chao Yu 已提交
607 608 609 610 611 612
 * get_data_block() now supported readahead/bmap/rw direct_IO with mapped bh.
 * If original data blocks are allocated, then give them to blockdev.
 * Otherwise,
 *     a. preallocate requested block addresses
 *     b. do not use extent cache for better performance
 *     c. give the block addresses to blockdev
613
 */
614 615
static int __get_data_block(struct inode *inode, sector_t iblock,
			struct buffer_head *bh_result, int create, bool fiemap)
616
{
617
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
618 619 620
	unsigned int blkbits = inode->i_sb->s_blocksize_bits;
	unsigned maxblocks = bh_result->b_size >> blkbits;
	struct dnode_of_data dn;
621 622 623 624
	int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
	pgoff_t pgofs, end_offset;
	int err = 0, ofs = 1;
	bool allocated = false;
625 626 627 628

	/* Get the page offset from the block offset(iblock) */
	pgofs =	(pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));

629 630 631
	if (check_extent_cache(inode, pgofs, bh_result))
		goto out;

632 633
	if (create) {
		f2fs_balance_fs(sbi);
634
		f2fs_lock_op(sbi);
635
	}
636 637 638

	/* When reading holes, we need its node page */
	set_new_dnode(&dn, inode, NULL, NULL, 0);
639
	err = get_dnode_of_data(&dn, pgofs, mode);
640
	if (err) {
641 642 643
		if (err == -ENOENT)
			err = 0;
		goto unlock_out;
644
	}
645
	if (dn.data_blkaddr == NEW_ADDR && !fiemap)
646
		goto put_out;
647

648 649 650 651 652 653 654 655 656 657 658 659
	if (dn.data_blkaddr != NULL_ADDR) {
		map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
	} else if (create) {
		err = __allocate_data_block(&dn);
		if (err)
			goto put_out;
		allocated = true;
		map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
	} else {
		goto put_out;
	}

660
	end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
661 662 663 664 665 666 667 668 669 670 671 672 673
	bh_result->b_size = (((size_t)1) << blkbits);
	dn.ofs_in_node++;
	pgofs++;

get_next:
	if (dn.ofs_in_node >= end_offset) {
		if (allocated)
			sync_inode_page(&dn);
		allocated = false;
		f2fs_put_dnode(&dn);

		set_new_dnode(&dn, inode, NULL, NULL, 0);
		err = get_dnode_of_data(&dn, pgofs, mode);
674
		if (err) {
675 676 677 678
			if (err == -ENOENT)
				err = 0;
			goto unlock_out;
		}
679
		if (dn.data_blkaddr == NEW_ADDR && !fiemap)
680 681
			goto put_out;

682
		end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
683
	}
684

685 686 687 688 689 690 691 692 693
	if (maxblocks > (bh_result->b_size >> blkbits)) {
		block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
		if (blkaddr == NULL_ADDR && create) {
			err = __allocate_data_block(&dn);
			if (err)
				goto sync_out;
			allocated = true;
			blkaddr = dn.data_blkaddr;
		}
694
		/* Give more consecutive addresses for the read ahead */
695 696 697 698 699 700 701
		if (blkaddr == (bh_result->b_blocknr + ofs)) {
			ofs++;
			dn.ofs_in_node++;
			pgofs++;
			bh_result->b_size += (((size_t)1) << blkbits);
			goto get_next;
		}
702
	}
703 704 705 706
sync_out:
	if (allocated)
		sync_inode_page(&dn);
put_out:
707
	f2fs_put_dnode(&dn);
708 709 710 711 712 713
unlock_out:
	if (create)
		f2fs_unlock_op(sbi);
out:
	trace_f2fs_get_data_block(inode, iblock, bh_result, err);
	return err;
714 715
}

716 717 718 719 720 721 722 723 724 725 726 727
static int get_data_block(struct inode *inode, sector_t iblock,
			struct buffer_head *bh_result, int create)
{
	return __get_data_block(inode, iblock, bh_result, create, false);
}

static int get_data_block_fiemap(struct inode *inode, sector_t iblock,
			struct buffer_head *bh_result, int create)
{
	return __get_data_block(inode, iblock, bh_result, create, true);
}

J
Jaegeuk Kim 已提交
728 729 730
int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
		u64 start, u64 len)
{
731 732
	return generic_block_fiemap(inode, fieinfo,
				start, len, get_data_block_fiemap);
J
Jaegeuk Kim 已提交
733 734
}

735 736
static int f2fs_read_data_page(struct file *file, struct page *page)
{
H
Huajun Li 已提交
737 738 739
	struct inode *inode = page->mapping->host;
	int ret;

740 741
	trace_f2fs_readpage(page, DATA);

H
Huajun Li 已提交
742 743 744 745 746 747 748
	/* If the file has inline data, try to read it directlly */
	if (f2fs_has_inline_data(inode))
		ret = f2fs_read_inline_data(inode, page);
	else
		ret = mpage_readpage(page, get_data_block);

	return ret;
749 750 751 752 753 754
}

static int f2fs_read_data_pages(struct file *file,
			struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
H
Huajun Li 已提交
755 756 757 758 759 760
	struct inode *inode = file->f_mapping->host;

	/* If the file has inline data, skip readpages */
	if (f2fs_has_inline_data(inode))
		return 0;

761
	return mpage_readpages(mapping, pages, nr_pages, get_data_block);
762 763
}

J
Jaegeuk Kim 已提交
764
int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
765 766
{
	struct inode *inode = page->mapping->host;
J
Jaegeuk Kim 已提交
767
	block_t old_blkaddr, new_blkaddr;
768 769 770 771
	struct dnode_of_data dn;
	int err = 0;

	set_new_dnode(&dn, inode, NULL, NULL, 0);
772
	err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
773 774 775
	if (err)
		return err;

J
Jaegeuk Kim 已提交
776
	old_blkaddr = dn.data_blkaddr;
777 778

	/* This page is already truncated */
J
Jaegeuk Kim 已提交
779
	if (old_blkaddr == NULL_ADDR)
780 781 782 783 784 785 786 787
		goto out_writepage;

	set_page_writeback(page);

	/*
	 * If current allocation needs SSR,
	 * it had better in-place writes for updated data.
	 */
J
Jaegeuk Kim 已提交
788
	if (unlikely(old_blkaddr != NEW_ADDR &&
789 790
			!is_cold_data(page) &&
			need_inplace_update(inode))) {
J
Jaegeuk Kim 已提交
791
		rewrite_data_page(page, old_blkaddr, fio);
792
		set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
793
	} else {
J
Jaegeuk Kim 已提交
794 795
		write_data_page(page, &dn, &new_blkaddr, fio);
		update_extent_cache(new_blkaddr, &dn);
796
		set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
797 798 799 800 801 802 803 804 805 806 807 808 809 810
	}
out_writepage:
	f2fs_put_dnode(&dn);
	return err;
}

static int f2fs_write_data_page(struct page *page,
					struct writeback_control *wbc)
{
	struct inode *inode = page->mapping->host;
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	loff_t i_size = i_size_read(inode);
	const pgoff_t end_index = ((unsigned long long) i_size)
							>> PAGE_CACHE_SHIFT;
H
Huajun Li 已提交
811
	unsigned offset = 0;
812
	bool need_balance_fs = false;
813
	int err = 0;
J
Jaegeuk Kim 已提交
814 815
	struct f2fs_io_info fio = {
		.type = DATA,
C
Chris Fries 已提交
816
		.rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
J
Jaegeuk Kim 已提交
817
	};
818

819 820
	trace_f2fs_writepage(page, DATA);

821
	if (page->index < end_index)
822
		goto write;
823 824 825 826 827 828

	/*
	 * If the offset is out-of-range of file size,
	 * this page does not have to be written to disk.
	 */
	offset = i_size & (PAGE_CACHE_SIZE - 1);
829
	if ((page->index >= end_index + 1) || !offset)
830
		goto out;
831 832

	zero_user_segment(page, offset, PAGE_CACHE_SIZE);
833
write:
834
	if (unlikely(sbi->por_doing))
835 836
		goto redirty_out;

837
	/* Dentry blocks are controlled by checkpoint */
838
	if (S_ISDIR(inode->i_mode)) {
J
Jaegeuk Kim 已提交
839
		err = do_write_data_page(page, &fio);
840 841
		goto done;
	}
H
Huajun Li 已提交
842

843
	if (!wbc->for_reclaim)
844
		need_balance_fs = true;
845
	else if (has_not_enough_free_secs(sbi, 0))
846
		goto redirty_out;
847

848 849 850 851 852 853 854 855 856
	f2fs_lock_op(sbi);
	if (f2fs_has_inline_data(inode) || f2fs_may_inline(inode))
		err = f2fs_write_inline_data(inode, page, offset);
	else
		err = do_write_data_page(page, &fio);
	f2fs_unlock_op(sbi);
done:
	if (err && err != -ENOENT)
		goto redirty_out;
857 858

	clear_cold_data(page);
859
out:
860
	inode_dec_dirty_dents(inode);
861
	unlock_page(page);
862
	if (need_balance_fs)
863
		f2fs_balance_fs(sbi);
864 865
	if (wbc->for_reclaim)
		f2fs_submit_merged_bio(sbi, DATA, WRITE);
866 867 868
	return 0;

redirty_out:
869
	redirty_page_for_writepage(wbc, page);
870
	return AOP_WRITEPAGE_ACTIVATE;
871 872
}

873 874 875 876 877 878 879 880 881
static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
			void *data)
{
	struct address_space *mapping = data;
	int ret = mapping->a_ops->writepage(page, wbc);
	mapping_set_error(mapping, ret);
	return ret;
}

882
static int f2fs_write_data_pages(struct address_space *mapping,
883 884 885 886
			    struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
887
	bool locked = false;
888
	int ret;
889
	long diff;
890

891 892
	trace_f2fs_writepages(mapping->host, wbc, DATA);

P
P J P 已提交
893 894 895 896
	/* deal with chardevs and other special file */
	if (!mapping->a_ops->writepage)
		return 0;

897
	if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
898 899
			get_dirty_dents(inode) < nr_pages_to_skip(sbi, DATA) &&
			available_free_memory(sbi, DIRTY_DENTS))
900
		goto skip_write;
901

902
	diff = nr_pages_to_write(sbi, DATA, wbc);
903

904
	if (!S_ISDIR(inode->i_mode)) {
905
		mutex_lock(&sbi->writepages);
906 907
		locked = true;
	}
908
	ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
909
	if (locked)
910
		mutex_unlock(&sbi->writepages);
J
Jaegeuk Kim 已提交
911 912

	f2fs_submit_merged_bio(sbi, DATA, WRITE);
913 914 915

	remove_dirty_dir_inode(inode);

916
	wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
917
	return ret;
918 919 920 921

skip_write:
	wbc->pages_skipped += get_dirty_dents(inode);
	return 0;
922 923
}

924 925 926 927 928 929 930 931 932 933
static void f2fs_write_failed(struct address_space *mapping, loff_t to)
{
	struct inode *inode = mapping->host;

	if (to > inode->i_size) {
		truncate_pagecache(inode, inode->i_size);
		truncate_blocks(inode, inode->i_size);
	}
}

934 935 936 937 938 939 940 941 942 943 944
static int f2fs_write_begin(struct file *file, struct address_space *mapping,
		loff_t pos, unsigned len, unsigned flags,
		struct page **pagep, void **fsdata)
{
	struct inode *inode = mapping->host;
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct page *page;
	pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
	struct dnode_of_data dn;
	int err = 0;

945 946
	trace_f2fs_write_begin(inode, pos, len, flags);

947
	f2fs_balance_fs(sbi);
948
repeat:
949 950
	err = f2fs_convert_inline_data(inode, pos + len);
	if (err)
951
		goto fail;
952

953
	page = grab_cache_page_write_begin(mapping, index, flags);
954 955 956 957
	if (!page) {
		err = -ENOMEM;
		goto fail;
	}
958 959 960 961

	/* to avoid latency during memory pressure */
	unlock_page(page);

962 963
	*pagep = page;

964 965
	if (f2fs_has_inline_data(inode) && (pos + len) <= MAX_INLINE_DATA)
		goto inline_data;
H
Huajun Li 已提交
966

967
	f2fs_lock_op(sbi);
968
	set_new_dnode(&dn, inode, NULL, NULL, 0);
969
	err = f2fs_reserve_block(&dn, index);
970
	f2fs_unlock_op(sbi);
971
	if (err) {
972
		f2fs_put_page(page, 0);
973
		goto fail;
974
	}
H
Huajun Li 已提交
975
inline_data:
976 977 978 979 980 981 982 983
	lock_page(page);
	if (unlikely(page->mapping != mapping)) {
		f2fs_put_page(page, 1);
		goto repeat;
	}

	f2fs_wait_on_page_writeback(page, DATA);

984 985 986 987 988 989 990 991 992
	if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
		return 0;

	if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
		unsigned start = pos & (PAGE_CACHE_SIZE - 1);
		unsigned end = start + len;

		/* Reading beyond i_size is simple: memset to zero */
		zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
993
		goto out;
994 995 996 997 998
	}

	if (dn.data_blkaddr == NEW_ADDR) {
		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
	} else {
999
		if (f2fs_has_inline_data(inode)) {
H
Huajun Li 已提交
1000
			err = f2fs_read_inline_data(inode, page);
1001 1002
			if (err) {
				page_cache_release(page);
1003
				goto fail;
1004 1005
			}
		} else {
H
Huajun Li 已提交
1006
			err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
1007
							READ_SYNC);
1008
			if (err)
1009
				goto fail;
1010 1011
		}

1012
		lock_page(page);
1013
		if (unlikely(!PageUptodate(page))) {
1014
			f2fs_put_page(page, 1);
1015 1016
			err = -EIO;
			goto fail;
1017
		}
1018
		if (unlikely(page->mapping != mapping)) {
1019 1020
			f2fs_put_page(page, 1);
			goto repeat;
1021 1022
		}
	}
1023
out:
1024 1025 1026
	SetPageUptodate(page);
	clear_cold_data(page);
	return 0;
1027 1028 1029
fail:
	f2fs_write_failed(mapping, pos + len);
	return err;
1030 1031
}

1032 1033 1034 1035 1036 1037 1038
static int f2fs_write_end(struct file *file,
			struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
{
	struct inode *inode = page->mapping->host;

1039 1040
	trace_f2fs_write_end(inode, pos, len, copied);

1041 1042 1043 1044 1045 1046 1047 1048
	set_page_dirty(page);

	if (pos + copied > i_size_read(inode)) {
		i_size_write(inode, pos + copied);
		mark_inode_dirty(inode);
		update_inode_page(inode);
	}

1049
	f2fs_put_page(page, 1);
1050 1051 1052
	return copied;
}

1053
static int check_direct_IO(struct inode *inode, int rw,
A
Al Viro 已提交
1054
		struct iov_iter *iter, loff_t offset)
1055 1056 1057 1058 1059 1060 1061 1062 1063
{
	unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;

	if (rw == READ)
		return 0;

	if (offset & blocksize_mask)
		return -EINVAL;

A
Al Viro 已提交
1064 1065 1066
	if (iov_iter_alignment(iter) & blocksize_mask)
		return -EINVAL;

1067 1068 1069
	return 0;
}

1070
static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
A
Al Viro 已提交
1071
		struct iov_iter *iter, loff_t offset)
1072 1073
{
	struct file *file = iocb->ki_filp;
1074 1075 1076 1077
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	size_t count = iov_iter_count(iter);
	int err;
1078

H
Huajun Li 已提交
1079 1080 1081 1082
	/* Let buffer I/O handle the inline data case. */
	if (f2fs_has_inline_data(inode))
		return 0;

A
Al Viro 已提交
1083
	if (check_direct_IO(inode, rw, iter, offset))
1084 1085
		return 0;

1086 1087 1088
	/* clear fsync mark to recover these blocks */
	fsync_mark_clear(F2FS_SB(inode->i_sb), inode->i_ino);

1089 1090 1091 1092
	err = blockdev_direct_IO(rw, iocb, inode, iter, offset, get_data_block);
	if (err < 0 && (rw & WRITE))
		f2fs_write_failed(mapping, offset + count);
	return err;
1093 1094
}

1095 1096
static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
				      unsigned int length)
1097 1098
{
	struct inode *inode = page->mapping->host;
1099
	if (PageDirty(page))
1100 1101 1102 1103 1104 1105 1106
		inode_dec_dirty_dents(inode);
	ClearPagePrivate(page);
}

static int f2fs_release_data_page(struct page *page, gfp_t wait)
{
	ClearPagePrivate(page);
1107
	return 1;
1108 1109 1110 1111 1112 1113 1114
}

static int f2fs_set_data_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;

1115 1116
	trace_f2fs_set_page_dirty(page, DATA);

1117
	SetPageUptodate(page);
1118 1119
	mark_inode_dirty(inode);

1120 1121 1122 1123 1124 1125 1126 1127
	if (!PageDirty(page)) {
		__set_page_dirty_nobuffers(page);
		set_dirty_dir_page(inode, page);
		return 1;
	}
	return 0;
}

J
Jaegeuk Kim 已提交
1128 1129
static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
{
1130 1131 1132 1133 1134
	struct inode *inode = mapping->host;

	if (f2fs_has_inline_data(inode))
		return 0;

1135
	return generic_block_bmap(mapping, block, get_data_block);
J
Jaegeuk Kim 已提交
1136 1137
}

1138 1139 1140 1141 1142 1143
const struct address_space_operations f2fs_dblock_aops = {
	.readpage	= f2fs_read_data_page,
	.readpages	= f2fs_read_data_pages,
	.writepage	= f2fs_write_data_page,
	.writepages	= f2fs_write_data_pages,
	.write_begin	= f2fs_write_begin,
1144
	.write_end	= f2fs_write_end,
1145 1146 1147 1148
	.set_page_dirty	= f2fs_set_data_page_dirty,
	.invalidatepage	= f2fs_invalidate_data_page,
	.releasepage	= f2fs_release_data_page,
	.direct_IO	= f2fs_direct_IO,
J
Jaegeuk Kim 已提交
1149
	.bmap		= f2fs_bmap,
1150
};