data.c 94.6 KB
Newer Older
C
Chao Yu 已提交
1
// SPDX-License-Identifier: GPL-2.0
J
Jaegeuk Kim 已提交
2
/*
3 4 5 6 7 8 9 10 11 12 13
 * fs/f2fs/data.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/buffer_head.h>
#include <linux/mpage.h>
#include <linux/writeback.h>
#include <linux/backing-dev.h>
C
Chao Yu 已提交
14
#include <linux/pagevec.h>
15 16
#include <linux/blkdev.h>
#include <linux/bio.h>
J
Jaegeuk Kim 已提交
17
#include <linux/swap.h>
18
#include <linux/prefetch.h>
19
#include <linux/uio.h>
J
Jaegeuk Kim 已提交
20
#include <linux/cleancache.h>
21
#include <linux/sched/signal.h>
22 23 24 25

#include "f2fs.h"
#include "node.h"
#include "segment.h"
J
Jaegeuk Kim 已提交
26
#include "trace.h"
27
#include <trace/events/f2fs.h>
28

29 30 31
#define NUM_PREALLOC_POST_READ_CTXS	128

static struct kmem_cache *bio_post_read_ctx_cache;
C
Chao Yu 已提交
32
static struct kmem_cache *bio_entry_slab;
33
static mempool_t *bio_post_read_ctx_pool;
C
Chao Yu 已提交
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
static struct bio_set f2fs_bioset;

#define	F2FS_BIO_POOL_SIZE	NR_CURSEG_TYPE

int __init f2fs_init_bioset(void)
{
	if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE,
					0, BIOSET_NEED_BVECS))
		return -ENOMEM;
	return 0;
}

void f2fs_destroy_bioset(void)
{
	bioset_exit(&f2fs_bioset);
}

static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask,
						unsigned int nr_iovecs)
{
	return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);
}

57
struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio)
C
Chao Yu 已提交
58
{
59
	if (noio) {
C
Chao Yu 已提交
60
		/* No failure on bio allocation */
61
		return __f2fs_bio_alloc(GFP_NOIO, npages);
C
Chao Yu 已提交
62
	}
63

C
Chao Yu 已提交
64 65 66 67 68 69 70
	if (time_to_inject(sbi, FAULT_ALLOC_BIO)) {
		f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO);
		return NULL;
	}

	return __f2fs_bio_alloc(GFP_KERNEL, npages);
}
71

72 73 74 75 76 77 78 79 80
static bool __is_cp_guaranteed(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct inode *inode;
	struct f2fs_sb_info *sbi;

	if (!mapping)
		return false;

C
Chao Yu 已提交
81 82 83
	if (f2fs_is_compressed_page(page))
		return false;

84 85 86 87 88 89
	inode = mapping->host;
	sbi = F2FS_I_SB(inode);

	if (inode->i_ino == F2FS_META_INO(sbi) ||
			inode->i_ino ==  F2FS_NODE_INO(sbi) ||
			S_ISDIR(inode->i_mode) ||
90
			(S_ISREG(inode->i_mode) &&
91
			(f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) ||
92 93 94 95 96
			is_cold_data(page))
		return true;
	return false;
}

97 98
static enum count_type __read_io_type(struct page *page)
{
J
Jaegeuk Kim 已提交
99
	struct address_space *mapping = page_file_mapping(page);
100 101 102 103 104 105 106 107 108 109 110 111 112 113

	if (mapping) {
		struct inode *inode = mapping->host;
		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);

		if (inode->i_ino == F2FS_META_INO(sbi))
			return F2FS_RD_META;

		if (inode->i_ino == F2FS_NODE_INO(sbi))
			return F2FS_RD_NODE;
	}
	return F2FS_RD_DATA;
}

114 115 116
/* postprocessing steps for read bios */
enum bio_post_read_step {
	STEP_DECRYPT,
C
Chao Yu 已提交
117
	STEP_DECOMPRESS,
E
Eric Biggers 已提交
118
	STEP_VERITY,
119 120 121 122
};

struct bio_post_read_ctx {
	struct bio *bio;
C
Chao Yu 已提交
123
	struct f2fs_sb_info *sbi;
124 125 126 127
	struct work_struct work;
	unsigned int enabled_steps;
};

C
Chao Yu 已提交
128
static void __read_end_io(struct bio *bio, bool compr, bool verity)
129
{
130 131
	struct page *page;
	struct bio_vec *bv;
132
	struct bvec_iter_all iter_all;
133

134
	bio_for_each_segment_all(bv, bio, iter_all) {
135 136
		page = bv->bv_page;

C
Chao Yu 已提交
137 138 139 140 141
#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (compr && f2fs_is_compressed_page(page)) {
			f2fs_decompress_pages(bio, page, verity);
			continue;
		}
C
Chao Yu 已提交
142 143
		if (verity)
			continue;
C
Chao Yu 已提交
144 145
#endif

146 147 148
		/* PG_error was set if any post_read step failed */
		if (bio->bi_status || PageError(page)) {
			ClearPageUptodate(page);
149 150
			/* will re-read again later */
			ClearPageError(page);
151 152 153
		} else {
			SetPageUptodate(page);
		}
154
		dec_page_count(F2FS_P_SB(page), __read_io_type(page));
155 156
		unlock_page(page);
	}
C
Chao Yu 已提交
157 158 159 160 161 162 163 164 165 166 167 168 169
}

static void f2fs_release_read_bio(struct bio *bio);
static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity)
{
	if (!compr)
		__read_end_io(bio, false, verity);
	f2fs_release_read_bio(bio);
}

static void f2fs_decompress_bio(struct bio *bio, bool verity)
{
	__read_end_io(bio, true, verity);
170 171 172 173
}

static void bio_post_read_processing(struct bio_post_read_ctx *ctx);

C
Chao Yu 已提交
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx)
{
	fscrypt_decrypt_bio(ctx->bio);
}

static void f2fs_decompress_work(struct bio_post_read_ctx *ctx)
{
	f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY));
}

#ifdef CONFIG_F2FS_FS_COMPRESSION
static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size)
{
	f2fs_decompress_end_io(rpages, cluster_size, false, true);
}

static void f2fs_verify_bio(struct bio *bio)
{
192 193 194 195 196 197
	struct bio_vec *bv;
	struct bvec_iter_all iter_all;

	bio_for_each_segment_all(bv, bio, iter_all) {
		struct page *page = bv->bv_page;
		struct decompress_io_ctx *dic;
C
Chao Yu 已提交
198

199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
		dic = (struct decompress_io_ctx *)page_private(page);

		if (dic) {
			if (refcount_dec_not_one(&dic->ref))
				continue;
			f2fs_verify_pages(dic->rpages,
						dic->cluster_size);
			f2fs_free_dic(dic);
			continue;
		}

		if (bio->bi_status || PageError(page))
			goto clear_uptodate;

		if (fsverity_verify_page(page)) {
			SetPageUptodate(page);
			goto unlock;
		}
clear_uptodate:
		ClearPageUptodate(page);
		ClearPageError(page);
unlock:
C
Chao Yu 已提交
221
		dec_page_count(F2FS_P_SB(page), __read_io_type(page));
222 223
		unlock_page(page);
	}
C
Chao Yu 已提交
224 225 226 227
}
#endif

static void f2fs_verity_work(struct work_struct *work)
228 229 230
{
	struct bio_post_read_ctx *ctx =
		container_of(work, struct bio_post_read_ctx, work);
231 232 233 234 235 236 237 238 239 240 241 242 243 244
	struct bio *bio = ctx->bio;
#ifdef CONFIG_F2FS_FS_COMPRESSION
	unsigned int enabled_steps = ctx->enabled_steps;
#endif

	/*
	 * fsverity_verify_bio() may call readpages() again, and while verity
	 * will be disabled for this, decryption may still be needed, resulting
	 * in another bio_post_read_ctx being allocated.  So to prevent
	 * deadlocks we need to release the current ctx to the mempool first.
	 * This assumes that verity is the last post-read step.
	 */
	mempool_free(ctx, bio_post_read_ctx_pool);
	bio->bi_private = NULL;
245

C
Chao Yu 已提交
246 247
#ifdef CONFIG_F2FS_FS_COMPRESSION
	/* previous step is decompression */
248 249 250
	if (enabled_steps & (1 << STEP_DECOMPRESS)) {
		f2fs_verify_bio(bio);
		f2fs_release_read_bio(bio);
C
Chao Yu 已提交
251 252 253
		return;
	}
#endif
254

255 256
	fsverity_verify_bio(bio);
	__f2fs_read_end_io(bio, false, false);
257 258
}

C
Chao Yu 已提交
259
static void f2fs_post_read_work(struct work_struct *work)
E
Eric Biggers 已提交
260 261 262 263
{
	struct bio_post_read_ctx *ctx =
		container_of(work, struct bio_post_read_ctx, work);

C
Chao Yu 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
	if (ctx->enabled_steps & (1 << STEP_DECRYPT))
		f2fs_decrypt_work(ctx);

	if (ctx->enabled_steps & (1 << STEP_DECOMPRESS))
		f2fs_decompress_work(ctx);

	if (ctx->enabled_steps & (1 << STEP_VERITY)) {
		INIT_WORK(&ctx->work, f2fs_verity_work);
		fsverity_enqueue_verify_work(&ctx->work);
		return;
	}

	__f2fs_read_end_io(ctx->bio,
		ctx->enabled_steps & (1 << STEP_DECOMPRESS), false);
}
E
Eric Biggers 已提交
279

C
Chao Yu 已提交
280 281 282 283
static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi,
						struct work_struct *work)
{
	queue_work(sbi->post_read_wq, work);
E
Eric Biggers 已提交
284 285
}

286 287
static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
{
E
Eric Biggers 已提交
288 289 290 291 292
	/*
	 * We use different work queues for decryption and for verity because
	 * verity may require reading metadata pages that need decryption, and
	 * we shouldn't recurse to the same workqueue.
	 */
C
Chao Yu 已提交
293 294 295 296 297 298 299 300 301 302 303 304

	if (ctx->enabled_steps & (1 << STEP_DECRYPT) ||
		ctx->enabled_steps & (1 << STEP_DECOMPRESS)) {
		INIT_WORK(&ctx->work, f2fs_post_read_work);
		f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work);
		return;
	}

	if (ctx->enabled_steps & (1 << STEP_VERITY)) {
		INIT_WORK(&ctx->work, f2fs_verity_work);
		fsverity_enqueue_verify_work(&ctx->work);
		return;
305
	}
C
Chao Yu 已提交
306 307

	__f2fs_read_end_io(ctx->bio, false, false);
308 309 310 311
}

static bool f2fs_bio_post_read_required(struct bio *bio)
{
C
Chao Yu 已提交
312
	return bio->bi_private;
313 314 315 316
}

static void f2fs_read_end_io(struct bio *bio)
{
317 318 319 320
	struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio));

	if (time_to_inject(sbi, FAULT_READ_IO)) {
		f2fs_show_injection_info(sbi, FAULT_READ_IO);
321
		bio->bi_status = BLK_STS_IOERR;
322
	}
C
Chao Yu 已提交
323

324 325
	if (f2fs_bio_post_read_required(bio)) {
		struct bio_post_read_ctx *ctx = bio->bi_private;
J
Jaegeuk Kim 已提交
326

327 328
		bio_post_read_processing(ctx);
		return;
J
Jaegeuk Kim 已提交
329
	}
330

C
Chao Yu 已提交
331
	__f2fs_read_end_io(bio, false, false);
J
Jaegeuk Kim 已提交
332 333
}

334
static void f2fs_write_end_io(struct bio *bio)
335
{
336
	struct f2fs_sb_info *sbi = bio->bi_private;
337
	struct bio_vec *bvec;
338
	struct bvec_iter_all iter_all;
339

340
	if (time_to_inject(sbi, FAULT_WRITE_IO)) {
341
		f2fs_show_injection_info(sbi, FAULT_WRITE_IO);
342 343 344
		bio->bi_status = BLK_STS_IOERR;
	}

345
	bio_for_each_segment_all(bvec, bio, iter_all) {
346
		struct page *page = bvec->bv_page;
347
		enum count_type type = WB_DATA_TYPE(page);
348

349 350 351 352 353 354
		if (IS_DUMMY_WRITTEN_PAGE(page)) {
			set_page_private(page, (unsigned long)NULL);
			ClearPagePrivate(page);
			unlock_page(page);
			mempool_free(page, sbi->write_io_dummy);

355
			if (unlikely(bio->bi_status))
356 357 358 359
				f2fs_stop_checkpoint(sbi, true);
			continue;
		}

360
		fscrypt_finalize_bounce_page(&page);
361

C
Chao Yu 已提交
362 363 364 365 366 367 368
#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (f2fs_is_compressed_page(page)) {
			f2fs_compress_write_end_io(bio, page);
			continue;
		}
#endif

369
		if (unlikely(bio->bi_status)) {
370
			mapping_set_error(page->mapping, -EIO);
371 372
			if (type == F2FS_WB_CP_DATA)
				f2fs_stop_checkpoint(sbi, true);
373
		}
374 375 376 377

		f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
					page->index != nid_of_node(page));

378
		dec_page_count(sbi, type);
379 380
		if (f2fs_in_warm_node_list(sbi, page))
			f2fs_del_fsync_node_entry(sbi, page);
381
		clear_cold_data(page);
382
		end_page_writeback(page);
383
	}
384
	if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
385
				wq_has_sleeper(&sbi->cp_wait))
386 387 388 389 390
		wake_up(&sbi->cp_wait);

	bio_put(bio);
}

J
Jaegeuk Kim 已提交
391 392 393 394 395 396
struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
				block_t blk_addr, struct bio *bio)
{
	struct block_device *bdev = sbi->sb->s_bdev;
	int i;

397 398 399 400 401 402 403 404
	if (f2fs_is_multi_device(sbi)) {
		for (i = 0; i < sbi->s_ndevs; i++) {
			if (FDEV(i).start_blk <= blk_addr &&
			    FDEV(i).end_blk >= blk_addr) {
				blk_addr -= FDEV(i).start_blk;
				bdev = FDEV(i).bdev;
				break;
			}
J
Jaegeuk Kim 已提交
405 406 407
		}
	}
	if (bio) {
408
		bio_set_dev(bio, bdev);
J
Jaegeuk Kim 已提交
409 410 411 412 413 414 415 416 417
		bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
	}
	return bdev;
}

int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
{
	int i;

418 419 420
	if (!f2fs_is_multi_device(sbi))
		return 0;

J
Jaegeuk Kim 已提交
421 422 423 424 425 426
	for (i = 0; i < sbi->s_ndevs; i++)
		if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
			return i;
	return 0;
}

C
Chao Yu 已提交
427 428 429
/*
 * Return true, if pre_bio's bdev is same as its target device.
 */
J
Jaegeuk Kim 已提交
430 431 432
static bool __same_bdev(struct f2fs_sb_info *sbi,
				block_t blk_addr, struct bio *bio)
{
433 434
	struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
	return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
J
Jaegeuk Kim 已提交
435 436
}

437
static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
438
{
439
	struct f2fs_sb_info *sbi = fio->sbi;
440 441
	struct bio *bio;

442
	bio = f2fs_bio_alloc(sbi, npages, true);
443

444 445
	f2fs_target_device(sbi, fio->new_blkaddr, bio);
	if (is_read_io(fio->op)) {
446 447 448 449 450
		bio->bi_end_io = f2fs_read_end_io;
		bio->bi_private = NULL;
	} else {
		bio->bi_end_io = f2fs_write_end_io;
		bio->bi_private = sbi;
451 452
		bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi,
						fio->type, fio->temp);
453
	}
454 455
	if (fio->io_wbc)
		wbc_init_bio(fio->io_wbc, bio);
456 457 458 459

	return bio;
}

460 461
static inline void __submit_bio(struct f2fs_sb_info *sbi,
				struct bio *bio, enum page_type type)
462
{
463
	if (!is_read_io(bio_op(bio))) {
464 465 466 467 468
		unsigned int start;

		if (type != DATA && type != NODE)
			goto submit_io;

469
		if (f2fs_lfs_mode(sbi) && current->plug)
470 471
			blk_finish_plug(current->plug);

472 473 474
		if (F2FS_IO_ALIGNED(sbi))
			goto submit_io;

475 476 477 478 479 480 481 482 483 484
		start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
		start %= F2FS_IO_SIZE(sbi);

		if (start == 0)
			goto submit_io;

		/* fill dummy pages */
		for (; start < F2FS_IO_SIZE(sbi); start++) {
			struct page *page =
				mempool_alloc(sbi->write_io_dummy,
485
					      GFP_NOIO | __GFP_NOFAIL);
486 487
			f2fs_bug_on(sbi, !page);

488
			zero_user_segment(page, 0, PAGE_SIZE);
489 490 491 492 493 494 495 496 497 498 499 500
			SetPagePrivate(page);
			set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
			lock_page(page);
			if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
				f2fs_bug_on(sbi, 1);
		}
		/*
		 * In the NODE case, we lose next block address chain. So, we
		 * need to do checkpoint in f2fs_sync_file.
		 */
		if (type == NODE)
			set_sbi_flag(sbi, SBI_NEED_CP);
J
Jaegeuk Kim 已提交
501
	}
502
submit_io:
J
Jaegeuk Kim 已提交
503 504 505 506
	if (is_read_io(bio_op(bio)))
		trace_f2fs_submit_read_bio(sbi->sb, type, bio);
	else
		trace_f2fs_submit_write_bio(sbi->sb, type, bio);
507
	submit_bio(bio);
508 509
}

C
Chao Yu 已提交
510 511 512 513 514 515
void f2fs_submit_bio(struct f2fs_sb_info *sbi,
				struct bio *bio, enum page_type type)
{
	__submit_bio(sbi, bio, type);
}

516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
static void __attach_data_io_flag(struct f2fs_io_info *fio)
{
	struct f2fs_sb_info *sbi = fio->sbi;
	unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1;
	unsigned int fua_flag = sbi->data_io_flag & temp_mask;
	unsigned int meta_flag = (sbi->data_io_flag >> NR_TEMP_TYPE) &
								temp_mask;
	/*
	 * data io flag bits per temp:
	 *      REQ_META     |      REQ_FUA      |
	 *    5 |    4 |   3 |    2 |    1 |   0 |
	 * Cold | Warm | Hot | Cold | Warm | Hot |
	 */
	if (fio->type != DATA)
		return;

	if ((1 << fio->temp) & meta_flag)
		fio->op_flags |= REQ_META;
	if ((1 << fio->temp) & fua_flag)
		fio->op_flags |= REQ_FUA;
}

J
Jaegeuk Kim 已提交
538
static void __submit_merged_bio(struct f2fs_bio_info *io)
539
{
J
Jaegeuk Kim 已提交
540
	struct f2fs_io_info *fio = &io->fio;
541 542 543 544

	if (!io->bio)
		return;

545
	__attach_data_io_flag(fio);
J
Jaegeuk Kim 已提交
546 547
	bio_set_op_attrs(io->bio, fio->op, fio->op_flags);

M
Mike Christie 已提交
548
	if (is_read_io(fio->op))
J
Jaegeuk Kim 已提交
549
		trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
550
	else
J
Jaegeuk Kim 已提交
551
		trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
M
Mike Christie 已提交
552

553
	__submit_bio(io->sbi, io->bio, fio->type);
554 555 556
	io->bio = NULL;
}

C
Chao Yu 已提交
557
static bool __has_merged_page(struct bio *bio, struct inode *inode,
558
						struct page *page, nid_t ino)
C
Chao Yu 已提交
559 560
{
	struct bio_vec *bvec;
561
	struct bvec_iter_all iter_all;
C
Chao Yu 已提交
562

C
Chao Yu 已提交
563
	if (!bio)
C
Chao Yu 已提交
564
		return false;
565

566
	if (!inode && !page && !ino)
567
		return true;
C
Chao Yu 已提交
568

C
Chao Yu 已提交
569
	bio_for_each_segment_all(bvec, bio, iter_all) {
C
Chao Yu 已提交
570
		struct page *target = bvec->bv_page;
C
Chao Yu 已提交
571

C
Chao Yu 已提交
572
		if (fscrypt_is_bounce_page(target)) {
573
			target = fscrypt_pagecache_page(target);
C
Chao Yu 已提交
574 575 576 577 578 579 580 581
			if (IS_ERR(target))
				continue;
		}
		if (f2fs_is_compressed_page(target)) {
			target = f2fs_compress_control_page(target);
			if (IS_ERR(target))
				continue;
		}
C
Chao Yu 已提交
582

583 584
		if (inode && inode == target->mapping->host)
			return true;
585 586
		if (page && page == target)
			return true;
587
		if (ino && ino == ino_of_node(target))
C
Chao Yu 已提交
588 589 590 591 592 593
			return true;
	}

	return false;
}

594
static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
J
Jaegeuk Kim 已提交
595
				enum page_type type, enum temp_type temp)
596 597
{
	enum page_type btype = PAGE_TYPE_OF_BIO(type);
J
Jaegeuk Kim 已提交
598
	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
599

600
	down_write(&io->io_rwsem);
J
Jaegeuk Kim 已提交
601 602 603 604

	/* change META to META_FLUSH in the checkpoint procedure */
	if (type >= META_FLUSH) {
		io->fio.type = META_FLUSH;
M
Mike Christie 已提交
605
		io->fio.op = REQ_OP_WRITE;
606
		io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
607
		if (!test_opt(sbi, NOBARRIER))
608
			io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
J
Jaegeuk Kim 已提交
609 610
	}
	__submit_merged_bio(io);
611
	up_write(&io->io_rwsem);
612 613
}

J
Jaegeuk Kim 已提交
614
static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
615 616
				struct inode *inode, struct page *page,
				nid_t ino, enum page_type type, bool force)
617
{
J
Jaegeuk Kim 已提交
618
	enum temp_type temp;
619
	bool ret = true;
J
Jaegeuk Kim 已提交
620 621

	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
622 623 624
		if (!force)	{
			enum page_type btype = PAGE_TYPE_OF_BIO(type);
			struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
J
Jaegeuk Kim 已提交
625

626
			down_read(&io->io_rwsem);
C
Chao Yu 已提交
627
			ret = __has_merged_page(io->bio, inode, page, ino);
628 629 630 631
			up_read(&io->io_rwsem);
		}
		if (ret)
			__f2fs_submit_merged_write(sbi, type, temp);
J
Jaegeuk Kim 已提交
632 633 634 635 636

		/* TODO: use HOT temp only for meta pages now. */
		if (type >= META)
			break;
	}
637 638
}

639
void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
640
{
641
	__submit_merged_write_cond(sbi, NULL, NULL, 0, type, true);
642 643
}

644
void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
645 646
				struct inode *inode, struct page *page,
				nid_t ino, enum page_type type)
647
{
648
	__submit_merged_write_cond(sbi, inode, page, ino, type, false);
649 650
}

651
void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
652
{
653 654 655
	f2fs_submit_merged_write(sbi, DATA);
	f2fs_submit_merged_write(sbi, NODE);
	f2fs_submit_merged_write(sbi, META);
656 657
}

658 659
/*
 * Fill the locked page with data located in the block address.
660
 * A caller needs to unlock the page on failure.
661
 */
662
int f2fs_submit_page_bio(struct f2fs_io_info *fio)
663 664
{
	struct bio *bio;
665 666
	struct page *page = fio->encrypted_page ?
			fio->encrypted_page : fio->page;
667

668
	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
C
Chao Yu 已提交
669 670
			fio->is_por ? META_POR : (__is_meta_io(fio) ?
			META_GENERIC : DATA_GENERIC_ENHANCE)))
671
		return -EFSCORRUPTED;
672

673
	trace_f2fs_submit_page_bio(page, fio);
674
	f2fs_trace_ios(fio, 0);
675 676

	/* Allocate a new bio */
677
	bio = __bio_alloc(fio, 1);
678

679
	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
680 681 682
		bio_put(bio);
		return -EFAULT;
	}
683 684

	if (fio->io_wbc && !is_read_io(fio->op))
685
		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
686

M
Mike Christie 已提交
687
	bio_set_op_attrs(bio, fio->op, fio->op_flags);
688

689 690
	inc_page_count(fio->sbi, is_read_io(fio->op) ?
			__read_io_type(page): WB_DATA_TYPE(fio->page));
C
Chao Yu 已提交
691 692

	__submit_bio(fio->sbi, bio, fio->type);
693 694 695
	return 0;
}

696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
				block_t last_blkaddr, block_t cur_blkaddr)
{
	if (last_blkaddr + 1 != cur_blkaddr)
		return false;
	return __same_bdev(sbi, cur_blkaddr, bio);
}

static bool io_type_is_mergeable(struct f2fs_bio_info *io,
						struct f2fs_io_info *fio)
{
	if (io->fio.op != fio->op)
		return false;
	return io->fio.op_flags == fio->op_flags;
}

static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
					struct f2fs_bio_info *io,
					struct f2fs_io_info *fio,
					block_t last_blkaddr,
					block_t cur_blkaddr)
{
718 719 720 721 722 723 724 725 726 727
	if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) {
		unsigned int filled_blocks =
				F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size);
		unsigned int io_size = F2FS_IO_SIZE(sbi);
		unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt;

		/* IOs in bio is aligned and left space of vectors is not enough */
		if (!(filled_blocks % io_size) && left_vecs < io_size)
			return false;
	}
728 729 730 731 732
	if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr))
		return false;
	return io_type_is_mergeable(io, fio);
}

C
Chao Yu 已提交
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
				struct page *page, enum temp_type temp)
{
	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
	struct bio_entry *be;

	be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);
	be->bio = bio;
	bio_get(bio);

	if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
		f2fs_bug_on(sbi, 1);

	down_write(&io->bio_list_lock);
	list_add_tail(&be->list, &io->bio_list);
	up_write(&io->bio_list_lock);
}

static void del_bio_entry(struct bio_entry *be)
{
	list_del(&be->list);
	kmem_cache_free(bio_entry_slab, be);
}

static int add_ipu_page(struct f2fs_sb_info *sbi, struct bio **bio,
							struct page *page)
{
	enum temp_type temp;
	bool found = false;
	int ret = -EAGAIN;

	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
		struct list_head *head = &io->bio_list;
		struct bio_entry *be;

		down_write(&io->bio_list_lock);
		list_for_each_entry(be, head, list) {
			if (be->bio != *bio)
				continue;

			found = true;

C
Chao Yu 已提交
776 777
			if (bio_add_page(*bio, page, PAGE_SIZE, 0) ==
							PAGE_SIZE) {
C
Chao Yu 已提交
778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
				ret = 0;
				break;
			}

			/* bio is full */
			del_bio_entry(be);
			__submit_bio(sbi, *bio, DATA);
			break;
		}
		up_write(&io->bio_list_lock);
	}

	if (ret) {
		bio_put(*bio);
		*bio = NULL;
	}

	return ret;
}

void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
					struct bio **bio, struct page *page)
{
	enum temp_type temp;
	bool found = false;
	struct bio *target = bio ? *bio : NULL;

	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
		struct list_head *head = &io->bio_list;
		struct bio_entry *be;

		if (list_empty(head))
			continue;

		down_read(&io->bio_list_lock);
		list_for_each_entry(be, head, list) {
			if (target)
				found = (target == be->bio);
			else
				found = __has_merged_page(be->bio, NULL,
								page, 0);
			if (found)
				break;
		}
		up_read(&io->bio_list_lock);

		if (!found)
			continue;

		found = false;

		down_write(&io->bio_list_lock);
		list_for_each_entry(be, head, list) {
			if (target)
				found = (target == be->bio);
			else
				found = __has_merged_page(be->bio, NULL,
								page, 0);
			if (found) {
				target = be->bio;
				del_bio_entry(be);
				break;
			}
		}
		up_write(&io->bio_list_lock);
	}

	if (found)
		__submit_bio(sbi, target, DATA);
	if (bio && *bio) {
		bio_put(*bio);
		*bio = NULL;
	}
}

C
Chao Yu 已提交
854 855 856 857 858 859 860 861
int f2fs_merge_page_bio(struct f2fs_io_info *fio)
{
	struct bio *bio = *fio->bio;
	struct page *page = fio->encrypted_page ?
			fio->encrypted_page : fio->page;

	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
			__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
862
		return -EFSCORRUPTED;
C
Chao Yu 已提交
863 864 865 866

	trace_f2fs_submit_page_bio(page, fio);
	f2fs_trace_ios(fio, 0);

867
	if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
C
Chao Yu 已提交
868 869
						fio->new_blkaddr))
		f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
C
Chao Yu 已提交
870 871
alloc_new:
	if (!bio) {
872
		bio = __bio_alloc(fio, BIO_MAX_PAGES);
C
Chao Yu 已提交
873 874
		bio_set_op_attrs(bio, fio->op, fio->op_flags);

C
Chao Yu 已提交
875 876 877 878
		add_bio_entry(fio->sbi, bio, page, fio->temp);
	} else {
		if (add_ipu_page(fio->sbi, &bio, page))
			goto alloc_new;
C
Chao Yu 已提交
879 880 881
	}

	if (fio->io_wbc)
882
		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
C
Chao Yu 已提交
883 884 885 886 887 888 889 890 891

	inc_page_count(fio->sbi, WB_DATA_TYPE(page));

	*fio->last_block = fio->new_blkaddr;
	*fio->bio = bio;

	return 0;
}

892
void f2fs_submit_page_write(struct f2fs_io_info *fio)
893
{
894
	struct f2fs_sb_info *sbi = fio->sbi;
J
Jaegeuk Kim 已提交
895
	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
J
Jaegeuk Kim 已提交
896
	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
897
	struct page *bio_page;
898

899
	f2fs_bug_on(sbi, is_read_io(fio->op));
900

901 902 903 904 905 906
	down_write(&io->io_rwsem);
next:
	if (fio->in_list) {
		spin_lock(&io->io_lock);
		if (list_empty(&io->io_list)) {
			spin_unlock(&io->io_lock);
907
			goto out;
908 909 910 911 912 913
		}
		fio = list_first_entry(&io->io_list,
						struct f2fs_io_info, list);
		list_del(&fio->list);
		spin_unlock(&io->io_lock);
	}
914

C
Chao Yu 已提交
915
	verify_fio_blkaddr(fio);
916

C
Chao Yu 已提交
917 918 919 920 921 922
	if (fio->encrypted_page)
		bio_page = fio->encrypted_page;
	else if (fio->compressed_page)
		bio_page = fio->compressed_page;
	else
		bio_page = fio->page;
923

924 925
	/* set submitted = true as a return value */
	fio->submitted = true;
926

927
	inc_page_count(sbi, WB_DATA_TYPE(bio_page));
928

929 930
	if (io->bio && !io_is_mergeable(sbi, io->bio, io, fio,
			io->last_block_in_bio, fio->new_blkaddr))
J
Jaegeuk Kim 已提交
931
		__submit_merged_bio(io);
932 933
alloc_new:
	if (io->bio == NULL) {
934 935
		if (F2FS_IO_ALIGNED(sbi) &&
				(fio->type == DATA || fio->type == NODE) &&
936
				fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
937
			dec_page_count(sbi, WB_DATA_TYPE(bio_page));
938 939
			fio->retry = true;
			goto skip;
940
		}
941
		io->bio = __bio_alloc(fio, BIO_MAX_PAGES);
J
Jaegeuk Kim 已提交
942
		io->fio = *fio;
943 944
	}

J
Jaegeuk Kim 已提交
945
	if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
J
Jaegeuk Kim 已提交
946
		__submit_merged_bio(io);
947 948 949
		goto alloc_new;
	}

950
	if (fio->io_wbc)
951
		wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);
952

953
	io->last_block_in_bio = fio->new_blkaddr;
954
	f2fs_trace_ios(fio, 0);
955 956

	trace_f2fs_submit_page_write(fio->page, fio);
957
skip:
958 959
	if (fio->in_list)
		goto next;
960
out:
D
Daniel Rosenberg 已提交
961
	if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
962
				!f2fs_is_checkpoint_ready(sbi))
J
Jaegeuk Kim 已提交
963
		__submit_merged_bio(io);
964
	up_write(&io->io_rwsem);
965 966
}

E
Eric Biggers 已提交
967 968 969 970 971 972
static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
{
	return fsverity_active(inode) &&
	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
}

973
static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
E
Eric Biggers 已提交
974
				      unsigned nr_pages, unsigned op_flag,
975
				      pgoff_t first_idx, bool for_write)
976 977 978
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct bio *bio;
979 980
	struct bio_post_read_ctx *ctx;
	unsigned int post_read_steps = 0;
981

982 983
	bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES),
								for_write);
984
	if (!bio)
985 986 987
		return ERR_PTR(-ENOMEM);
	f2fs_target_device(sbi, blkaddr, bio);
	bio->bi_end_io = f2fs_read_end_io;
988
	bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
989

990 991
	if (f2fs_encrypted_file(inode))
		post_read_steps |= 1 << STEP_DECRYPT;
C
Chao Yu 已提交
992 993
	if (f2fs_compressed_file(inode))
		post_read_steps |= 1 << STEP_DECOMPRESS;
E
Eric Biggers 已提交
994 995 996
	if (f2fs_need_verity(inode, first_idx))
		post_read_steps |= 1 << STEP_VERITY;

997
	if (post_read_steps) {
998
		/* Due to the mempool, this never fails. */
999 1000
		ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
		ctx->bio = bio;
C
Chao Yu 已提交
1001
		ctx->sbi = sbi;
1002 1003 1004 1005
		ctx->enabled_steps = post_read_steps;
		bio->bi_private = ctx;
	}

1006 1007 1008
	return bio;
}

C
Chao Yu 已提交
1009 1010 1011 1012 1013 1014 1015
static void f2fs_release_read_bio(struct bio *bio)
{
	if (bio->bi_private)
		mempool_free(bio->bi_private, bio_post_read_ctx_pool);
	bio_put(bio);
}

1016 1017
/* This can handle encryption stuffs */
static int f2fs_submit_page_read(struct inode *inode, struct page *page,
1018
						block_t blkaddr, bool for_write)
1019
{
C
Chao Yu 已提交
1020 1021
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct bio *bio;
1022

1023
	bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0, page->index, for_write);
1024 1025 1026
	if (IS_ERR(bio))
		return PTR_ERR(bio);

1027 1028 1029
	/* wait for GCed page writeback via META_MAPPING */
	f2fs_wait_on_block_writeback(inode, blkaddr);

1030 1031 1032 1033
	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
		bio_put(bio);
		return -EFAULT;
	}
1034
	ClearPageError(page);
C
Chao Yu 已提交
1035
	inc_page_count(sbi, F2FS_RD_DATA);
C
Chao Yu 已提交
1036
	f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
C
Chao Yu 已提交
1037
	__submit_bio(sbi, bio, DATA);
1038 1039 1040
	return 0;
}

1041 1042 1043 1044
static void __set_data_blkaddr(struct dnode_of_data *dn)
{
	struct f2fs_node *rn = F2FS_NODE(dn->node_page);
	__le32 *addr_array;
1045 1046 1047 1048
	int base = 0;

	if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
		base = get_extra_isize(dn->inode);
1049 1050 1051

	/* Get physical address of data block */
	addr_array = blkaddr_in_node(rn);
1052
	addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
1053 1054
}

J
Jaegeuk Kim 已提交
1055
/*
1056 1057 1058 1059 1060
 * Lock ordering for the change of data block address:
 * ->data_page
 *  ->node_page
 *    update block addresses in the node page
 */
C
Chao Yu 已提交
1061
void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
1062
{
1063
	f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
1064 1065
	__set_data_blkaddr(dn);
	if (set_page_dirty(dn->node_page))
1066
		dn->node_changed = true;
1067 1068
}

1069 1070 1071
void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
{
	dn->data_blkaddr = blkaddr;
C
Chao Yu 已提交
1072
	f2fs_set_data_blkaddr(dn);
1073 1074 1075
	f2fs_update_extent_cache(dn);
}

1076
/* dn->ofs_in_node will be returned with up-to-date last block pointer */
C
Chao Yu 已提交
1077
int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
1078
{
1079
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
C
Chao Yu 已提交
1080
	int err;
1081

1082 1083 1084
	if (!count)
		return 0;

1085
	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1086
		return -EPERM;
C
Chao Yu 已提交
1087 1088
	if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
		return err;
1089

1090 1091 1092
	trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
						dn->ofs_in_node, count);

1093
	f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
1094 1095

	for (; count > 0; dn->ofs_in_node++) {
1096
		block_t blkaddr = f2fs_data_blkaddr(dn);
1097 1098 1099 1100 1101 1102 1103 1104 1105
		if (blkaddr == NULL_ADDR) {
			dn->data_blkaddr = NEW_ADDR;
			__set_data_blkaddr(dn);
			count--;
		}
	}

	if (set_page_dirty(dn->node_page))
		dn->node_changed = true;
1106 1107 1108
	return 0;
}

1109
/* Should keep dn->ofs_in_node unchanged */
C
Chao Yu 已提交
1110
int f2fs_reserve_new_block(struct dnode_of_data *dn)
1111 1112 1113 1114
{
	unsigned int ofs_in_node = dn->ofs_in_node;
	int ret;

C
Chao Yu 已提交
1115
	ret = f2fs_reserve_new_blocks(dn, 1);
1116 1117 1118 1119
	dn->ofs_in_node = ofs_in_node;
	return ret;
}

1120 1121 1122 1123 1124
int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
{
	bool need_put = dn->inode_page ? false : true;
	int err;

C
Chao Yu 已提交
1125
	err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
1126 1127
	if (err)
		return err;
1128

1129
	if (dn->data_blkaddr == NULL_ADDR)
C
Chao Yu 已提交
1130
		err = f2fs_reserve_new_block(dn);
1131
	if (err || need_put)
1132 1133 1134 1135
		f2fs_put_dnode(dn);
	return err;
}

1136
int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
1137
{
1138
	struct extent_info ei  = {0,0,0};
1139
	struct inode *inode = dn->inode;
1140

1141 1142 1143
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn->data_blkaddr = ei.blk + index - ei.fofs;
		return 0;
1144
	}
1145

1146
	return f2fs_reserve_block(dn, index);
1147 1148
}

C
Chao Yu 已提交
1149
struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
M
Mike Christie 已提交
1150
						int op_flags, bool for_write)
1151 1152 1153 1154
{
	struct address_space *mapping = inode->i_mapping;
	struct dnode_of_data dn;
	struct page *page;
1155
	struct extent_info ei = {0,0,0};
1156
	int err;
1157

1158
	page = f2fs_grab_cache_page(mapping, index, for_write);
1159 1160 1161
	if (!page)
		return ERR_PTR(-ENOMEM);

C
Chao Yu 已提交
1162 1163
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn.data_blkaddr = ei.blk + index - ei.fofs;
C
Chao Yu 已提交
1164 1165
		if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
						DATA_GENERIC_ENHANCE_READ)) {
1166
			err = -EFSCORRUPTED;
C
Chao Yu 已提交
1167 1168
			goto put_err;
		}
C
Chao Yu 已提交
1169 1170 1171
		goto got_it;
	}

1172
	set_new_dnode(&dn, inode, NULL, NULL, 0);
C
Chao Yu 已提交
1173
	err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
1174 1175
	if (err)
		goto put_err;
1176 1177
	f2fs_put_dnode(&dn);

1178
	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
1179 1180
		err = -ENOENT;
		goto put_err;
1181
	}
C
Chao Yu 已提交
1182 1183 1184 1185
	if (dn.data_blkaddr != NEW_ADDR &&
			!f2fs_is_valid_blkaddr(F2FS_I_SB(inode),
						dn.data_blkaddr,
						DATA_GENERIC_ENHANCE)) {
1186
		err = -EFSCORRUPTED;
C
Chao Yu 已提交
1187 1188
		goto put_err;
	}
C
Chao Yu 已提交
1189
got_it:
1190 1191
	if (PageUptodate(page)) {
		unlock_page(page);
1192
		return page;
1193
	}
1194

J
Jaegeuk Kim 已提交
1195 1196 1197 1198
	/*
	 * A new dentry page is allocated but not able to be written, since its
	 * new inode page couldn't be allocated due to -ENOSPC.
	 * In such the case, its blkaddr can be remained as NEW_ADDR.
C
Chao Yu 已提交
1199 1200
	 * see, f2fs_add_link -> f2fs_get_new_data_page ->
	 * f2fs_init_inode_metadata.
J
Jaegeuk Kim 已提交
1201 1202
	 */
	if (dn.data_blkaddr == NEW_ADDR) {
1203
		zero_user_segment(page, 0, PAGE_SIZE);
1204 1205
		if (!PageUptodate(page))
			SetPageUptodate(page);
1206
		unlock_page(page);
J
Jaegeuk Kim 已提交
1207 1208
		return page;
	}
1209

1210
	err = f2fs_submit_page_read(inode, page, dn.data_blkaddr, for_write);
1211
	if (err)
1212
		goto put_err;
1213
	return page;
1214 1215 1216 1217

put_err:
	f2fs_put_page(page, 1);
	return ERR_PTR(err);
1218 1219
}

C
Chao Yu 已提交
1220
struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
1221 1222 1223 1224 1225 1226 1227 1228 1229
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;

	page = find_get_page(mapping, index);
	if (page && PageUptodate(page))
		return page;
	f2fs_put_page(page, 0);

C
Chao Yu 已提交
1230
	page = f2fs_get_read_data_page(inode, index, 0, false);
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
	if (IS_ERR(page))
		return page;

	if (PageUptodate(page))
		return page;

	wait_on_page_locked(page);
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 0);
		return ERR_PTR(-EIO);
	}
	return page;
}

/*
 * If it tries to access a hole, return an error.
 * Because, the callers, functions in dir.c and GC, should be able to know
 * whether this page exists or not.
 */
C
Chao Yu 已提交
1250
struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
1251
							bool for_write)
1252 1253 1254 1255
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
repeat:
C
Chao Yu 已提交
1256
	page = f2fs_get_read_data_page(inode, index, 0, for_write);
1257 1258
	if (IS_ERR(page))
		return page;
1259

1260
	/* wait for read completion */
1261
	lock_page(page);
1262
	if (unlikely(page->mapping != mapping)) {
1263 1264
		f2fs_put_page(page, 1);
		goto repeat;
1265
	}
1266 1267 1268 1269
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
	}
1270 1271 1272
	return page;
}

J
Jaegeuk Kim 已提交
1273
/*
1274 1275
 * Caller ensures that this data page is never allocated.
 * A new zero-filled data page is allocated in the page cache.
1276
 *
C
Chao Yu 已提交
1277 1278
 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
1279 1280
 * Note that, ipage is set only by make_empty_dir, and if any error occur,
 * ipage should be released by this function.
1281
 */
C
Chao Yu 已提交
1282
struct page *f2fs_get_new_data_page(struct inode *inode,
1283
		struct page *ipage, pgoff_t index, bool new_i_size)
1284 1285 1286 1287 1288
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
	struct dnode_of_data dn;
	int err;
1289

1290
	page = f2fs_grab_cache_page(mapping, index, true);
1291 1292 1293 1294 1295 1296
	if (!page) {
		/*
		 * before exiting, we should make sure ipage will be released
		 * if any error occur.
		 */
		f2fs_put_page(ipage, 1);
1297
		return ERR_PTR(-ENOMEM);
1298
	}
1299

1300
	set_new_dnode(&dn, inode, ipage, NULL, 0);
1301
	err = f2fs_reserve_block(&dn, index);
1302 1303
	if (err) {
		f2fs_put_page(page, 1);
1304
		return ERR_PTR(err);
1305
	}
1306 1307
	if (!ipage)
		f2fs_put_dnode(&dn);
1308 1309

	if (PageUptodate(page))
1310
		goto got_it;
1311 1312

	if (dn.data_blkaddr == NEW_ADDR) {
1313
		zero_user_segment(page, 0, PAGE_SIZE);
1314 1315
		if (!PageUptodate(page))
			SetPageUptodate(page);
1316
	} else {
1317
		f2fs_put_page(page, 1);
1318

1319 1320
		/* if ipage exists, blkaddr should be NEW_ADDR */
		f2fs_bug_on(F2FS_I_SB(inode), ipage);
C
Chao Yu 已提交
1321
		page = f2fs_get_lock_data_page(inode, index, true);
1322
		if (IS_ERR(page))
1323
			return page;
1324
	}
1325
got_it:
C
Chao Yu 已提交
1326
	if (new_i_size && i_size_read(inode) <
1327
				((loff_t)(index + 1) << PAGE_SHIFT))
1328
		f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
1329 1330 1331
	return page;
}

1332
static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
1333
{
1334
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1335 1336
	struct f2fs_summary sum;
	struct node_info ni;
1337
	block_t old_blkaddr;
1338
	blkcnt_t count = 1;
C
Chao Yu 已提交
1339
	int err;
1340

1341
	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1342
		return -EPERM;
1343

1344 1345 1346 1347
	err = f2fs_get_node_info(sbi, dn->nid, &ni);
	if (err)
		return err;

1348
	dn->data_blkaddr = f2fs_data_blkaddr(dn);
1349
	if (dn->data_blkaddr != NULL_ADDR)
1350 1351
		goto alloc;

C
Chao Yu 已提交
1352 1353
	if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
		return err;
1354

1355
alloc:
1356
	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1357 1358
	old_blkaddr = dn->data_blkaddr;
	f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
1359
					&sum, seg_type, NULL, false);
1360 1361 1362
	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
		invalidate_mapping_pages(META_MAPPING(sbi),
					old_blkaddr, old_blkaddr);
1363
	f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1364

1365 1366 1367 1368
	/*
	 * i_size will be updated by direct_IO. Otherwise, we'll get stale
	 * data from unwritten block via dio_read.
	 */
1369 1370 1371
	return 0;
}

1372
int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
1373
{
1374
	struct inode *inode = file_inode(iocb->ki_filp);
C
Chao Yu 已提交
1375
	struct f2fs_map_blocks map;
1376
	int flag;
1377
	int err = 0;
1378
	bool direct_io = iocb->ki_flags & IOCB_DIRECT;
1379

1380
	map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
1381 1382 1383 1384 1385 1386
	map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
	if (map.m_len > map.m_lblk)
		map.m_len -= map.m_lblk;
	else
		map.m_len = 0;

1387
	map.m_next_pgofs = NULL;
1388
	map.m_next_extent = NULL;
1389
	map.m_seg_type = NO_CHECK_TYPE;
C
Chao Yu 已提交
1390
	map.m_may_create = true;
1391

1392
	if (direct_io) {
C
Chao Yu 已提交
1393
		map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint);
1394
		flag = f2fs_force_buffered_io(inode, iocb, from) ?
1395 1396 1397
					F2FS_GET_BLOCK_PRE_AIO :
					F2FS_GET_BLOCK_PRE_DIO;
		goto map_blocks;
1398
	}
C
Chao Yu 已提交
1399
	if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
1400 1401 1402
		err = f2fs_convert_inline_inode(inode);
		if (err)
			return err;
1403
	}
1404
	if (f2fs_has_inline_data(inode))
1405
		return err;
1406 1407 1408 1409 1410 1411 1412 1413 1414

	flag = F2FS_GET_BLOCK_PRE_AIO;

map_blocks:
	err = f2fs_map_blocks(inode, &map, 1, flag);
	if (map.m_len > 0 && err == -ENOSPC) {
		if (!direct_io)
			set_inode_flag(inode, FI_NO_PREALLOC);
		err = 0;
1415
	}
1416
	return err;
1417 1418
}

C
Chao Yu 已提交
1419
void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
{
	if (flag == F2FS_GET_BLOCK_PRE_AIO) {
		if (lock)
			down_read(&sbi->node_change);
		else
			up_read(&sbi->node_change);
	} else {
		if (lock)
			f2fs_lock_op(sbi);
		else
			f2fs_unlock_op(sbi);
	}
}

J
Jaegeuk Kim 已提交
1434
/*
C
Chao Yu 已提交
1435 1436 1437
 * f2fs_map_blocks() tries to find or build mapping relationship which
 * maps continuous logical blocks to physical blocks, and return such
 * info via f2fs_map_blocks structure.
1438
 */
C
Chao Yu 已提交
1439
int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
C
Chao Yu 已提交
1440
						int create, int flag)
1441
{
J
Jaegeuk Kim 已提交
1442
	unsigned int maxblocks = map->m_len;
1443
	struct dnode_of_data dn;
1444
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
C
Chao Yu 已提交
1445
	int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
1446
	pgoff_t pgofs, end_offset, end;
1447
	int err = 0, ofs = 1;
1448 1449
	unsigned int ofs_in_node, last_ofs_in_node;
	blkcnt_t prealloc;
1450
	struct extent_info ei = {0,0,0};
1451
	block_t blkaddr;
1452
	unsigned int start_pgofs;
1453

1454 1455 1456
	if (!maxblocks)
		return 0;

J
Jaegeuk Kim 已提交
1457 1458 1459 1460 1461
	map->m_len = 0;
	map->m_flags = 0;

	/* it only supports block size == page size */
	pgofs =	(pgoff_t)map->m_lblk;
1462
	end = pgofs + maxblocks;
1463

1464
	if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
1465
		if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
1466 1467 1468
							map->m_may_create)
			goto next_dnode;

J
Jaegeuk Kim 已提交
1469 1470 1471
		map->m_pblk = ei.blk + pgofs - ei.fofs;
		map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
		map->m_flags = F2FS_MAP_MAPPED;
1472 1473
		if (map->m_next_extent)
			*map->m_next_extent = pgofs + map->m_len;
1474 1475 1476 1477 1478

		/* for hardware encryption, but to avoid potential issue in future */
		if (flag == F2FS_GET_BLOCK_DIO)
			f2fs_wait_on_block_writeback_range(inode,
						map->m_pblk, map->m_len);
1479
		goto out;
1480
	}
1481

C
Chao Yu 已提交
1482
next_dnode:
C
Chao Yu 已提交
1483
	if (map->m_may_create)
1484
		__do_map_lock(sbi, flag, true);
1485 1486 1487

	/* When reading holes, we need its node page */
	set_new_dnode(&dn, inode, NULL, NULL, 0);
C
Chao Yu 已提交
1488
	err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1489
	if (err) {
C
Chao Yu 已提交
1490 1491
		if (flag == F2FS_GET_BLOCK_BMAP)
			map->m_pblk = 0;
1492
		if (err == -ENOENT) {
1493
			err = 0;
1494 1495
			if (map->m_next_pgofs)
				*map->m_next_pgofs =
C
Chao Yu 已提交
1496
					f2fs_get_next_page_offset(&dn, pgofs);
1497 1498
			if (map->m_next_extent)
				*map->m_next_extent =
C
Chao Yu 已提交
1499
					f2fs_get_next_page_offset(&dn, pgofs);
1500
		}
1501
		goto unlock_out;
1502
	}
C
Chao Yu 已提交
1503

1504
	start_pgofs = pgofs;
1505
	prealloc = 0;
1506
	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1507
	end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
C
Chao Yu 已提交
1508 1509

next_block:
1510
	blkaddr = f2fs_data_blkaddr(&dn);
C
Chao Yu 已提交
1511

1512
	if (__is_valid_data_blkaddr(blkaddr) &&
C
Chao Yu 已提交
1513
		!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
1514
		err = -EFSCORRUPTED;
1515 1516 1517
		goto sync_out;
	}

C
Chao Yu 已提交
1518
	if (__is_valid_data_blkaddr(blkaddr)) {
1519
		/* use out-place-update for driect IO under LFS mode */
1520
		if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
C
Chao Yu 已提交
1521
							map->m_may_create) {
1522
			err = __allocate_data_block(&dn, map->m_seg_type);
1523 1524 1525 1526
			if (err)
				goto sync_out;
			blkaddr = dn.data_blkaddr;
			set_inode_flag(inode, FI_APPEND_WRITE);
1527 1528
		}
	} else {
C
Chao Yu 已提交
1529
		if (create) {
1530 1531
			if (unlikely(f2fs_cp_error(sbi))) {
				err = -EIO;
C
Chao Yu 已提交
1532
				goto sync_out;
1533
			}
1534
			if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1535 1536 1537 1538
				if (blkaddr == NULL_ADDR) {
					prealloc++;
					last_ofs_in_node = dn.ofs_in_node;
				}
1539
			} else {
1540 1541
				WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO &&
					flag != F2FS_GET_BLOCK_DIO);
1542 1543
				err = __allocate_data_block(&dn,
							map->m_seg_type);
1544
				if (!err)
1545
					set_inode_flag(inode, FI_APPEND_WRITE);
1546
			}
C
Chao Yu 已提交
1547
			if (err)
C
Chao Yu 已提交
1548
				goto sync_out;
1549
			map->m_flags |= F2FS_MAP_NEW;
C
Chao Yu 已提交
1550
			blkaddr = dn.data_blkaddr;
C
Chao Yu 已提交
1551
		} else {
C
Chao Yu 已提交
1552 1553 1554 1555
			if (flag == F2FS_GET_BLOCK_BMAP) {
				map->m_pblk = 0;
				goto sync_out;
			}
1556 1557
			if (flag == F2FS_GET_BLOCK_PRECACHE)
				goto sync_out;
1558 1559 1560 1561
			if (flag == F2FS_GET_BLOCK_FIEMAP &&
						blkaddr == NULL_ADDR) {
				if (map->m_next_pgofs)
					*map->m_next_pgofs = pgofs + 1;
C
Chao Yu 已提交
1562
				goto sync_out;
1563
			}
1564 1565 1566 1567
			if (flag != F2FS_GET_BLOCK_FIEMAP) {
				/* for defragment case */
				if (map->m_next_pgofs)
					*map->m_next_pgofs = pgofs + 1;
C
Chao Yu 已提交
1568
				goto sync_out;
1569
			}
C
Chao Yu 已提交
1570 1571
		}
	}
1572

1573 1574 1575
	if (flag == F2FS_GET_BLOCK_PRE_AIO)
		goto skip;

C
Chao Yu 已提交
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585
	if (map->m_len == 0) {
		/* preallocated unwritten block should be mapped for fiemap. */
		if (blkaddr == NEW_ADDR)
			map->m_flags |= F2FS_MAP_UNWRITTEN;
		map->m_flags |= F2FS_MAP_MAPPED;

		map->m_pblk = blkaddr;
		map->m_len = 1;
	} else if ((map->m_pblk != NEW_ADDR &&
			blkaddr == (map->m_pblk + ofs)) ||
1586
			(map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1587
			flag == F2FS_GET_BLOCK_PRE_DIO) {
C
Chao Yu 已提交
1588 1589 1590 1591 1592
		ofs++;
		map->m_len++;
	} else {
		goto sync_out;
	}
1593

1594
skip:
1595 1596 1597
	dn.ofs_in_node++;
	pgofs++;

1598 1599 1600
	/* preallocate blocks in batch for one dnode page */
	if (flag == F2FS_GET_BLOCK_PRE_AIO &&
			(pgofs == end || dn.ofs_in_node == end_offset)) {
1601

1602
		dn.ofs_in_node = ofs_in_node;
C
Chao Yu 已提交
1603
		err = f2fs_reserve_new_blocks(&dn, prealloc);
1604 1605
		if (err)
			goto sync_out;
1606

1607 1608 1609 1610
		map->m_len += dn.ofs_in_node - ofs_in_node;
		if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
			err = -ENOSPC;
			goto sync_out;
1611
		}
1612 1613 1614 1615 1616 1617 1618 1619
		dn.ofs_in_node = end_offset;
	}

	if (pgofs >= end)
		goto sync_out;
	else if (dn.ofs_in_node < end_offset)
		goto next_block;

1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
	if (flag == F2FS_GET_BLOCK_PRECACHE) {
		if (map->m_flags & F2FS_MAP_MAPPED) {
			unsigned int ofs = start_pgofs - map->m_lblk;

			f2fs_update_extent_cache_range(&dn,
				start_pgofs, map->m_pblk + ofs,
				map->m_len - ofs);
		}
	}

1630 1631
	f2fs_put_dnode(&dn);

C
Chao Yu 已提交
1632
	if (map->m_may_create) {
1633
		__do_map_lock(sbi, flag, false);
1634
		f2fs_balance_fs(sbi, dn.node_changed);
1635
	}
1636
	goto next_dnode;
1637

1638
sync_out:
1639 1640 1641 1642 1643 1644

	/* for hardware encryption, but to avoid potential issue in future */
	if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED)
		f2fs_wait_on_block_writeback_range(inode,
						map->m_pblk, map->m_len);

1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655
	if (flag == F2FS_GET_BLOCK_PRECACHE) {
		if (map->m_flags & F2FS_MAP_MAPPED) {
			unsigned int ofs = start_pgofs - map->m_lblk;

			f2fs_update_extent_cache_range(&dn,
				start_pgofs, map->m_pblk + ofs,
				map->m_len - ofs);
		}
		if (map->m_next_extent)
			*map->m_next_extent = pgofs + 1;
	}
1656
	f2fs_put_dnode(&dn);
1657
unlock_out:
C
Chao Yu 已提交
1658
	if (map->m_may_create) {
1659
		__do_map_lock(sbi, flag, false);
1660
		f2fs_balance_fs(sbi, dn.node_changed);
1661
	}
1662
out:
J
Jaegeuk Kim 已提交
1663
	trace_f2fs_map_blocks(inode, map, err);
1664
	return err;
1665 1666
}

H
Hyunchul Lee 已提交
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
{
	struct f2fs_map_blocks map;
	block_t last_lblk;
	int err;

	if (pos + len > i_size_read(inode))
		return false;

	map.m_lblk = F2FS_BYTES_TO_BLK(pos);
	map.m_next_pgofs = NULL;
	map.m_next_extent = NULL;
	map.m_seg_type = NO_CHECK_TYPE;
1680
	map.m_may_create = false;
H
Hyunchul Lee 已提交
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
	last_lblk = F2FS_BLK_ALIGN(pos + len);

	while (map.m_lblk < last_lblk) {
		map.m_len = last_lblk - map.m_lblk;
		err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
		if (err || map.m_len == 0)
			return false;
		map.m_lblk += map.m_len;
	}
	return true;
}

J
Jaegeuk Kim 已提交
1693
static int __get_data_block(struct inode *inode, sector_t iblock,
1694
			struct buffer_head *bh, int create, int flag,
C
Chao Yu 已提交
1695
			pgoff_t *next_pgofs, int seg_type, bool may_write)
J
Jaegeuk Kim 已提交
1696 1697
{
	struct f2fs_map_blocks map;
1698
	int err;
J
Jaegeuk Kim 已提交
1699 1700 1701

	map.m_lblk = iblock;
	map.m_len = bh->b_size >> inode->i_blkbits;
1702
	map.m_next_pgofs = next_pgofs;
1703
	map.m_next_extent = NULL;
1704
	map.m_seg_type = seg_type;
C
Chao Yu 已提交
1705
	map.m_may_create = may_write;
J
Jaegeuk Kim 已提交
1706

1707 1708
	err = f2fs_map_blocks(inode, &map, create, flag);
	if (!err) {
J
Jaegeuk Kim 已提交
1709 1710
		map_bh(bh, inode->i_sb, map.m_pblk);
		bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1711
		bh->b_size = (u64)map.m_len << inode->i_blkbits;
J
Jaegeuk Kim 已提交
1712
	}
1713
	return err;
J
Jaegeuk Kim 已提交
1714 1715
}

1716
static int get_data_block(struct inode *inode, sector_t iblock,
1717 1718
			struct buffer_head *bh_result, int create, int flag,
			pgoff_t *next_pgofs)
C
Chao Yu 已提交
1719
{
1720
	return __get_data_block(inode, iblock, bh_result, create,
1721
							flag, next_pgofs,
C
Chao Yu 已提交
1722 1723 1724 1725 1726 1727 1728 1729 1730
							NO_CHECK_TYPE, create);
}

static int get_data_block_dio_write(struct inode *inode, sector_t iblock,
			struct buffer_head *bh_result, int create)
{
	return __get_data_block(inode, iblock, bh_result, create,
				F2FS_GET_BLOCK_DIO, NULL,
				f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1731
				IS_SWAPFILE(inode) ? false : true);
C
Chao Yu 已提交
1732 1733 1734
}

static int get_data_block_dio(struct inode *inode, sector_t iblock,
1735 1736
			struct buffer_head *bh_result, int create)
{
C
Chao Yu 已提交
1737
	return __get_data_block(inode, iblock, bh_result, create,
C
Chao Yu 已提交
1738 1739 1740
				F2FS_GET_BLOCK_DIO, NULL,
				f2fs_rw_hint_to_seg_type(inode->i_write_hint),
				false);
1741 1742
}

C
Chao Yu 已提交
1743
static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1744 1745
			struct buffer_head *bh_result, int create)
{
1746
	/* Block number less than F2FS MAX BLOCKS */
C
Chao Yu 已提交
1747
	if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1748 1749
		return -EFBIG;

C
Chao Yu 已提交
1750
	return __get_data_block(inode, iblock, bh_result, create,
1751
						F2FS_GET_BLOCK_BMAP, NULL,
C
Chao Yu 已提交
1752
						NO_CHECK_TYPE, create);
1753 1754
}

1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
{
	return (offset >> inode->i_blkbits);
}

static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
{
	return (blk << inode->i_blkbits);
}

C
Chao Yu 已提交
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
static int f2fs_xattr_fiemap(struct inode *inode,
				struct fiemap_extent_info *fieinfo)
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct page *page;
	struct node_info ni;
	__u64 phys = 0, len;
	__u32 flags;
	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
	int err = 0;

	if (f2fs_has_inline_xattr(inode)) {
		int offset;

		page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
						inode->i_ino, false);
		if (!page)
			return -ENOMEM;

1784 1785 1786 1787 1788
		err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
		if (err) {
			f2fs_put_page(page, 1);
			return err;
		}
C
Chao Yu 已提交
1789 1790 1791 1792

		phys = (__u64)blk_to_logical(inode, ni.blk_addr);
		offset = offsetof(struct f2fs_inode, i_addr) +
					sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1793
					get_inline_xattr_addrs(inode));
C
Chao Yu 已提交
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814

		phys += offset;
		len = inline_xattr_size(inode);

		f2fs_put_page(page, 1);

		flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;

		if (!xnid)
			flags |= FIEMAP_EXTENT_LAST;

		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
		if (err || err == 1)
			return err;
	}

	if (xnid) {
		page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
		if (!page)
			return -ENOMEM;

1815 1816 1817 1818 1819
		err = f2fs_get_node_info(sbi, xnid, &ni);
		if (err) {
			f2fs_put_page(page, 1);
			return err;
		}
C
Chao Yu 已提交
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834

		phys = (__u64)blk_to_logical(inode, ni.blk_addr);
		len = inode->i_sb->s_blocksize;

		f2fs_put_page(page, 1);

		flags = FIEMAP_EXTENT_LAST;
	}

	if (phys)
		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);

	return (err < 0 ? err : 0);
}

1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
static loff_t max_inode_blocks(struct inode *inode)
{
	loff_t result = ADDRS_PER_INODE(inode);
	loff_t leaf_count = ADDRS_PER_BLOCK(inode);

	/* two direct node blocks */
	result += (leaf_count * 2);

	/* two indirect node blocks */
	leaf_count *= NIDS_PER_BLOCK;
	result += (leaf_count * 2);

	/* one double indirect node block */
	leaf_count *= NIDS_PER_BLOCK;
	result += leaf_count;

	return result;
}

J
Jaegeuk Kim 已提交
1854 1855 1856
int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
		u64 start, u64 len)
{
1857 1858
	struct buffer_head map_bh;
	sector_t start_blk, last_blk;
1859
	pgoff_t next_pgofs;
1860 1861 1862
	u64 logical = 0, phys = 0, size = 0;
	u32 flags = 0;
	int ret = 0;
1863 1864
	bool compr_cluster = false;
	unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
1865

1866 1867 1868 1869 1870 1871
	if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
		ret = f2fs_precache_extents(inode);
		if (ret)
			return ret;
	}

C
Chao Yu 已提交
1872
	ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
1873 1874 1875
	if (ret)
		return ret;

1876 1877
	inode_lock(inode);

C
Chao Yu 已提交
1878 1879 1880 1881 1882
	if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
		ret = f2fs_xattr_fiemap(inode, fieinfo);
		goto out;
	}

1883
	if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
J
Jaegeuk Kim 已提交
1884 1885
		ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
		if (ret != -EAGAIN)
1886
			goto out;
J
Jaegeuk Kim 已提交
1887 1888
	}

1889 1890 1891 1892 1893
	if (logical_to_blk(inode, len) == 0)
		len = blk_to_logical(inode, 1);

	start_blk = logical_to_blk(inode, start);
	last_blk = logical_to_blk(inode, start + len - 1);
1894

1895 1896 1897 1898
next:
	memset(&map_bh, 0, sizeof(struct buffer_head));
	map_bh.b_size = len;

1899 1900 1901
	if (compr_cluster)
		map_bh.b_size = blk_to_logical(inode, cluster_size - 1);

C
Chao Yu 已提交
1902
	ret = get_data_block(inode, start_blk, &map_bh, 0,
1903
					F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1904 1905 1906 1907 1908
	if (ret)
		goto out;

	/* HOLE */
	if (!buffer_mapped(&map_bh)) {
1909
		start_blk = next_pgofs;
1910 1911

		if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1912
						max_inode_blocks(inode)))
1913
			goto prep_next;
1914

1915 1916
		flags |= FIEMAP_EXTENT_LAST;
	}
1917

1918
	if (size) {
1919
		if (IS_ENCRYPTED(inode))
1920 1921
			flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;

1922 1923
		ret = fiemap_fill_next_extent(fieinfo, logical,
				phys, size, flags);
1924 1925 1926
		if (ret)
			goto out;
		size = 0;
1927
	}
1928

1929
	if (start_blk > last_blk)
1930
		goto out;
1931

1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
	if (compr_cluster) {
		compr_cluster = false;


		logical = blk_to_logical(inode, start_blk - 1);
		phys = blk_to_logical(inode, map_bh.b_blocknr);
		size = blk_to_logical(inode, cluster_size);

		flags |= FIEMAP_EXTENT_ENCODED;

		start_blk += cluster_size - 1;

		if (start_blk > last_blk)
			goto out;

		goto prep_next;
	}

	if (map_bh.b_blocknr == COMPRESS_ADDR) {
		compr_cluster = true;
		start_blk++;
		goto prep_next;
	}

1956 1957 1958 1959 1960 1961
	logical = blk_to_logical(inode, start_blk);
	phys = blk_to_logical(inode, map_bh.b_blocknr);
	size = map_bh.b_size;
	flags = 0;
	if (buffer_unwritten(&map_bh))
		flags = FIEMAP_EXTENT_UNWRITTEN;
1962

1963
	start_blk += logical_to_blk(inode, size);
1964

1965
prep_next:
1966 1967 1968 1969 1970 1971 1972 1973 1974
	cond_resched();
	if (fatal_signal_pending(current))
		ret = -EINTR;
	else
		goto next;
out:
	if (ret == 1)
		ret = 0;

A
Al Viro 已提交
1975
	inode_unlock(inode);
1976
	return ret;
J
Jaegeuk Kim 已提交
1977 1978
}

E
Eric Biggers 已提交
1979 1980 1981 1982 1983 1984 1985 1986 1987
static inline loff_t f2fs_readpage_limit(struct inode *inode)
{
	if (IS_ENABLED(CONFIG_FS_VERITY) &&
	    (IS_VERITY(inode) || f2fs_verity_in_progress(inode)))
		return inode->i_sb->s_maxbytes;

	return i_size_read(inode);
}

1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
static int f2fs_read_single_page(struct inode *inode, struct page *page,
					unsigned nr_pages,
					struct f2fs_map_blocks *map,
					struct bio **bio_ret,
					sector_t *last_block_in_bio,
					bool is_readahead)
{
	struct bio *bio = *bio_ret;
	const unsigned blkbits = inode->i_blkbits;
	const unsigned blocksize = 1 << blkbits;
	sector_t block_in_file;
	sector_t last_block;
	sector_t last_block_in_file;
	sector_t block_nr;
	int ret = 0;

J
Jaegeuk Kim 已提交
2004
	block_in_file = (sector_t)page_index(page);
2005
	last_block = block_in_file + nr_pages;
E
Eric Biggers 已提交
2006
	last_block_in_file = (f2fs_readpage_limit(inode) + blocksize - 1) >>
2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
							blkbits;
	if (last_block > last_block_in_file)
		last_block = last_block_in_file;

	/* just zeroing out page which is beyond EOF */
	if (block_in_file >= last_block)
		goto zero_out;
	/*
	 * Map blocks using the previous result first.
	 */
	if ((map->m_flags & F2FS_MAP_MAPPED) &&
			block_in_file > map->m_lblk &&
			block_in_file < (map->m_lblk + map->m_len))
		goto got_it;

	/*
	 * Then do more f2fs_map_blocks() calls until we are
	 * done with this page.
	 */
	map->m_lblk = block_in_file;
	map->m_len = last_block - block_in_file;

	ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT);
	if (ret)
		goto out;
got_it:
	if ((map->m_flags & F2FS_MAP_MAPPED)) {
		block_nr = map->m_pblk + block_in_file - map->m_lblk;
		SetPageMappedToDisk(page);

J
Jaegeuk Kim 已提交
2037 2038
		if (!PageUptodate(page) && (!PageSwapCache(page) &&
					!cleancache_get_page(page))) {
2039 2040 2041 2042 2043
			SetPageUptodate(page);
			goto confused;
		}

		if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
C
Chao Yu 已提交
2044
						DATA_GENERIC_ENHANCE_READ)) {
2045
			ret = -EFSCORRUPTED;
2046 2047 2048 2049 2050
			goto out;
		}
	} else {
zero_out:
		zero_user_segment(page, 0, PAGE_SIZE);
E
Eric Biggers 已提交
2051 2052 2053 2054 2055
		if (f2fs_need_verity(inode, page->index) &&
		    !fsverity_verify_page(page)) {
			ret = -EIO;
			goto out;
		}
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
		if (!PageUptodate(page))
			SetPageUptodate(page);
		unlock_page(page);
		goto out;
	}

	/*
	 * This page will go to BIO.  Do we need to send this
	 * BIO off first?
	 */
2066 2067
	if (bio && !page_is_mergeable(F2FS_I_SB(inode), bio,
				*last_block_in_bio, block_nr)) {
2068 2069 2070 2071 2072 2073
submit_and_realloc:
		__submit_bio(F2FS_I_SB(inode), bio, DATA);
		bio = NULL;
	}
	if (bio == NULL) {
		bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2074 2075
				is_readahead ? REQ_RAHEAD : 0, page->index,
				false);
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
		if (IS_ERR(bio)) {
			ret = PTR_ERR(bio);
			bio = NULL;
			goto out;
		}
	}

	/*
	 * If the page is under writeback, we need to wait for
	 * its completion to see the correct decrypted data.
	 */
	f2fs_wait_on_block_writeback(inode, block_nr);

	if (bio_add_page(bio, page, blocksize, 0) < blocksize)
		goto submit_and_realloc;

	inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
C
Chao Yu 已提交
2093
	f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE);
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
	ClearPageError(page);
	*last_block_in_bio = block_nr;
	goto out;
confused:
	if (bio) {
		__submit_bio(F2FS_I_SB(inode), bio, DATA);
		bio = NULL;
	}
	unlock_page(page);
out:
	*bio_ret = bio;
	return ret;
}

C
Chao Yu 已提交
2108 2109 2110
#ifdef CONFIG_F2FS_FS_COMPRESSION
int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
				unsigned nr_pages, sector_t *last_block_in_bio,
2111
				bool is_readahead, bool for_write)
C
Chao Yu 已提交
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126
{
	struct dnode_of_data dn;
	struct inode *inode = cc->inode;
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct bio *bio = *bio_ret;
	unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
	sector_t last_block_in_file;
	const unsigned blkbits = inode->i_blkbits;
	const unsigned blocksize = 1 << blkbits;
	struct decompress_io_ctx *dic = NULL;
	int i;
	int ret = 0;

	f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));

2127 2128
	last_block_in_file = (f2fs_readpage_limit(inode) +
					blocksize - 1) >> blkbits;
C
Chao Yu 已提交
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163

	/* get rid of pages beyond EOF */
	for (i = 0; i < cc->cluster_size; i++) {
		struct page *page = cc->rpages[i];

		if (!page)
			continue;
		if ((sector_t)page->index >= last_block_in_file) {
			zero_user_segment(page, 0, PAGE_SIZE);
			if (!PageUptodate(page))
				SetPageUptodate(page);
		} else if (!PageUptodate(page)) {
			continue;
		}
		unlock_page(page);
		cc->rpages[i] = NULL;
		cc->nr_rpages--;
	}

	/* we are done since all pages are beyond EOF */
	if (f2fs_cluster_is_empty(cc))
		goto out;

	set_new_dnode(&dn, inode, NULL, NULL, 0);
	ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
	if (ret)
		goto out;

	/* cluster was overwritten as normal cluster */
	if (dn.data_blkaddr != COMPRESS_ADDR)
		goto out;

	for (i = 1; i < cc->cluster_size; i++) {
		block_t blkaddr;

2164
		blkaddr = data_blkaddr(dn.inode, dn.node_page,
C
Chao Yu 已提交
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
						dn.ofs_in_node + i);

		if (!__is_valid_data_blkaddr(blkaddr))
			break;

		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
			ret = -EFAULT;
			goto out_put_dnode;
		}
		cc->nr_cpages++;
	}

	/* nothing to decompress */
	if (cc->nr_cpages == 0) {
		ret = 0;
		goto out_put_dnode;
	}

	dic = f2fs_alloc_dic(cc);
	if (IS_ERR(dic)) {
		ret = PTR_ERR(dic);
		goto out_put_dnode;
	}

	for (i = 0; i < dic->nr_cpages; i++) {
		struct page *page = dic->cpages[i];
		block_t blkaddr;

2193
		blkaddr = data_blkaddr(dn.inode, dn.node_page,
C
Chao Yu 已提交
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
						dn.ofs_in_node + i + 1);

		if (bio && !page_is_mergeable(sbi, bio,
					*last_block_in_bio, blkaddr)) {
submit_and_realloc:
			__submit_bio(sbi, bio, DATA);
			bio = NULL;
		}

		if (!bio) {
			bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
					is_readahead ? REQ_RAHEAD : 0,
2206
					page->index, for_write);
C
Chao Yu 已提交
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
			if (IS_ERR(bio)) {
				ret = PTR_ERR(bio);
				bio = NULL;
				dic->failed = true;
				if (refcount_sub_and_test(dic->nr_cpages - i,
							&dic->ref))
					f2fs_decompress_end_io(dic->rpages,
							cc->cluster_size, true,
							false);
				f2fs_free_dic(dic);
				f2fs_put_dnode(&dn);
				*bio_ret = bio;
				return ret;
			}
		}

		f2fs_wait_on_block_writeback(inode, blkaddr);

		if (bio_add_page(bio, page, blocksize, 0) < blocksize)
			goto submit_and_realloc;

		inc_page_count(sbi, F2FS_RD_DATA);
C
Chao Yu 已提交
2229
		f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
C
Chao Yu 已提交
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
		ClearPageError(page);
		*last_block_in_bio = blkaddr;
	}

	f2fs_put_dnode(&dn);

	*bio_ret = bio;
	return 0;

out_put_dnode:
	f2fs_put_dnode(&dn);
out:
	f2fs_decompress_end_io(cc->rpages, cc->cluster_size, true, false);
	*bio_ret = bio;
	return ret;
}
#endif

J
Jaegeuk Kim 已提交
2248 2249 2250
/*
 * This function was originally taken from fs/mpage.c, and customized for f2fs.
 * Major change was from block_size == page_size in f2fs by default.
2251 2252 2253 2254 2255
 *
 * Note that the aops->readpages() function is ONLY used for read-ahead. If
 * this function ever deviates from doing just read-ahead, it should either
 * use ->readpage() or do the necessary surgery to decouple ->readpages()
 * from read-ahead.
J
Jaegeuk Kim 已提交
2256
 */
C
Chao Yu 已提交
2257
int f2fs_mpage_readpages(struct address_space *mapping,
J
Jaegeuk Kim 已提交
2258
			struct list_head *pages, struct page *page,
2259
			unsigned nr_pages, bool is_readahead)
J
Jaegeuk Kim 已提交
2260 2261 2262 2263 2264
{
	struct bio *bio = NULL;
	sector_t last_block_in_bio = 0;
	struct inode *inode = mapping->host;
	struct f2fs_map_blocks map;
C
Chao Yu 已提交
2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
#ifdef CONFIG_F2FS_FS_COMPRESSION
	struct compress_ctx cc = {
		.inode = inode,
		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
		.cluster_size = F2FS_I(inode)->i_cluster_size,
		.cluster_idx = NULL_CLUSTER,
		.rpages = NULL,
		.cpages = NULL,
		.nr_rpages = 0,
		.nr_cpages = 0,
	};
#endif
	unsigned max_nr_pages = nr_pages;
2278
	int ret = 0;
J
Jaegeuk Kim 已提交
2279 2280 2281 2282 2283

	map.m_pblk = 0;
	map.m_lblk = 0;
	map.m_len = 0;
	map.m_flags = 0;
2284
	map.m_next_pgofs = NULL;
2285
	map.m_next_extent = NULL;
2286
	map.m_seg_type = NO_CHECK_TYPE;
C
Chao Yu 已提交
2287
	map.m_may_create = false;
J
Jaegeuk Kim 已提交
2288

L
LiFan 已提交
2289
	for (; nr_pages; nr_pages--) {
J
Jaegeuk Kim 已提交
2290
		if (pages) {
2291
			page = list_last_entry(pages, struct page, lru);
2292 2293

			prefetchw(&page->flags);
J
Jaegeuk Kim 已提交
2294 2295
			list_del(&page->lru);
			if (add_to_page_cache_lru(page, mapping,
J
Jaegeuk Kim 已提交
2296
						  page_index(page),
2297
						  readahead_gfp_mask(mapping)))
J
Jaegeuk Kim 已提交
2298 2299 2300
				goto next_page;
		}

C
Chao Yu 已提交
2301 2302 2303 2304 2305 2306 2307
#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (f2fs_compressed_file(inode)) {
			/* there are remained comressed pages, submit them */
			if (!f2fs_cluster_can_merge_page(&cc, page->index)) {
				ret = f2fs_read_multi_pages(&cc, &bio,
							max_nr_pages,
							&last_block_in_bio,
2308
							is_readahead, false);
C
Chao Yu 已提交
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
				f2fs_destroy_compress_ctx(&cc);
				if (ret)
					goto set_error_page;
			}
			ret = f2fs_is_compressed_cluster(inode, page->index);
			if (ret < 0)
				goto set_error_page;
			else if (!ret)
				goto read_single_page;

			ret = f2fs_init_compress_ctx(&cc);
			if (ret)
				goto set_error_page;

			f2fs_compress_ctx_add_page(&cc, page);

			goto next_page;
		}
read_single_page:
#endif

		ret = f2fs_read_single_page(inode, page, max_nr_pages, &map,
					&bio, &last_block_in_bio, is_readahead);
2332
		if (ret) {
C
Chao Yu 已提交
2333 2334 2335
#ifdef CONFIG_F2FS_FS_COMPRESSION
set_error_page:
#endif
2336
			SetPageError(page);
2337
			zero_user_segment(page, 0, PAGE_SIZE);
J
Jaegeuk Kim 已提交
2338 2339 2340 2341
			unlock_page(page);
		}
next_page:
		if (pages)
2342
			put_page(page);
C
Chao Yu 已提交
2343 2344 2345 2346 2347 2348 2349 2350

#ifdef CONFIG_F2FS_FS_COMPRESSION
		if (f2fs_compressed_file(inode)) {
			/* last page */
			if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
				ret = f2fs_read_multi_pages(&cc, &bio,
							max_nr_pages,
							&last_block_in_bio,
2351
							is_readahead, false);
C
Chao Yu 已提交
2352 2353 2354 2355
				f2fs_destroy_compress_ctx(&cc);
			}
		}
#endif
J
Jaegeuk Kim 已提交
2356 2357 2358
	}
	BUG_ON(pages && !list_empty(pages));
	if (bio)
2359
		__submit_bio(F2FS_I_SB(inode), bio, DATA);
2360
	return pages ? 0 : ret;
J
Jaegeuk Kim 已提交
2361 2362
}

2363 2364
static int f2fs_read_data_page(struct file *file, struct page *page)
{
J
Jaegeuk Kim 已提交
2365
	struct inode *inode = page_file_mapping(page)->host;
2366
	int ret = -EAGAIN;
H
Huajun Li 已提交
2367

2368 2369
	trace_f2fs_readpage(page, DATA);

C
Chao Yu 已提交
2370 2371 2372 2373 2374
	if (!f2fs_is_compress_backend_ready(inode)) {
		unlock_page(page);
		return -EOPNOTSUPP;
	}

A
arter97 已提交
2375
	/* If the file has inline data, try to read it directly */
H
Huajun Li 已提交
2376 2377
	if (f2fs_has_inline_data(inode))
		ret = f2fs_read_inline_data(inode, page);
2378
	if (ret == -EAGAIN)
J
Jaegeuk Kim 已提交
2379 2380
		ret = f2fs_mpage_readpages(page_file_mapping(page),
						NULL, page, 1, false);
H
Huajun Li 已提交
2381
	return ret;
2382 2383 2384 2385 2386 2387
}

static int f2fs_read_data_pages(struct file *file,
			struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
2388
	struct inode *inode = mapping->host;
2389
	struct page *page = list_last_entry(pages, struct page, lru);
2390 2391

	trace_f2fs_readpages(inode, page, nr_pages);
H
Huajun Li 已提交
2392

C
Chao Yu 已提交
2393 2394 2395
	if (!f2fs_is_compress_backend_ready(inode))
		return 0;

H
Huajun Li 已提交
2396 2397 2398 2399
	/* If the file has inline data, skip readpages */
	if (f2fs_has_inline_data(inode))
		return 0;

2400
	return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true);
2401 2402
}

C
Chao Yu 已提交
2403
int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
2404 2405
{
	struct inode *inode = fio->page->mapping->host;
C
Chao Yu 已提交
2406
	struct page *mpage, *page;
2407 2408
	gfp_t gfp_flags = GFP_NOFS;

2409
	if (!f2fs_encrypted_file(inode))
2410 2411
		return 0;

C
Chao Yu 已提交
2412 2413
	page = fio->compressed_page ? fio->compressed_page : fio->page;

2414
	/* wait for GCed page writeback via META_MAPPING */
2415
	f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
2416 2417

retry_encrypt:
C
Chao Yu 已提交
2418 2419
	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,
					PAGE_SIZE, 0, gfp_flags);
2420 2421 2422 2423
	if (IS_ERR(fio->encrypted_page)) {
		/* flush pending IOs and wait for a while in the ENOMEM case */
		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
			f2fs_flush_merged_writes(fio->sbi);
C
Chao Yu 已提交
2424
			congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
2425 2426 2427 2428 2429
			gfp_flags |= __GFP_NOFAIL;
			goto retry_encrypt;
		}
		return PTR_ERR(fio->encrypted_page);
	}
2430

2431 2432 2433 2434 2435 2436
	mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
	if (mpage) {
		if (PageUptodate(mpage))
			memcpy(page_address(mpage),
				page_address(fio->encrypted_page), PAGE_SIZE);
		f2fs_put_page(mpage, 1);
2437
	}
2438
	return 0;
2439 2440
}

C
Chao Yu 已提交
2441 2442
static inline bool check_inplace_update_policy(struct inode *inode,
				struct f2fs_io_info *fio)
2443
{
C
Chao Yu 已提交
2444 2445
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	unsigned int policy = SM_I(sbi)->ipu_policy;
2446

C
Chao Yu 已提交
2447 2448
	if (policy & (0x1 << F2FS_IPU_FORCE))
		return true;
C
Chao Yu 已提交
2449
	if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
C
Chao Yu 已提交
2450 2451 2452 2453
		return true;
	if (policy & (0x1 << F2FS_IPU_UTIL) &&
			utilization(sbi) > SM_I(sbi)->min_ipu_util)
		return true;
C
Chao Yu 已提交
2454
	if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
C
Chao Yu 已提交
2455 2456 2457 2458 2459 2460 2461 2462 2463
			utilization(sbi) > SM_I(sbi)->min_ipu_util)
		return true;

	/*
	 * IPU for rewrite async pages
	 */
	if (policy & (0x1 << F2FS_IPU_ASYNC) &&
			fio && fio->op == REQ_OP_WRITE &&
			!(fio->op_flags & REQ_SYNC) &&
2464
			!IS_ENCRYPTED(inode))
C
Chao Yu 已提交
2465 2466 2467 2468 2469 2470 2471
		return true;

	/* this is only set during fdatasync */
	if (policy & (0x1 << F2FS_IPU_FSYNC) &&
			is_inode_flag_set(inode, FI_NEED_IPU))
		return true;

D
Daniel Rosenberg 已提交
2472 2473 2474 2475
	if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
			!f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
		return true;

C
Chao Yu 已提交
2476 2477 2478
	return false;
}

C
Chao Yu 已提交
2479
bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
C
Chao Yu 已提交
2480
{
2481 2482
	if (f2fs_is_pinned_file(inode))
		return true;
C
Chao Yu 已提交
2483 2484 2485 2486 2487 2488 2489 2490

	/* if this is cold file, we should overwrite to avoid fragmentation */
	if (file_is_cold(inode))
		return true;

	return check_inplace_update_policy(inode, fio);
}

C
Chao Yu 已提交
2491
bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
C
Chao Yu 已提交
2492 2493 2494
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);

2495
	if (f2fs_lfs_mode(sbi))
C
Chao Yu 已提交
2496 2497 2498
		return true;
	if (S_ISDIR(inode->i_mode))
		return true;
2499 2500
	if (IS_NOQUOTA(inode))
		return true;
C
Chao Yu 已提交
2501 2502 2503 2504 2505 2506 2507
	if (f2fs_is_atomic_file(inode))
		return true;
	if (fio) {
		if (is_cold_data(fio->page))
			return true;
		if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
			return true;
D
Daniel Rosenberg 已提交
2508 2509 2510
		if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
			f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
			return true;
C
Chao Yu 已提交
2511 2512 2513 2514
	}
	return false;
}

2515 2516 2517 2518
static inline bool need_inplace_update(struct f2fs_io_info *fio)
{
	struct inode *inode = fio->page->mapping->host;

C
Chao Yu 已提交
2519
	if (f2fs_should_update_outplace(inode, fio))
2520 2521
		return false;

C
Chao Yu 已提交
2522
	return f2fs_should_update_inplace(inode, fio);
2523 2524
}

C
Chao Yu 已提交
2525
int f2fs_do_write_data_page(struct f2fs_io_info *fio)
2526
{
2527
	struct page *page = fio->page;
2528 2529
	struct inode *inode = page->mapping->host;
	struct dnode_of_data dn;
2530
	struct extent_info ei = {0,0,0};
2531
	struct node_info ni;
2532
	bool ipu_force = false;
2533 2534 2535
	int err = 0;

	set_new_dnode(&dn, inode, NULL, NULL, 0);
2536 2537 2538
	if (need_inplace_update(fio) &&
			f2fs_lookup_extent_cache(inode, page->index, &ei)) {
		fio->old_blkaddr = ei.blk + page->index - ei.fofs;
2539

2540
		if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
C
Chao Yu 已提交
2541
						DATA_GENERIC_ENHANCE))
2542
			return -EFSCORRUPTED;
2543 2544 2545 2546

		ipu_force = true;
		fio->need_lock = LOCK_DONE;
		goto got_it;
2547
	}
2548

2549 2550 2551
	/* Deadlock due to between page->lock and f2fs_lock_op */
	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
		return -EAGAIN;
2552

C
Chao Yu 已提交
2553
	err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
2554
	if (err)
2555
		goto out;
2556

2557
	fio->old_blkaddr = dn.data_blkaddr;
2558 2559

	/* This page is already truncated */
2560
	if (fio->old_blkaddr == NULL_ADDR) {
2561
		ClearPageUptodate(page);
C
Chao Yu 已提交
2562
		clear_cold_data(page);
2563
		goto out_writepage;
2564
	}
2565
got_it:
2566 2567
	if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
		!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
C
Chao Yu 已提交
2568
						DATA_GENERIC_ENHANCE)) {
2569
		err = -EFSCORRUPTED;
2570 2571
		goto out_writepage;
	}
2572 2573 2574 2575
	/*
	 * If current allocation needs SSR,
	 * it had better in-place writes for updated data.
	 */
C
Chao Yu 已提交
2576 2577
	if (ipu_force ||
		(__is_valid_data_blkaddr(fio->old_blkaddr) &&
C
Chao Yu 已提交
2578
					need_inplace_update(fio))) {
C
Chao Yu 已提交
2579
		err = f2fs_encrypt_one_page(fio);
2580 2581 2582 2583
		if (err)
			goto out_writepage;

		set_page_writeback(page);
J
Jaegeuk Kim 已提交
2584
		ClearPageError(page);
2585
		f2fs_put_dnode(&dn);
2586
		if (fio->need_lock == LOCK_REQ)
2587
			f2fs_unlock_op(fio->sbi);
C
Chao Yu 已提交
2588
		err = f2fs_inplace_write_data(fio);
C
Chao Yu 已提交
2589 2590
		if (err) {
			if (f2fs_encrypted_file(inode))
2591
				fscrypt_finalize_bounce_page(&fio->encrypted_page);
C
Chao Yu 已提交
2592 2593
			if (PageWriteback(page))
				end_page_writeback(page);
2594 2595
		} else {
			set_inode_flag(inode, FI_UPDATE_WRITE);
C
Chao Yu 已提交
2596
		}
2597
		trace_f2fs_do_write_data_page(fio->page, IPU);
2598
		return err;
2599
	}
2600

2601 2602 2603 2604 2605 2606 2607 2608
	if (fio->need_lock == LOCK_RETRY) {
		if (!f2fs_trylock_op(fio->sbi)) {
			err = -EAGAIN;
			goto out_writepage;
		}
		fio->need_lock = LOCK_REQ;
	}

2609 2610 2611 2612 2613 2614
	err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
	if (err)
		goto out_writepage;

	fio->version = ni.version;

C
Chao Yu 已提交
2615
	err = f2fs_encrypt_one_page(fio);
2616 2617 2618 2619
	if (err)
		goto out_writepage;

	set_page_writeback(page);
J
Jaegeuk Kim 已提交
2620
	ClearPageError(page);
2621

C
Chao Yu 已提交
2622 2623 2624
	if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)
		f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);

2625
	/* LFS mode write path */
C
Chao Yu 已提交
2626
	f2fs_outplace_write_data(&dn, fio);
2627 2628 2629 2630
	trace_f2fs_do_write_data_page(page, OPU);
	set_inode_flag(inode, FI_APPEND_WRITE);
	if (page->index == 0)
		set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
2631 2632
out_writepage:
	f2fs_put_dnode(&dn);
2633
out:
2634
	if (fio->need_lock == LOCK_REQ)
2635
		f2fs_unlock_op(fio->sbi);
2636 2637 2638
	return err;
}

C
Chao Yu 已提交
2639
int f2fs_write_single_data_page(struct page *page, int *submitted,
C
Chao Yu 已提交
2640 2641
				struct bio **bio,
				sector_t *last_block,
C
Chao Yu 已提交
2642
				struct writeback_control *wbc,
C
Chao Yu 已提交
2643 2644
				enum iostat_type io_type,
				int compr_blocks)
2645 2646
{
	struct inode *inode = page->mapping->host;
2647
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2648
	loff_t i_size = i_size_read(inode);
C
Chao Yu 已提交
2649
	const pgoff_t end_index = ((unsigned long long)i_size)
2650
							>> PAGE_SHIFT;
C
Chao Yu 已提交
2651
	loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT;
H
Huajun Li 已提交
2652
	unsigned offset = 0;
2653
	bool need_balance_fs = false;
2654
	int err = 0;
J
Jaegeuk Kim 已提交
2655
	struct f2fs_io_info fio = {
2656
		.sbi = sbi,
C
Chao Yu 已提交
2657
		.ino = inode->i_ino,
J
Jaegeuk Kim 已提交
2658
		.type = DATA,
M
Mike Christie 已提交
2659
		.op = REQ_OP_WRITE,
J
Jens Axboe 已提交
2660
		.op_flags = wbc_to_write_flags(wbc),
2661
		.old_blkaddr = NULL_ADDR,
2662
		.page = page,
2663
		.encrypted_page = NULL,
2664
		.submitted = false,
C
Chao Yu 已提交
2665
		.compr_blocks = compr_blocks,
2666
		.need_lock = LOCK_RETRY,
C
Chao Yu 已提交
2667
		.io_type = io_type,
2668
		.io_wbc = wbc,
C
Chao Yu 已提交
2669 2670
		.bio = bio,
		.last_block = last_block,
J
Jaegeuk Kim 已提交
2671
	};
2672

2673 2674
	trace_f2fs_writepage(page, DATA);

2675 2676 2677
	/* we should bypass data pages to proceed the kworkder jobs */
	if (unlikely(f2fs_cp_error(sbi))) {
		mapping_set_error(page->mapping, -EIO);
2678 2679 2680 2681 2682 2683
		/*
		 * don't drop any dirty dentry pages for keeping lastest
		 * directory structure.
		 */
		if (S_ISDIR(inode->i_mode))
			goto redirty_out;
2684 2685 2686
		goto out;
	}

2687 2688 2689
	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
		goto redirty_out;

C
Chao Yu 已提交
2690 2691 2692
	if (page->index < end_index ||
			f2fs_verity_in_progress(inode) ||
			compr_blocks)
2693
		goto write;
2694 2695 2696 2697 2698

	/*
	 * If the offset is out-of-range of file size,
	 * this page does not have to be written to disk.
	 */
2699
	offset = i_size & (PAGE_SIZE - 1);
2700
	if ((page->index >= end_index + 1) || !offset)
2701
		goto out;
2702

2703
	zero_user_segment(page, offset, PAGE_SIZE);
2704
write:
2705 2706
	if (f2fs_is_drop_cache(inode))
		goto out;
2707 2708 2709
	/* we should not write 0'th page having journal header */
	if (f2fs_is_volatile_file(inode) && (!page->index ||
			(!wbc->for_reclaim &&
C
Chao Yu 已提交
2710
			f2fs_available_free_memory(sbi, BASE_CHECK))))
2711
		goto redirty_out;
2712

2713 2714
	/* Dentry/quota blocks are controlled by checkpoint */
	if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) {
2715
		fio.need_lock = LOCK_DONE;
C
Chao Yu 已提交
2716
		err = f2fs_do_write_data_page(&fio);
2717 2718
		goto done;
	}
H
Huajun Li 已提交
2719

2720
	if (!wbc->for_reclaim)
2721
		need_balance_fs = true;
2722
	else if (has_not_enough_free_secs(sbi, 0, 0))
2723
		goto redirty_out;
2724 2725
	else
		set_inode_flag(inode, FI_HOT_DATA);
2726

2727
	err = -EAGAIN;
2728
	if (f2fs_has_inline_data(inode)) {
2729
		err = f2fs_write_inline_data(inode, page);
2730 2731 2732
		if (!err)
			goto out;
	}
2733

2734
	if (err == -EAGAIN) {
C
Chao Yu 已提交
2735
		err = f2fs_do_write_data_page(&fio);
2736 2737
		if (err == -EAGAIN) {
			fio.need_lock = LOCK_REQ;
C
Chao Yu 已提交
2738
			err = f2fs_do_write_data_page(&fio);
2739 2740
		}
	}
2741

2742 2743 2744
	if (err) {
		file_set_keep_isize(inode);
	} else {
2745
		spin_lock(&F2FS_I(inode)->i_size_lock);
2746 2747
		if (F2FS_I(inode)->last_disk_size < psize)
			F2FS_I(inode)->last_disk_size = psize;
2748
		spin_unlock(&F2FS_I(inode)->i_size_lock);
2749
	}
2750

2751 2752 2753
done:
	if (err && err != -ENOENT)
		goto redirty_out;
2754

2755
out:
2756
	inode_dec_dirty_pages(inode);
C
Chao Yu 已提交
2757
	if (err) {
2758
		ClearPageUptodate(page);
C
Chao Yu 已提交
2759 2760
		clear_cold_data(page);
	}
2761 2762

	if (wbc->for_reclaim) {
2763
		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA);
2764
		clear_inode_flag(inode, FI_HOT_DATA);
C
Chao Yu 已提交
2765
		f2fs_remove_dirty_inode(inode);
2766
		submitted = NULL;
2767
	}
2768
	unlock_page(page);
2769
	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
C
Chao Yu 已提交
2770
					!F2FS_I(inode)->cp_task)
J
Jaegeuk Kim 已提交
2771
		f2fs_balance_fs(sbi, need_balance_fs);
2772

2773
	if (unlikely(f2fs_cp_error(sbi))) {
2774
		f2fs_submit_merged_write(sbi, DATA);
C
Chao Yu 已提交
2775
		f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2776 2777 2778 2779
		submitted = NULL;
	}

	if (submitted)
C
Chao Yu 已提交
2780
		*submitted = fio.submitted ? 1 : 0;
2781

2782 2783 2784
	return 0;

redirty_out:
2785
	redirty_page_for_writepage(wbc, page);
2786 2787 2788 2789 2790 2791 2792
	/*
	 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
	 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
	 * file_write_and_wait_range() will see EIO error, which is critical
	 * to return value of fsync() followed by atomic_write failure to user.
	 */
	if (!err || wbc->for_reclaim)
2793
		return AOP_WRITEPAGE_ACTIVATE;
J
Jaegeuk Kim 已提交
2794 2795
	unlock_page(page);
	return err;
2796 2797
}

2798 2799 2800
static int f2fs_write_data_page(struct page *page,
					struct writeback_control *wbc)
{
C
Chao Yu 已提交
2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817
#ifdef CONFIG_F2FS_FS_COMPRESSION
	struct inode *inode = page->mapping->host;

	if (unlikely(f2fs_cp_error(F2FS_I_SB(inode))))
		goto out;

	if (f2fs_compressed_file(inode)) {
		if (f2fs_is_compressed_cluster(inode, page->index)) {
			redirty_page_for_writepage(wbc, page);
			return AOP_WRITEPAGE_ACTIVATE;
		}
	}
out:
#endif

	return f2fs_write_single_data_page(page, NULL, NULL, NULL,
						wbc, FS_DATA_IO, 0);
2818 2819
}

C
Chao Yu 已提交
2820 2821 2822 2823 2824 2825
/*
 * This function was copied from write_cche_pages from mm/page-writeback.c.
 * The major change is making write step of cold data page separately from
 * warm/hot data page.
 */
static int f2fs_write_cache_pages(struct address_space *mapping,
C
Chao Yu 已提交
2826 2827
					struct writeback_control *wbc,
					enum iostat_type io_type)
C
Chao Yu 已提交
2828 2829
{
	int ret = 0;
C
Chao Yu 已提交
2830
	int done = 0, retry = 0;
C
Chao Yu 已提交
2831
	struct pagevec pvec;
2832
	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
C
Chao Yu 已提交
2833 2834
	struct bio *bio = NULL;
	sector_t last_block;
C
Chao Yu 已提交
2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850
#ifdef CONFIG_F2FS_FS_COMPRESSION
	struct inode *inode = mapping->host;
	struct compress_ctx cc = {
		.inode = inode,
		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
		.cluster_size = F2FS_I(inode)->i_cluster_size,
		.cluster_idx = NULL_CLUSTER,
		.rpages = NULL,
		.nr_rpages = 0,
		.cpages = NULL,
		.rbuf = NULL,
		.cbuf = NULL,
		.rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size,
		.private = NULL,
	};
#endif
C
Chao Yu 已提交
2851 2852 2853 2854 2855 2856 2857
	int nr_pages;
	pgoff_t uninitialized_var(writeback_index);
	pgoff_t index;
	pgoff_t end;		/* Inclusive */
	pgoff_t done_index;
	int cycled;
	int range_whole = 0;
M
Matthew Wilcox 已提交
2858
	xa_mark_t tag;
2859
	int nwritten = 0;
C
Chao Yu 已提交
2860 2861
	int submitted = 0;
	int i;
C
Chao Yu 已提交
2862

2863
	pagevec_init(&pvec);
2864

2865 2866 2867 2868 2869 2870
	if (get_dirty_pages(mapping->host) <=
				SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
		set_inode_flag(mapping->host, FI_HOT_DATA);
	else
		clear_inode_flag(mapping->host, FI_HOT_DATA);

C
Chao Yu 已提交
2871 2872 2873 2874 2875 2876 2877 2878 2879
	if (wbc->range_cyclic) {
		writeback_index = mapping->writeback_index; /* prev offset */
		index = writeback_index;
		if (index == 0)
			cycled = 1;
		else
			cycled = 0;
		end = -1;
	} else {
2880 2881
		index = wbc->range_start >> PAGE_SHIFT;
		end = wbc->range_end >> PAGE_SHIFT;
C
Chao Yu 已提交
2882 2883 2884 2885 2886 2887 2888 2889 2890
		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
			range_whole = 1;
		cycled = 1; /* ignore range_cyclic tests */
	}
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
		tag = PAGECACHE_TAG_TOWRITE;
	else
		tag = PAGECACHE_TAG_DIRTY;
retry:
C
Chao Yu 已提交
2891
	retry = 0;
C
Chao Yu 已提交
2892 2893 2894
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
		tag_pages_for_writeback(mapping, index, end);
	done_index = index;
C
Chao Yu 已提交
2895
	while (!done && !retry && (index <= end)) {
J
Jan Kara 已提交
2896
		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2897
				tag);
C
Chao Yu 已提交
2898 2899 2900 2901 2902
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
C
Chao Yu 已提交
2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
			bool need_readd;
readd:
			need_readd = false;
#ifdef CONFIG_F2FS_FS_COMPRESSION
			if (f2fs_compressed_file(inode)) {
				ret = f2fs_init_compress_ctx(&cc);
				if (ret) {
					done = 1;
					break;
				}

				if (!f2fs_cluster_can_merge_page(&cc,
								page->index)) {
					ret = f2fs_write_multi_pages(&cc,
						&submitted, wbc, io_type);
					if (!ret)
						need_readd = true;
					goto result;
				}
C
Chao Yu 已提交
2922

C
Chao Yu 已提交
2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
				if (unlikely(f2fs_cp_error(sbi)))
					goto lock_page;

				if (f2fs_cluster_is_empty(&cc)) {
					void *fsdata = NULL;
					struct page *pagep;
					int ret2;

					ret2 = f2fs_prepare_compress_overwrite(
							inode, &pagep,
							page->index, &fsdata);
					if (ret2 < 0) {
						ret = ret2;
						done = 1;
						break;
					} else if (ret2 &&
						!f2fs_compress_write_end(inode,
								fsdata, page->index,
								1)) {
						retry = 1;
						break;
					}
				} else {
					goto lock_page;
				}
			}
#endif
2950
			/* give a priority to WB_SYNC threads */
2951
			if (atomic_read(&sbi->wb_sync_req[DATA]) &&
2952 2953 2954 2955
					wbc->sync_mode == WB_SYNC_NONE) {
				done = 1;
				break;
			}
C
Chao Yu 已提交
2956 2957 2958
#ifdef CONFIG_F2FS_FS_COMPRESSION
lock_page:
#endif
C
Chao Yu 已提交
2959
			done_index = page->index;
2960
retry_write:
C
Chao Yu 已提交
2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974
			lock_page(page);

			if (unlikely(page->mapping != mapping)) {
continue_unlock:
				unlock_page(page);
				continue;
			}

			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

			if (PageWriteback(page)) {
C
Chao Yu 已提交
2975
				if (wbc->sync_mode != WB_SYNC_NONE)
2976
					f2fs_wait_on_page_writeback(page,
2977
							DATA, true, true);
C
Chao Yu 已提交
2978
				else
C
Chao Yu 已提交
2979 2980 2981 2982 2983 2984
					goto continue_unlock;
			}

			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;

C
Chao Yu 已提交
2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001
#ifdef CONFIG_F2FS_FS_COMPRESSION
			if (f2fs_compressed_file(inode)) {
				get_page(page);
				f2fs_compress_ctx_add_page(&cc, page);
				continue;
			}
#endif
			ret = f2fs_write_single_data_page(page, &submitted,
					&bio, &last_block, wbc, io_type, 0);
			if (ret == AOP_WRITEPAGE_ACTIVATE)
				unlock_page(page);
#ifdef CONFIG_F2FS_FS_COMPRESSION
result:
#endif
			nwritten += submitted;
			wbc->nr_to_write -= submitted;

C
Chao Yu 已提交
3002
			if (unlikely(ret)) {
3003 3004 3005 3006 3007 3008
				/*
				 * keep nr_to_write, since vfs uses this to
				 * get # of written pages.
				 */
				if (ret == AOP_WRITEPAGE_ACTIVATE) {
					ret = 0;
C
Chao Yu 已提交
3009
					goto next;
3010 3011 3012 3013 3014
				} else if (ret == -EAGAIN) {
					ret = 0;
					if (wbc->sync_mode == WB_SYNC_ALL) {
						cond_resched();
						congestion_wait(BLK_RW_ASYNC,
C
Chao Yu 已提交
3015
							DEFAULT_IO_TIMEOUT);
3016 3017
						goto retry_write;
					}
C
Chao Yu 已提交
3018
					goto next;
3019
				}
J
Jaegeuk Kim 已提交
3020 3021 3022
				done_index = page->index + 1;
				done = 1;
				break;
C
Chao Yu 已提交
3023 3024
			}

C
Chao Yu 已提交
3025
			if (wbc->nr_to_write <= 0 &&
3026
					wbc->sync_mode == WB_SYNC_NONE) {
C
Chao Yu 已提交
3027 3028 3029
				done = 1;
				break;
			}
C
Chao Yu 已提交
3030 3031 3032
next:
			if (need_readd)
				goto readd;
C
Chao Yu 已提交
3033 3034 3035 3036
		}
		pagevec_release(&pvec);
		cond_resched();
	}
C
Chao Yu 已提交
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
#ifdef CONFIG_F2FS_FS_COMPRESSION
	/* flush remained pages in compress cluster */
	if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) {
		ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type);
		nwritten += submitted;
		wbc->nr_to_write -= submitted;
		if (ret) {
			done = 1;
			retry = 0;
		}
	}
#endif
	if ((!cycled && !done) || retry) {
C
Chao Yu 已提交
3050 3051 3052 3053 3054 3055 3056 3057
		cycled = 1;
		index = 0;
		end = writeback_index - 1;
		goto retry;
	}
	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
		mapping->writeback_index = done_index;

3058
	if (nwritten)
3059
		f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
3060
								NULL, 0, DATA);
C
Chao Yu 已提交
3061 3062
	/* submit cached bio of IPU write */
	if (bio)
C
Chao Yu 已提交
3063
		f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
C
Chao Yu 已提交
3064

C
Chao Yu 已提交
3065 3066 3067
	return ret;
}

3068 3069 3070
static inline bool __should_serialize_io(struct inode *inode,
					struct writeback_control *wbc)
{
C
Chao Yu 已提交
3071 3072 3073 3074
	/* to avoid deadlock in path of data flush */
	if (F2FS_I(inode)->cp_task)
		return false;

3075 3076
	if (!S_ISREG(inode->i_mode))
		return false;
3077 3078
	if (IS_NOQUOTA(inode))
		return false;
C
Chao Yu 已提交
3079 3080 3081

	if (f2fs_compressed_file(inode))
		return true;
3082 3083 3084 3085 3086 3087 3088
	if (wbc->sync_mode != WB_SYNC_ALL)
		return true;
	if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
		return true;
	return false;
}

3089
static int __f2fs_write_data_pages(struct address_space *mapping,
C
Chao Yu 已提交
3090 3091
						struct writeback_control *wbc,
						enum iostat_type io_type)
3092 3093
{
	struct inode *inode = mapping->host;
3094
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3095
	struct blk_plug plug;
3096
	int ret;
3097
	bool locked = false;
3098

P
P J P 已提交
3099 3100 3101 3102
	/* deal with chardevs and other special file */
	if (!mapping->a_ops->writepage)
		return 0;

3103 3104 3105 3106
	/* skip writing if there is no dirty page in this inode */
	if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
		return 0;

3107 3108 3109 3110
	/* during POR, we don't need to trigger writepage at all. */
	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
		goto skip_write;

3111 3112
	if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
			wbc->sync_mode == WB_SYNC_NONE &&
3113
			get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
C
Chao Yu 已提交
3114
			f2fs_available_free_memory(sbi, DIRTY_DENTS))
3115 3116
		goto skip_write;

C
Chao Yu 已提交
3117
	/* skip writing during file defragment */
3118
	if (is_inode_flag_set(inode, FI_DO_DEFRAG))
C
Chao Yu 已提交
3119 3120
		goto skip_write;

Y
Yunlei He 已提交
3121 3122
	trace_f2fs_writepages(mapping->host, wbc, DATA);

3123 3124
	/* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
	if (wbc->sync_mode == WB_SYNC_ALL)
3125 3126
		atomic_inc(&sbi->wb_sync_req[DATA]);
	else if (atomic_read(&sbi->wb_sync_req[DATA]))
3127 3128
		goto skip_write;

3129 3130 3131 3132 3133
	if (__should_serialize_io(inode, wbc)) {
		mutex_lock(&sbi->writepages);
		locked = true;
	}

3134
	blk_start_plug(&plug);
C
Chao Yu 已提交
3135
	ret = f2fs_write_cache_pages(mapping, wbc, io_type);
3136
	blk_finish_plug(&plug);
3137

3138 3139 3140
	if (locked)
		mutex_unlock(&sbi->writepages);

3141
	if (wbc->sync_mode == WB_SYNC_ALL)
3142
		atomic_dec(&sbi->wb_sync_req[DATA]);
3143 3144 3145 3146
	/*
	 * if some pages were truncated, we cannot guarantee its mapping->host
	 * to detect pending bios.
	 */
J
Jaegeuk Kim 已提交
3147

C
Chao Yu 已提交
3148
	f2fs_remove_dirty_inode(inode);
3149
	return ret;
3150 3151

skip_write:
3152
	wbc->pages_skipped += get_dirty_pages(inode);
Y
Yunlei He 已提交
3153
	trace_f2fs_writepages(mapping->host, wbc, DATA);
3154
	return 0;
3155 3156
}

C
Chao Yu 已提交
3157 3158 3159 3160 3161 3162 3163 3164 3165 3166
static int f2fs_write_data_pages(struct address_space *mapping,
			    struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;

	return __f2fs_write_data_pages(mapping, wbc,
			F2FS_I(inode)->cp_task == current ?
			FS_CP_DATA_IO : FS_DATA_IO);
}

3167 3168 3169
static void f2fs_write_failed(struct address_space *mapping, loff_t to)
{
	struct inode *inode = mapping->host;
J
Jaegeuk Kim 已提交
3170
	loff_t i_size = i_size_read(inode);
3171

3172 3173 3174
	if (IS_NOQUOTA(inode))
		return;

E
Eric Biggers 已提交
3175 3176
	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
	if (to > i_size && !f2fs_verity_in_progress(inode)) {
3177
		down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3178
		down_write(&F2FS_I(inode)->i_mmap_sem);
3179

J
Jaegeuk Kim 已提交
3180
		truncate_pagecache(inode, i_size);
3181
		f2fs_truncate_blocks(inode, i_size, true);
3182

3183
		up_write(&F2FS_I(inode)->i_mmap_sem);
3184
		up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3185 3186 3187
	}
}

3188 3189 3190 3191 3192 3193 3194 3195
static int prepare_write_begin(struct f2fs_sb_info *sbi,
			struct page *page, loff_t pos, unsigned len,
			block_t *blk_addr, bool *node_changed)
{
	struct inode *inode = page->mapping->host;
	pgoff_t index = page->index;
	struct dnode_of_data dn;
	struct page *ipage;
3196
	bool locked = false;
3197
	struct extent_info ei = {0,0,0};
3198
	int err = 0;
3199
	int flag;
3200

3201 3202 3203 3204
	/*
	 * we already allocated all the blocks, so we don't need to get
	 * the block addresses when there is no need to fill the page.
	 */
3205
	if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
E
Eric Biggers 已提交
3206 3207
	    !is_inode_flag_set(inode, FI_NO_PREALLOC) &&
	    !f2fs_verity_in_progress(inode))
3208 3209
		return 0;

3210 3211 3212 3213 3214 3215
	/* f2fs_lock_op avoids race between write CP and convert_inline_page */
	if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
		flag = F2FS_GET_BLOCK_DEFAULT;
	else
		flag = F2FS_GET_BLOCK_PRE_AIO;

3216
	if (f2fs_has_inline_data(inode) ||
3217
			(pos & PAGE_MASK) >= i_size_read(inode)) {
3218
		__do_map_lock(sbi, flag, true);
3219 3220
		locked = true;
	}
C
Chao Yu 已提交
3221

3222
restart:
3223
	/* check inline_data */
C
Chao Yu 已提交
3224
	ipage = f2fs_get_node_page(sbi, inode->i_ino);
3225 3226 3227 3228 3229 3230 3231 3232
	if (IS_ERR(ipage)) {
		err = PTR_ERR(ipage);
		goto unlock_out;
	}

	set_new_dnode(&dn, inode, ipage, ipage, 0);

	if (f2fs_has_inline_data(inode)) {
C
Chao Yu 已提交
3233
		if (pos + len <= MAX_INLINE_DATA(inode)) {
C
Chao Yu 已提交
3234
			f2fs_do_read_inline_data(page, ipage);
3235
			set_inode_flag(inode, FI_DATA_EXIST);
3236 3237
			if (inode->i_nlink)
				set_inline_node(ipage);
3238 3239 3240
		} else {
			err = f2fs_convert_inline_page(&dn, page);
			if (err)
3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
				goto out;
			if (dn.data_blkaddr == NULL_ADDR)
				err = f2fs_get_block(&dn, index);
		}
	} else if (locked) {
		err = f2fs_get_block(&dn, index);
	} else {
		if (f2fs_lookup_extent_cache(inode, index, &ei)) {
			dn.data_blkaddr = ei.blk + index - ei.fofs;
		} else {
			/* hole case */
C
Chao Yu 已提交
3252
			err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3253
			if (err || dn.data_blkaddr == NULL_ADDR) {
3254
				f2fs_put_dnode(&dn);
3255 3256
				__do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
								true);
3257
				WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
3258 3259 3260
				locked = true;
				goto restart;
			}
3261 3262
		}
	}
3263

3264 3265 3266
	/* convert_inline_page can make node_changed */
	*blk_addr = dn.data_blkaddr;
	*node_changed = dn.node_changed;
3267
out:
3268 3269
	f2fs_put_dnode(&dn);
unlock_out:
3270
	if (locked)
3271
		__do_map_lock(sbi, flag, false);
3272 3273 3274
	return err;
}

3275 3276 3277 3278 3279
static int f2fs_write_begin(struct file *file, struct address_space *mapping,
		loff_t pos, unsigned len, unsigned flags,
		struct page **pagep, void **fsdata)
{
	struct inode *inode = mapping->host;
3280
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3281
	struct page *page = NULL;
3282
	pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3283
	bool need_balance = false, drop_atomic = false;
3284
	block_t blkaddr = NULL_ADDR;
3285 3286
	int err = 0;

3287 3288
	trace_f2fs_write_begin(inode, pos, len, flags);

3289 3290
	if (!f2fs_is_checkpoint_ready(sbi)) {
		err = -ENOSPC;
D
Daniel Rosenberg 已提交
3291
		goto fail;
3292
	}
D
Daniel Rosenberg 已提交
3293

3294 3295 3296
	if ((f2fs_is_atomic_file(inode) &&
			!f2fs_available_free_memory(sbi, INMEM_PAGES)) ||
			is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) {
J
Jaegeuk Kim 已提交
3297
		err = -ENOMEM;
3298
		drop_atomic = true;
J
Jaegeuk Kim 已提交
3299 3300 3301
		goto fail;
	}

3302 3303 3304 3305 3306 3307 3308 3309 3310 3311
	/*
	 * We should check this at this moment to avoid deadlock on inode page
	 * and #0 page. The locking rule for inline_data conversion should be:
	 * lock_page(page #0) -> lock_page(inode_page)
	 */
	if (index != 0) {
		err = f2fs_convert_inline_inode(inode);
		if (err)
			goto fail;
	}
C
Chao Yu 已提交
3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329

#ifdef CONFIG_F2FS_FS_COMPRESSION
	if (f2fs_compressed_file(inode)) {
		int ret;

		*fsdata = NULL;

		ret = f2fs_prepare_compress_overwrite(inode, pagep,
							index, fsdata);
		if (ret < 0) {
			err = ret;
			goto fail;
		} else if (ret) {
			return 0;
		}
	}
#endif

3330
repeat:
3331 3332 3333 3334
	/*
	 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
	 * wait_for_stable_page. Will wait that below with our IO control.
	 */
C
Chao Yu 已提交
3335
	page = f2fs_pagecache_get_page(mapping, index,
3336
				FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3337 3338 3339 3340
	if (!page) {
		err = -ENOMEM;
		goto fail;
	}
3341

C
Chao Yu 已提交
3342 3343
	/* TODO: cluster can be compressed due to race with .writepage */

3344 3345
	*pagep = page;

3346 3347
	err = prepare_write_begin(sbi, page, pos, len,
					&blkaddr, &need_balance);
3348
	if (err)
3349
		goto fail;
3350

3351 3352
	if (need_balance && !IS_NOQUOTA(inode) &&
			has_not_enough_free_secs(sbi, 0, 0)) {
3353
		unlock_page(page);
J
Jaegeuk Kim 已提交
3354
		f2fs_balance_fs(sbi, true);
3355 3356 3357 3358 3359 3360 3361 3362
		lock_page(page);
		if (page->mapping != mapping) {
			/* The page got truncated from under us */
			f2fs_put_page(page, 1);
			goto repeat;
		}
	}

3363
	f2fs_wait_on_page_writeback(page, DATA, false, true);
3364

3365 3366
	if (len == PAGE_SIZE || PageUptodate(page))
		return 0;
3367

E
Eric Biggers 已提交
3368 3369
	if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) &&
	    !f2fs_verity_in_progress(inode)) {
3370 3371 3372 3373
		zero_user_segment(page, len, PAGE_SIZE);
		return 0;
	}

3374
	if (blkaddr == NEW_ADDR) {
3375
		zero_user_segment(page, 0, PAGE_SIZE);
3376
		SetPageUptodate(page);
3377
	} else {
C
Chao Yu 已提交
3378 3379
		if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
				DATA_GENERIC_ENHANCE_READ)) {
3380
			err = -EFSCORRUPTED;
C
Chao Yu 已提交
3381 3382
			goto fail;
		}
3383
		err = f2fs_submit_page_read(inode, page, blkaddr, true);
3384
		if (err)
3385
			goto fail;
3386

3387
		lock_page(page);
3388
		if (unlikely(page->mapping != mapping)) {
3389 3390
			f2fs_put_page(page, 1);
			goto repeat;
3391
		}
3392 3393 3394
		if (unlikely(!PageUptodate(page))) {
			err = -EIO;
			goto fail;
3395
		}
3396 3397
	}
	return 0;
3398

3399
fail:
3400
	f2fs_put_page(page, 1);
3401
	f2fs_write_failed(mapping, pos + len);
3402
	if (drop_atomic)
C
Chao Yu 已提交
3403
		f2fs_drop_inmem_pages_all(sbi, false);
3404
	return err;
3405 3406
}

3407 3408 3409 3410 3411 3412 3413
static int f2fs_write_end(struct file *file,
			struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
{
	struct inode *inode = page->mapping->host;

3414 3415
	trace_f2fs_write_end(inode, pos, len, copied);

3416 3417 3418 3419 3420 3421
	/*
	 * This should be come from len == PAGE_SIZE, and we expect copied
	 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
	 * let generic_perform_write() try to copy data again through copied=0.
	 */
	if (!PageUptodate(page)) {
3422
		if (unlikely(copied != len))
3423 3424 3425 3426
			copied = 0;
		else
			SetPageUptodate(page);
	}
C
Chao Yu 已提交
3427 3428 3429 3430 3431 3432 3433 3434 3435 3436

#ifdef CONFIG_F2FS_FS_COMPRESSION
	/* overwrite compressed file */
	if (f2fs_compressed_file(inode) && fsdata) {
		f2fs_compress_write_end(inode, fsdata, page->index, copied);
		f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
		return copied;
	}
#endif

3437 3438 3439
	if (!copied)
		goto unlock_out;

3440
	set_page_dirty(page);
3441

E
Eric Biggers 已提交
3442 3443
	if (pos + copied > i_size_read(inode) &&
	    !f2fs_verity_in_progress(inode))
3444
		f2fs_i_size_write(inode, pos + copied);
3445
unlock_out:
3446
	f2fs_put_page(page, 1);
3447
	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3448 3449 3450
	return copied;
}

3451 3452
static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
			   loff_t offset)
3453
{
3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467
	unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
	unsigned blkbits = i_blkbits;
	unsigned blocksize_mask = (1 << blkbits) - 1;
	unsigned long align = offset | iov_iter_alignment(iter);
	struct block_device *bdev = inode->i_sb->s_bdev;

	if (align & blocksize_mask) {
		if (bdev)
			blkbits = blksize_bits(bdev_logical_block_size(bdev));
		blocksize_mask = (1 << blkbits) - 1;
		if (align & blocksize_mask)
			return -EINVAL;
		return 1;
	}
3468 3469 3470
	return 0;
}

C
Chao Yu 已提交
3471 3472 3473 3474 3475 3476 3477 3478 3479 3480
static void f2fs_dio_end_io(struct bio *bio)
{
	struct f2fs_private_dio *dio = bio->bi_private;

	dec_page_count(F2FS_I_SB(dio->inode),
			dio->write ? F2FS_DIO_WRITE : F2FS_DIO_READ);

	bio->bi_private = dio->orig_private;
	bio->bi_end_io = dio->orig_end_io;

3481
	kvfree(dio);
C
Chao Yu 已提交
3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493

	bio_endio(bio);
}

static void f2fs_dio_submit_bio(struct bio *bio, struct inode *inode,
							loff_t file_offset)
{
	struct f2fs_private_dio *dio;
	bool write = (bio_op(bio) == REQ_OP_WRITE);

	dio = f2fs_kzalloc(F2FS_I_SB(inode),
			sizeof(struct f2fs_private_dio), GFP_NOFS);
3494
	if (!dio)
C
Chao Yu 已提交
3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514
		goto out;

	dio->inode = inode;
	dio->orig_end_io = bio->bi_end_io;
	dio->orig_private = bio->bi_private;
	dio->write = write;

	bio->bi_end_io = f2fs_dio_end_io;
	bio->bi_private = dio;

	inc_page_count(F2FS_I_SB(inode),
			write ? F2FS_DIO_WRITE : F2FS_DIO_READ);

	submit_bio(bio);
	return;
out:
	bio->bi_status = BLK_STS_IOERR;
	bio_endio(bio);
}

3515
static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3516
{
3517
	struct address_space *mapping = iocb->ki_filp->f_mapping;
3518
	struct inode *inode = mapping->host;
3519
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3520
	struct f2fs_inode_info *fi = F2FS_I(inode);
3521
	size_t count = iov_iter_count(iter);
3522
	loff_t offset = iocb->ki_pos;
3523
	int rw = iov_iter_rw(iter);
3524
	int err;
3525
	enum rw_hint hint = iocb->ki_hint;
3526
	int whint_mode = F2FS_OPTION(sbi).whint_mode;
3527
	bool do_opu;
3528

3529
	err = check_direct_IO(inode, iter, offset);
3530
	if (err)
3531
		return err < 0 ? err : 0;
H
Huajun Li 已提交
3532

3533
	if (f2fs_force_buffered_io(inode, iocb, iter))
3534
		return 0;
3535

3536 3537
	do_opu = allow_outplace_dio(inode, iocb, iter);

3538
	trace_f2fs_direct_IO_enter(inode, offset, count, rw);
3539

3540 3541 3542
	if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
		iocb->ki_hint = WRITE_LIFE_NOT_SET;

3543 3544 3545 3546 3547 3548 3549 3550
	if (iocb->ki_flags & IOCB_NOWAIT) {
		if (!down_read_trylock(&fi->i_gc_rwsem[rw])) {
			iocb->ki_hint = hint;
			err = -EAGAIN;
			goto out;
		}
		if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) {
			up_read(&fi->i_gc_rwsem[rw]);
H
Hyunchul Lee 已提交
3551 3552 3553 3554
			iocb->ki_hint = hint;
			err = -EAGAIN;
			goto out;
		}
3555 3556 3557 3558
	} else {
		down_read(&fi->i_gc_rwsem[rw]);
		if (do_opu)
			down_read(&fi->i_gc_rwsem[READ]);
H
Hyunchul Lee 已提交
3559 3560
	}

C
Chao Yu 已提交
3561
	err = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
C
Chao Yu 已提交
3562 3563
			iter, rw == WRITE ? get_data_block_dio_write :
			get_data_block_dio, NULL, f2fs_dio_submit_bio,
D
DongDongJu 已提交
3564 3565
			rw == WRITE ? DIO_LOCKING | DIO_SKIP_HOLES :
			DIO_SKIP_HOLES);
3566 3567 3568 3569 3570

	if (do_opu)
		up_read(&fi->i_gc_rwsem[READ]);

	up_read(&fi->i_gc_rwsem[rw]);
3571 3572

	if (rw == WRITE) {
3573 3574
		if (whint_mode == WHINT_MODE_OFF)
			iocb->ki_hint = hint;
C
Chao Yu 已提交
3575 3576 3577
		if (err > 0) {
			f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
									err);
3578 3579
			if (!do_opu)
				set_inode_flag(inode, FI_UPDATE_WRITE);
C
Chao Yu 已提交
3580
		} else if (err < 0) {
3581
			f2fs_write_failed(mapping, offset + count);
C
Chao Yu 已提交
3582
		}
C
Chao Yu 已提交
3583 3584 3585
	} else {
		if (err > 0)
			f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, err);
3586
	}
3587

H
Hyunchul Lee 已提交
3588
out:
3589
	trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
3590

3591
	return err;
3592 3593
}

3594 3595
void f2fs_invalidate_page(struct page *page, unsigned int offset,
							unsigned int length)
3596 3597
{
	struct inode *inode = page->mapping->host;
3598
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3599

3600
	if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
3601
		(offset % PAGE_SIZE || length != PAGE_SIZE))
3602 3603
		return;

3604
	if (PageDirty(page)) {
3605
		if (inode->i_ino == F2FS_META_INO(sbi)) {
3606
			dec_page_count(sbi, F2FS_DIRTY_META);
3607
		} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3608
			dec_page_count(sbi, F2FS_DIRTY_NODES);
3609
		} else {
3610
			inode_dec_dirty_pages(inode);
C
Chao Yu 已提交
3611
			f2fs_remove_dirty_inode(inode);
3612
		}
3613
	}
C
Chao Yu 已提交
3614

C
Chao Yu 已提交
3615 3616
	clear_cold_data(page);

C
Chao Yu 已提交
3617
	if (IS_ATOMIC_WRITTEN_PAGE(page))
C
Chao Yu 已提交
3618
		return f2fs_drop_inmem_page(inode, page);
C
Chao Yu 已提交
3619

3620
	f2fs_clear_page_private(page);
3621 3622
}

3623
int f2fs_release_page(struct page *page, gfp_t wait)
3624
{
3625 3626 3627 3628
	/* If this is dirty page, keep PagePrivate */
	if (PageDirty(page))
		return 0;

C
Chao Yu 已提交
3629 3630 3631 3632
	/* This is atomic written page, keep Private */
	if (IS_ATOMIC_WRITTEN_PAGE(page))
		return 0;

C
Chao Yu 已提交
3633
	clear_cold_data(page);
3634
	f2fs_clear_page_private(page);
3635
	return 1;
3636 3637 3638 3639
}

static int f2fs_set_data_page_dirty(struct page *page)
{
J
Jaegeuk Kim 已提交
3640
	struct inode *inode = page_file_mapping(page)->host;
3641

3642 3643
	trace_f2fs_set_page_dirty(page, DATA);

3644 3645
	if (!PageUptodate(page))
		SetPageUptodate(page);
J
Jaegeuk Kim 已提交
3646 3647
	if (PageSwapCache(page))
		return __set_page_dirty_nobuffers(page);
3648

C
Chao Yu 已提交
3649
	if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
C
Chao Yu 已提交
3650
		if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
C
Chao Yu 已提交
3651
			f2fs_register_inmem_page(inode, page);
C
Chao Yu 已提交
3652 3653 3654 3655 3656 3657 3658
			return 1;
		}
		/*
		 * Previously, this page has been registered, we just
		 * return here.
		 */
		return 0;
3659 3660
	}

3661
	if (!PageDirty(page)) {
3662
		__set_page_dirty_nobuffers(page);
C
Chao Yu 已提交
3663
		f2fs_update_dirty_page(inode, page);
3664 3665 3666 3667 3668
		return 1;
	}
	return 0;
}

C
Chao Yu 已提交
3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699

static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
{
#ifdef CONFIG_F2FS_FS_COMPRESSION
	struct dnode_of_data dn;
	sector_t start_idx, blknr = 0;
	int ret;

	start_idx = round_down(block, F2FS_I(inode)->i_cluster_size);

	set_new_dnode(&dn, inode, NULL, NULL, 0);
	ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
	if (ret)
		return 0;

	if (dn.data_blkaddr != COMPRESS_ADDR) {
		dn.ofs_in_node += block - start_idx;
		blknr = f2fs_data_blkaddr(&dn);
		if (!__is_valid_data_blkaddr(blknr))
			blknr = 0;
	}

	f2fs_put_dnode(&dn);

	return blknr;
#else
	return -EOPNOTSUPP;
#endif
}


J
Jaegeuk Kim 已提交
3700 3701
static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
{
3702 3703
	struct inode *inode = mapping->host;

J
Jaegeuk Kim 已提交
3704 3705 3706 3707 3708 3709 3710
	if (f2fs_has_inline_data(inode))
		return 0;

	/* make sure allocating whole blocks */
	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
		filemap_write_and_wait(mapping);

C
Chao Yu 已提交
3711 3712 3713
	if (f2fs_compressed_file(inode))
		return f2fs_bmap_compress(inode, block);

C
Chao Yu 已提交
3714
	return generic_block_bmap(mapping, block, get_data_block_bmap);
3715 3716
}

3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729
#ifdef CONFIG_MIGRATION
#include <linux/migrate.h>

int f2fs_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	int rc, extra_count;
	struct f2fs_inode_info *fi = F2FS_I(mapping->host);
	bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);

	BUG_ON(PageWriteback(page));

	/* migrating an atomic written page is safe with the inmem_lock hold */
3730 3731 3732 3733 3734 3735
	if (atomic_written) {
		if (mode != MIGRATE_SYNC)
			return -EBUSY;
		if (!mutex_trylock(&fi->inmem_lock))
			return -EAGAIN;
	}
3736

3737 3738
	/* one extra reference was held for atomic_write page */
	extra_count = atomic_written ? 1 : 0;
3739
	rc = migrate_page_move_mapping(mapping, newpage,
3740
				page, extra_count);
3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758
	if (rc != MIGRATEPAGE_SUCCESS) {
		if (atomic_written)
			mutex_unlock(&fi->inmem_lock);
		return rc;
	}

	if (atomic_written) {
		struct inmem_pages *cur;
		list_for_each_entry(cur, &fi->inmem_pages, list)
			if (cur->page == page) {
				cur->page = newpage;
				break;
			}
		mutex_unlock(&fi->inmem_lock);
		put_page(page);
		get_page(newpage);
	}

3759 3760 3761 3762
	if (PagePrivate(page)) {
		f2fs_set_page_private(newpage, page_private(page));
		f2fs_clear_page_private(page);
	}
3763

3764 3765 3766 3767
	if (mode != MIGRATE_SYNC_NO_COPY)
		migrate_page_copy(newpage, page);
	else
		migrate_page_states(newpage, page);
3768 3769 3770 3771 3772

	return MIGRATEPAGE_SUCCESS;
}
#endif

J
Jaegeuk Kim 已提交
3773 3774
#ifdef CONFIG_SWAP
/* Copied from generic_swapfile_activate() to check any holes */
C
Chao Yu 已提交
3775 3776
static int check_swap_activate(struct swap_info_struct *sis,
				struct file *swap_file, sector_t *span)
J
Jaegeuk Kim 已提交
3777 3778 3779 3780 3781 3782 3783 3784 3785 3786
{
	struct address_space *mapping = swap_file->f_mapping;
	struct inode *inode = mapping->host;
	unsigned blocks_per_page;
	unsigned long page_no;
	unsigned blkbits;
	sector_t probe_block;
	sector_t last_block;
	sector_t lowest_block = -1;
	sector_t highest_block = 0;
C
Chao Yu 已提交
3787 3788
	int nr_extents = 0;
	int ret;
J
Jaegeuk Kim 已提交
3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799

	blkbits = inode->i_blkbits;
	blocks_per_page = PAGE_SIZE >> blkbits;

	/*
	 * Map all the blocks into the extent list.  This code doesn't try
	 * to be very smart.
	 */
	probe_block = 0;
	page_no = 0;
	last_block = i_size_read(inode) >> blkbits;
C
Chao Yu 已提交
3800 3801
	while ((probe_block + blocks_per_page) <= last_block &&
			page_no < sis->max) {
J
Jaegeuk Kim 已提交
3802 3803
		unsigned block_in_page;
		sector_t first_block;
3804 3805
		sector_t block = 0;
		int	 err = 0;
J
Jaegeuk Kim 已提交
3806 3807 3808

		cond_resched();

3809 3810 3811
		block = probe_block;
		err = bmap(inode, &block);
		if (err || !block)
J
Jaegeuk Kim 已提交
3812
			goto bad_bmap;
3813
		first_block = block;
J
Jaegeuk Kim 已提交
3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825

		/*
		 * It must be PAGE_SIZE aligned on-disk
		 */
		if (first_block & (blocks_per_page - 1)) {
			probe_block++;
			goto reprobe;
		}

		for (block_in_page = 1; block_in_page < blocks_per_page;
					block_in_page++) {

3826 3827 3828 3829
			block = probe_block + block_in_page;
			err = bmap(inode, &block);

			if (err || !block)
J
Jaegeuk Kim 已提交
3830
				goto bad_bmap;
3831

J
Jaegeuk Kim 已提交
3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
			if (block != first_block + block_in_page) {
				/* Discontiguity */
				probe_block++;
				goto reprobe;
			}
		}

		first_block >>= (PAGE_SHIFT - blkbits);
		if (page_no) {	/* exclude the header page */
			if (first_block < lowest_block)
				lowest_block = first_block;
			if (first_block > highest_block)
				highest_block = first_block;
		}

C
Chao Yu 已提交
3847 3848 3849 3850 3851 3852 3853
		/*
		 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
		 */
		ret = add_swap_extent(sis, page_no, 1, first_block);
		if (ret < 0)
			goto out;
		nr_extents += ret;
J
Jaegeuk Kim 已提交
3854 3855 3856 3857 3858
		page_no++;
		probe_block += blocks_per_page;
reprobe:
		continue;
	}
C
Chao Yu 已提交
3859 3860 3861 3862 3863 3864 3865 3866 3867
	ret = nr_extents;
	*span = 1 + highest_block - lowest_block;
	if (page_no == 0)
		page_no = 1;	/* force Empty message */
	sis->max = page_no;
	sis->pages = page_no - 1;
	sis->highest_bit = page_no - 1;
out:
	return ret;
J
Jaegeuk Kim 已提交
3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888
bad_bmap:
	pr_err("swapon: swapfile has holes\n");
	return -EINVAL;
}

static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
				sector_t *span)
{
	struct inode *inode = file_inode(file);
	int ret;

	if (!S_ISREG(inode->i_mode))
		return -EINVAL;

	if (f2fs_readonly(F2FS_I_SB(inode)->sb))
		return -EROFS;

	ret = f2fs_convert_inline_inode(inode);
	if (ret)
		return ret;

C
Chao Yu 已提交
3889 3890 3891
	if (f2fs_disable_compressed_file(inode))
		return -EINVAL;

C
Chao Yu 已提交
3892 3893
	ret = check_swap_activate(sis, file, span);
	if (ret < 0)
J
Jaegeuk Kim 已提交
3894 3895 3896 3897 3898
		return ret;

	set_inode_flag(inode, FI_PIN_FILE);
	f2fs_precache_extents(inode);
	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
C
Chao Yu 已提交
3899
	return ret;
J
Jaegeuk Kim 已提交
3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919
}

static void f2fs_swap_deactivate(struct file *file)
{
	struct inode *inode = file_inode(file);

	clear_inode_flag(inode, FI_PIN_FILE);
}
#else
static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
				sector_t *span)
{
	return -EOPNOTSUPP;
}

static void f2fs_swap_deactivate(struct file *file)
{
}
#endif

3920 3921 3922 3923 3924 3925
const struct address_space_operations f2fs_dblock_aops = {
	.readpage	= f2fs_read_data_page,
	.readpages	= f2fs_read_data_pages,
	.writepage	= f2fs_write_data_page,
	.writepages	= f2fs_write_data_pages,
	.write_begin	= f2fs_write_begin,
3926
	.write_end	= f2fs_write_end,
3927
	.set_page_dirty	= f2fs_set_data_page_dirty,
3928 3929
	.invalidatepage	= f2fs_invalidate_page,
	.releasepage	= f2fs_release_page,
3930
	.direct_IO	= f2fs_direct_IO,
J
Jaegeuk Kim 已提交
3931
	.bmap		= f2fs_bmap,
J
Jaegeuk Kim 已提交
3932 3933
	.swap_activate  = f2fs_swap_activate,
	.swap_deactivate = f2fs_swap_deactivate,
3934 3935 3936
#ifdef CONFIG_MIGRATION
	.migratepage    = f2fs_migrate_page,
#endif
3937
};
3938

M
Matthew Wilcox 已提交
3939
void f2fs_clear_page_cache_dirty_tag(struct page *page)
3940 3941 3942 3943 3944
{
	struct address_space *mapping = page_mapping(page);
	unsigned long flags;

	xa_lock_irqsave(&mapping->i_pages, flags);
M
Matthew Wilcox 已提交
3945
	__xa_clear_mark(&mapping->i_pages, page_index(page),
3946 3947 3948 3949
						PAGECACHE_TAG_DIRTY);
	xa_unlock_irqrestore(&mapping->i_pages, flags);
}

3950 3951
int __init f2fs_init_post_read_processing(void)
{
E
Eric Biggers 已提交
3952 3953 3954
	bio_post_read_ctx_cache =
		kmem_cache_create("f2fs_bio_post_read_ctx",
				  sizeof(struct bio_post_read_ctx), 0, 0, NULL);
3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969
	if (!bio_post_read_ctx_cache)
		goto fail;
	bio_post_read_ctx_pool =
		mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
					 bio_post_read_ctx_cache);
	if (!bio_post_read_ctx_pool)
		goto fail_free_cache;
	return 0;

fail_free_cache:
	kmem_cache_destroy(bio_post_read_ctx_cache);
fail:
	return -ENOMEM;
}

C
Chao Yu 已提交
3970
void f2fs_destroy_post_read_processing(void)
3971 3972 3973 3974
{
	mempool_destroy(bio_post_read_ctx_pool);
	kmem_cache_destroy(bio_post_read_ctx_cache);
}
C
Chao Yu 已提交
3975

C
Chao Yu 已提交
3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi)
{
	if (!f2fs_sb_has_encrypt(sbi) &&
		!f2fs_sb_has_verity(sbi) &&
		!f2fs_sb_has_compression(sbi))
		return 0;

	sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq",
						 WQ_UNBOUND | WQ_HIGHPRI,
						 num_online_cpus());
	if (!sbi->post_read_wq)
		return -ENOMEM;
	return 0;
}

void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi)
{
	if (sbi->post_read_wq)
		destroy_workqueue(sbi->post_read_wq);
}

C
Chao Yu 已提交
3997 3998
int __init f2fs_init_bio_entry_cache(void)
{
3999
	bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab",
C
Chao Yu 已提交
4000 4001 4002 4003 4004 4005
			sizeof(struct bio_entry));
	if (!bio_entry_slab)
		return -ENOMEM;
	return 0;
}

C
Chao Yu 已提交
4006
void f2fs_destroy_bio_entry_cache(void)
C
Chao Yu 已提交
4007 4008 4009
{
	kmem_cache_destroy(bio_entry_slab);
}