checkpoint.c 23.5 KB
Newer Older
J
Jaegeuk Kim 已提交
1
/*
J
Jaegeuk Kim 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 * fs/f2fs/checkpoint.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/bio.h>
#include <linux/mpage.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/f2fs_fs.h>
#include <linux/pagevec.h>
#include <linux/swap.h>

#include "f2fs.h"
#include "node.h"
#include "segment.h"
23
#include <trace/events/f2fs.h>
J
Jaegeuk Kim 已提交
24 25 26 27

static struct kmem_cache *orphan_entry_slab;
static struct kmem_cache *inode_entry_slab;

J
Jaegeuk Kim 已提交
28
/*
J
Jaegeuk Kim 已提交
29 30 31 32
 * We guarantee no failure on the returned page.
 */
struct page *grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
{
G
Gu Zheng 已提交
33
	struct address_space *mapping = META_MAPPING(sbi);
J
Jaegeuk Kim 已提交
34 35 36 37 38 39 40 41 42 43 44 45 46 47
	struct page *page = NULL;
repeat:
	page = grab_cache_page(mapping, index);
	if (!page) {
		cond_resched();
		goto repeat;
	}

	/* We wait writeback only inside grab_meta_page() */
	wait_on_page_writeback(page);
	SetPageUptodate(page);
	return page;
}

J
Jaegeuk Kim 已提交
48
/*
J
Jaegeuk Kim 已提交
49 50 51 52
 * We guarantee no failure on the returned page.
 */
struct page *get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
{
G
Gu Zheng 已提交
53
	struct address_space *mapping = META_MAPPING(sbi);
J
Jaegeuk Kim 已提交
54 55 56 57 58 59 60
	struct page *page;
repeat:
	page = grab_cache_page(mapping, index);
	if (!page) {
		cond_resched();
		goto repeat;
	}
61 62 63
	if (PageUptodate(page))
		goto out;

64 65
	if (f2fs_submit_page_bio(sbi, page, index,
				READ_SYNC | REQ_META | REQ_PRIO))
J
Jaegeuk Kim 已提交
66 67
		goto repeat;

68
	lock_page(page);
69
	if (unlikely(page->mapping != mapping)) {
70 71 72
		f2fs_put_page(page, 1);
		goto repeat;
	}
73 74
out:
	mark_page_accessed(page);
J
Jaegeuk Kim 已提交
75 76 77
	return page;
}

78 79 80 81 82 83 84
inline int get_max_meta_blks(struct f2fs_sb_info *sbi, int type)
{
	switch (type) {
	case META_NAT:
		return NM_I(sbi)->max_nid / NAT_ENTRY_PER_BLOCK;
	case META_SIT:
		return SIT_BLK_CNT(sbi);
85
	case META_SSA:
86 87 88 89 90 91 92 93
	case META_CP:
		return 0;
	default:
		BUG();
	}
}

/*
94
 * Readahead CP/NAT/SIT/SSA pages
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
 */
int ra_meta_pages(struct f2fs_sb_info *sbi, int start, int nrpages, int type)
{
	block_t prev_blk_addr = 0;
	struct page *page;
	int blkno = start;
	int max_blks = get_max_meta_blks(sbi, type);

	struct f2fs_io_info fio = {
		.type = META,
		.rw = READ_SYNC | REQ_META | REQ_PRIO
	};

	for (; nrpages-- > 0; blkno++) {
		block_t blk_addr;

		switch (type) {
		case META_NAT:
			/* get nat block addr */
			if (unlikely(blkno >= max_blks))
				blkno = 0;
			blk_addr = current_nat_addr(sbi,
					blkno * NAT_ENTRY_PER_BLOCK);
			break;
		case META_SIT:
			/* get sit block addr */
			if (unlikely(blkno >= max_blks))
				goto out;
			blk_addr = current_sit_addr(sbi,
					blkno * SIT_ENTRY_PER_BLOCK);
			if (blkno != start && prev_blk_addr + 1 != blk_addr)
				goto out;
			prev_blk_addr = blk_addr;
			break;
129
		case META_SSA:
130
		case META_CP:
131
			/* get ssa/cp block addr */
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
			blk_addr = blkno;
			break;
		default:
			BUG();
		}

		page = grab_cache_page(META_MAPPING(sbi), blk_addr);
		if (!page)
			continue;
		if (PageUptodate(page)) {
			mark_page_accessed(page);
			f2fs_put_page(page, 1);
			continue;
		}

		f2fs_submit_page_mbio(sbi, page, blk_addr, &fio);
		mark_page_accessed(page);
		f2fs_put_page(page, 0);
	}
out:
	f2fs_submit_merged_bio(sbi, META, READ);
	return blkno - start;
}

J
Jaegeuk Kim 已提交
156 157 158 159 160 161
static int f2fs_write_meta_page(struct page *page,
				struct writeback_control *wbc)
{
	struct inode *inode = page->mapping->host;
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);

162
	if (unlikely(sbi->por_doing))
163 164 165
		goto redirty_out;
	if (wbc->for_reclaim)
		goto redirty_out;
J
Jaegeuk Kim 已提交
166

167 168 169
	/* Should not write any meta pages, if any IO error was occurred */
	if (unlikely(is_set_ckpt_flags(F2FS_CKPT(sbi), CP_ERROR_FLAG)))
		goto no_write;
J
Jaegeuk Kim 已提交
170

171
	wait_on_page_writeback(page);
172
	write_meta_page(sbi, page);
173
no_write:
174 175 176
	dec_page_count(sbi, F2FS_DIRTY_META);
	unlock_page(page);
	return 0;
177 178 179 180

redirty_out:
	dec_page_count(sbi, F2FS_DIRTY_META);
	wbc->pages_skipped++;
181
	account_page_redirty(page);
182 183
	set_page_dirty(page);
	return AOP_WRITEPAGE_ACTIVATE;
J
Jaegeuk Kim 已提交
184 185 186 187 188 189
}

static int f2fs_write_meta_pages(struct address_space *mapping,
				struct writeback_control *wbc)
{
	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
190
	int nrpages = nr_pages_to_skip(sbi, META);
J
Jaegeuk Kim 已提交
191 192
	long written;

193
	/* collect a number of dirty meta pages and write together */
194 195
	if (wbc->for_kupdate || get_pages(sbi, F2FS_DIRTY_META) < nrpages)
		goto skip_write;
J
Jaegeuk Kim 已提交
196 197 198

	/* if mounting is failed, skip writing node pages */
	mutex_lock(&sbi->cp_mutex);
199
	written = sync_meta_pages(sbi, META, nrpages);
J
Jaegeuk Kim 已提交
200 201 202
	mutex_unlock(&sbi->cp_mutex);
	wbc->nr_to_write -= written;
	return 0;
203 204 205 206

skip_write:
	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
	return 0;
J
Jaegeuk Kim 已提交
207 208 209 210 211
}

long sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
						long nr_to_write)
{
G
Gu Zheng 已提交
212
	struct address_space *mapping = META_MAPPING(sbi);
J
Jaegeuk Kim 已提交
213 214 215 216 217 218 219 220 221 222 223 224 225 226
	pgoff_t index = 0, end = LONG_MAX;
	struct pagevec pvec;
	long nwritten = 0;
	struct writeback_control wbc = {
		.for_reclaim = 0,
	};

	pagevec_init(&pvec, 0);

	while (index <= end) {
		int i, nr_pages;
		nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
				PAGECACHE_TAG_DIRTY,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
227
		if (unlikely(nr_pages == 0))
J
Jaegeuk Kim 已提交
228 229 230 231
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
232

J
Jaegeuk Kim 已提交
233
			lock_page(page);
234 235 236 237 238 239 240 241 242 243 244 245 246 247

			if (unlikely(page->mapping != mapping)) {
continue_unlock:
				unlock_page(page);
				continue;
			}
			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;

248 249 250 251
			if (f2fs_write_meta_page(page, &wbc)) {
				unlock_page(page);
				break;
			}
252 253
			nwritten++;
			if (unlikely(nwritten >= nr_to_write))
J
Jaegeuk Kim 已提交
254 255 256 257 258 259 260
				break;
		}
		pagevec_release(&pvec);
		cond_resched();
	}

	if (nwritten)
J
Jaegeuk Kim 已提交
261
		f2fs_submit_merged_bio(sbi, type, WRITE);
J
Jaegeuk Kim 已提交
262 263 264 265 266 267 268 269 270

	return nwritten;
}

static int f2fs_set_meta_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);

271 272
	trace_f2fs_set_page_dirty(page, META);

J
Jaegeuk Kim 已提交
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
	SetPageUptodate(page);
	if (!PageDirty(page)) {
		__set_page_dirty_nobuffers(page);
		inc_page_count(sbi, F2FS_DIRTY_META);
		return 1;
	}
	return 0;
}

const struct address_space_operations f2fs_meta_aops = {
	.writepage	= f2fs_write_meta_page,
	.writepages	= f2fs_write_meta_pages,
	.set_page_dirty	= f2fs_set_meta_page_dirty,
};

J
Jaegeuk Kim 已提交
288
int acquire_orphan_inode(struct f2fs_sb_info *sbi)
J
Jaegeuk Kim 已提交
289 290 291
{
	int err = 0;

292
	spin_lock(&sbi->orphan_inode_lock);
293
	if (unlikely(sbi->n_orphans >= sbi->max_orphans))
J
Jaegeuk Kim 已提交
294
		err = -ENOSPC;
J
Jaegeuk Kim 已提交
295 296
	else
		sbi->n_orphans++;
297
	spin_unlock(&sbi->orphan_inode_lock);
298

J
Jaegeuk Kim 已提交
299 300 301
	return err;
}

J
Jaegeuk Kim 已提交
302 303
void release_orphan_inode(struct f2fs_sb_info *sbi)
{
304
	spin_lock(&sbi->orphan_inode_lock);
305
	f2fs_bug_on(sbi->n_orphans == 0);
J
Jaegeuk Kim 已提交
306
	sbi->n_orphans--;
307
	spin_unlock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
308 309
}

J
Jaegeuk Kim 已提交
310 311 312 313 314
void add_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
{
	struct list_head *head, *this;
	struct orphan_inode_entry *new = NULL, *orphan = NULL;

315 316 317
	new = f2fs_kmem_cache_alloc(orphan_entry_slab, GFP_ATOMIC);
	new->ino = ino;

318
	spin_lock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
319 320 321
	head = &sbi->orphan_inode_list;
	list_for_each(this, head) {
		orphan = list_entry(this, struct orphan_inode_entry, list);
322
		if (orphan->ino == ino) {
323
			spin_unlock(&sbi->orphan_inode_lock);
324 325 326 327
			kmem_cache_free(orphan_entry_slab, new);
			return;
		}

J
Jaegeuk Kim 已提交
328 329 330 331
		if (orphan->ino > ino)
			break;
		orphan = NULL;
	}
332

J
Jaegeuk Kim 已提交
333
	/* add new_oentry into list which is sorted by inode number */
334 335 336
	if (orphan)
		list_add(&new->list, this->prev);
	else
J
Jaegeuk Kim 已提交
337
		list_add_tail(&new->list, head);
338
	spin_unlock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
339 340 341 342
}

void remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
{
343
	struct list_head *head;
J
Jaegeuk Kim 已提交
344 345
	struct orphan_inode_entry *orphan;

346
	spin_lock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
347
	head = &sbi->orphan_inode_list;
348
	list_for_each_entry(orphan, head, list) {
J
Jaegeuk Kim 已提交
349 350 351
		if (orphan->ino == ino) {
			list_del(&orphan->list);
			kmem_cache_free(orphan_entry_slab, orphan);
352
			f2fs_bug_on(sbi->n_orphans == 0);
J
Jaegeuk Kim 已提交
353 354 355 356
			sbi->n_orphans--;
			break;
		}
	}
357
	spin_unlock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
358 359 360 361 362
}

static void recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
{
	struct inode *inode = f2fs_iget(sbi->sb, ino);
363
	f2fs_bug_on(IS_ERR(inode));
J
Jaegeuk Kim 已提交
364 365 366 367 368 369
	clear_nlink(inode);

	/* truncate all the data during iput */
	iput(inode);
}

370
void recover_orphan_inodes(struct f2fs_sb_info *sbi)
J
Jaegeuk Kim 已提交
371 372 373
{
	block_t start_blk, orphan_blkaddr, i, j;

374
	if (!is_set_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG))
375
		return;
J
Jaegeuk Kim 已提交
376

H
Haicheng Li 已提交
377
	sbi->por_doing = true;
J
Jaegeuk Kim 已提交
378 379 380
	start_blk = __start_cp_addr(sbi) + 1;
	orphan_blkaddr = __start_sum_addr(sbi) - 1;

381 382
	ra_meta_pages(sbi, start_blk, orphan_blkaddr, META_CP);

J
Jaegeuk Kim 已提交
383 384 385 386 387 388 389 390 391 392 393 394
	for (i = 0; i < orphan_blkaddr; i++) {
		struct page *page = get_meta_page(sbi, start_blk + i);
		struct f2fs_orphan_block *orphan_blk;

		orphan_blk = (struct f2fs_orphan_block *)page_address(page);
		for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
			nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
			recover_orphan_inode(sbi, ino);
		}
		f2fs_put_page(page, 1);
	}
	/* clear Orphan Flag */
395
	clear_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG);
H
Haicheng Li 已提交
396
	sbi->por_doing = false;
397
	return;
J
Jaegeuk Kim 已提交
398 399 400 401
}

static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
{
402
	struct list_head *head;
J
Jaegeuk Kim 已提交
403 404
	struct f2fs_orphan_block *orphan_blk = NULL;
	unsigned int nentries = 0;
405 406 407 408
	unsigned short index;
	unsigned short orphan_blocks = (unsigned short)((sbi->n_orphans +
		(F2FS_ORPHANS_PER_BLOCK - 1)) / F2FS_ORPHANS_PER_BLOCK);
	struct page *page = NULL;
409
	struct orphan_inode_entry *orphan = NULL;
J
Jaegeuk Kim 已提交
410

411
	for (index = 0; index < orphan_blocks; index++)
412
		grab_meta_page(sbi, start_blk + index);
J
Jaegeuk Kim 已提交
413

414
	index = 1;
415
	spin_lock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
416 417 418
	head = &sbi->orphan_inode_list;

	/* loop for each orphan inode entry and write them in Jornal block */
419 420
	list_for_each_entry(orphan, head, list) {
		if (!page) {
421 422
			page = find_get_page(META_MAPPING(sbi), start_blk++);
			f2fs_bug_on(!page);
423 424 425
			orphan_blk =
				(struct f2fs_orphan_block *)page_address(page);
			memset(orphan_blk, 0, sizeof(*orphan_blk));
426
			f2fs_put_page(page, 0);
427
		}
J
Jaegeuk Kim 已提交
428

429
		orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
J
Jaegeuk Kim 已提交
430

431
		if (nentries == F2FS_ORPHANS_PER_BLOCK) {
J
Jaegeuk Kim 已提交
432 433 434 435 436 437 438 439 440 441 442 443 444 445
			/*
			 * an orphan block is full of 1020 entries,
			 * then we need to flush current orphan blocks
			 * and bring another one in memory
			 */
			orphan_blk->blk_addr = cpu_to_le16(index);
			orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
			orphan_blk->entry_count = cpu_to_le32(nentries);
			set_page_dirty(page);
			f2fs_put_page(page, 1);
			index++;
			nentries = 0;
			page = NULL;
		}
446
	}
J
Jaegeuk Kim 已提交
447

448 449 450 451 452 453
	if (page) {
		orphan_blk->blk_addr = cpu_to_le16(index);
		orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
		orphan_blk->entry_count = cpu_to_le32(nentries);
		set_page_dirty(page);
		f2fs_put_page(page, 1);
J
Jaegeuk Kim 已提交
454
	}
455

456
	spin_unlock(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
457 458 459 460 461 462 463 464 465 466
}

static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
				block_t cp_addr, unsigned long long *version)
{
	struct page *cp_page_1, *cp_page_2 = NULL;
	unsigned long blk_size = sbi->blocksize;
	struct f2fs_checkpoint *cp_block;
	unsigned long long cur_version = 0, pre_version = 0;
	size_t crc_offset;
J
Jaegeuk Kim 已提交
467
	__u32 crc = 0;
J
Jaegeuk Kim 已提交
468 469 470 471 472 473 474 475 476 477

	/* Read the 1st cp block in this CP pack */
	cp_page_1 = get_meta_page(sbi, cp_addr);

	/* get the version number */
	cp_block = (struct f2fs_checkpoint *)page_address(cp_page_1);
	crc_offset = le32_to_cpu(cp_block->checksum_offset);
	if (crc_offset >= blk_size)
		goto invalid_cp1;

J
Jaegeuk Kim 已提交
478
	crc = le32_to_cpu(*((__u32 *)((unsigned char *)cp_block + crc_offset)));
J
Jaegeuk Kim 已提交
479 480 481
	if (!f2fs_crc_valid(crc, cp_block, crc_offset))
		goto invalid_cp1;

482
	pre_version = cur_cp_version(cp_block);
J
Jaegeuk Kim 已提交
483 484

	/* Read the 2nd cp block in this CP pack */
485
	cp_addr += le32_to_cpu(cp_block->cp_pack_total_block_count) - 1;
J
Jaegeuk Kim 已提交
486 487 488 489 490 491 492
	cp_page_2 = get_meta_page(sbi, cp_addr);

	cp_block = (struct f2fs_checkpoint *)page_address(cp_page_2);
	crc_offset = le32_to_cpu(cp_block->checksum_offset);
	if (crc_offset >= blk_size)
		goto invalid_cp2;

J
Jaegeuk Kim 已提交
493
	crc = le32_to_cpu(*((__u32 *)((unsigned char *)cp_block + crc_offset)));
J
Jaegeuk Kim 已提交
494 495 496
	if (!f2fs_crc_valid(crc, cp_block, crc_offset))
		goto invalid_cp2;

497
	cur_version = cur_cp_version(cp_block);
J
Jaegeuk Kim 已提交
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530

	if (cur_version == pre_version) {
		*version = cur_version;
		f2fs_put_page(cp_page_2, 1);
		return cp_page_1;
	}
invalid_cp2:
	f2fs_put_page(cp_page_2, 1);
invalid_cp1:
	f2fs_put_page(cp_page_1, 1);
	return NULL;
}

int get_valid_checkpoint(struct f2fs_sb_info *sbi)
{
	struct f2fs_checkpoint *cp_block;
	struct f2fs_super_block *fsb = sbi->raw_super;
	struct page *cp1, *cp2, *cur_page;
	unsigned long blk_size = sbi->blocksize;
	unsigned long long cp1_version = 0, cp2_version = 0;
	unsigned long long cp_start_blk_no;

	sbi->ckpt = kzalloc(blk_size, GFP_KERNEL);
	if (!sbi->ckpt)
		return -ENOMEM;
	/*
	 * Finding out valid cp block involves read both
	 * sets( cp pack1 and cp pack 2)
	 */
	cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
	cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);

	/* The second checkpoint pack should start at the next segment */
531 532
	cp_start_blk_no += ((unsigned long long)1) <<
				le32_to_cpu(fsb->log_blocks_per_seg);
J
Jaegeuk Kim 已提交
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
	cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);

	if (cp1 && cp2) {
		if (ver_after(cp2_version, cp1_version))
			cur_page = cp2;
		else
			cur_page = cp1;
	} else if (cp1) {
		cur_page = cp1;
	} else if (cp2) {
		cur_page = cp2;
	} else {
		goto fail_no_cp;
	}

	cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
	memcpy(sbi->ckpt, cp_block, blk_size);

	f2fs_put_page(cp1, 1);
	f2fs_put_page(cp2, 1);
	return 0;

fail_no_cp:
	kfree(sbi->ckpt);
	return -EINVAL;
}

560
static int __add_dirty_inode(struct inode *inode, struct dir_inode_entry *new)
J
Jaegeuk Kim 已提交
561 562 563 564 565
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct list_head *head = &sbi->dir_inode_list;
	struct list_head *this;

566 567 568
	list_for_each(this, head) {
		struct dir_inode_entry *entry;
		entry = list_entry(this, struct dir_inode_entry, list);
569
		if (unlikely(entry->inode == inode))
570 571 572
			return -EEXIST;
	}
	list_add_tail(&new->list, head);
573
	stat_inc_dirty_dir(sbi);
574 575 576 577 578 579 580 581
	return 0;
}

void set_dirty_dir_page(struct inode *inode, struct page *page)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct dir_inode_entry *new;

J
Jaegeuk Kim 已提交
582 583
	if (!S_ISDIR(inode->i_mode))
		return;
584 585

	new = f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS);
J
Jaegeuk Kim 已提交
586 587 588 589
	new->inode = inode;
	INIT_LIST_HEAD(&new->list);

	spin_lock(&sbi->dir_inode_lock);
590 591
	if (__add_dirty_inode(inode, new))
		kmem_cache_free(inode_entry_slab, new);
J
Jaegeuk Kim 已提交
592 593 594

	inode_inc_dirty_dents(inode);
	SetPagePrivate(page);
595 596 597 598 599 600
	spin_unlock(&sbi->dir_inode_lock);
}

void add_dirty_dir_inode(struct inode *inode)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
601 602 603
	struct dir_inode_entry *new =
			f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS);

604 605
	new->inode = inode;
	INIT_LIST_HEAD(&new->list);
J
Jaegeuk Kim 已提交
606

607 608 609
	spin_lock(&sbi->dir_inode_lock);
	if (__add_dirty_inode(inode, new))
		kmem_cache_free(inode_entry_slab, new);
J
Jaegeuk Kim 已提交
610 611 612 613 614 615
	spin_unlock(&sbi->dir_inode_lock);
}

void remove_dirty_dir_inode(struct inode *inode)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
616 617

	struct list_head *this, *head;
J
Jaegeuk Kim 已提交
618 619 620 621 622

	if (!S_ISDIR(inode->i_mode))
		return;

	spin_lock(&sbi->dir_inode_lock);
623
	if (get_dirty_dents(inode)) {
624 625 626
		spin_unlock(&sbi->dir_inode_lock);
		return;
	}
J
Jaegeuk Kim 已提交
627

628
	head = &sbi->dir_inode_list;
J
Jaegeuk Kim 已提交
629 630 631 632 633 634
	list_for_each(this, head) {
		struct dir_inode_entry *entry;
		entry = list_entry(this, struct dir_inode_entry, list);
		if (entry->inode == inode) {
			list_del(&entry->list);
			kmem_cache_free(inode_entry_slab, entry);
635
			stat_dec_dirty_dir(sbi);
J
Jaegeuk Kim 已提交
636 637 638 639
			break;
		}
	}
	spin_unlock(&sbi->dir_inode_lock);
640 641

	/* Only from the recovery routine */
642 643
	if (is_inode_flag_set(F2FS_I(inode), FI_DELAY_IPUT)) {
		clear_inode_flag(F2FS_I(inode), FI_DELAY_IPUT);
644
		iput(inode);
645
	}
646 647 648 649
}

struct inode *check_dirty_dir_inode(struct f2fs_sb_info *sbi, nid_t ino)
{
650 651

	struct list_head *this, *head;
652 653 654
	struct inode *inode = NULL;

	spin_lock(&sbi->dir_inode_lock);
655 656

	head = &sbi->dir_inode_list;
657 658 659 660 661 662 663 664 665 666
	list_for_each(this, head) {
		struct dir_inode_entry *entry;
		entry = list_entry(this, struct dir_inode_entry, list);
		if (entry->inode->i_ino == ino) {
			inode = entry->inode;
			break;
		}
	}
	spin_unlock(&sbi->dir_inode_lock);
	return inode;
J
Jaegeuk Kim 已提交
667 668 669 670
}

void sync_dirty_dir_inodes(struct f2fs_sb_info *sbi)
{
671
	struct list_head *head;
J
Jaegeuk Kim 已提交
672 673 674 675
	struct dir_inode_entry *entry;
	struct inode *inode;
retry:
	spin_lock(&sbi->dir_inode_lock);
676 677

	head = &sbi->dir_inode_list;
J
Jaegeuk Kim 已提交
678 679 680 681 682 683 684 685
	if (list_empty(head)) {
		spin_unlock(&sbi->dir_inode_lock);
		return;
	}
	entry = list_entry(head->next, struct dir_inode_entry, list);
	inode = igrab(entry->inode);
	spin_unlock(&sbi->dir_inode_lock);
	if (inode) {
686
		filemap_fdatawrite(inode->i_mapping);
J
Jaegeuk Kim 已提交
687 688 689 690 691 692
		iput(inode);
	} else {
		/*
		 * We should submit bio, since it exists several
		 * wribacking dentry pages in the freeing inode.
		 */
J
Jaegeuk Kim 已提交
693
		f2fs_submit_merged_bio(sbi, DATA, WRITE);
J
Jaegeuk Kim 已提交
694 695 696 697
	}
	goto retry;
}

J
Jaegeuk Kim 已提交
698
/*
J
Jaegeuk Kim 已提交
699 700
 * Freeze all the FS-operations for checkpoint.
 */
701
static void block_operations(struct f2fs_sb_info *sbi)
J
Jaegeuk Kim 已提交
702 703 704 705 706 707
{
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = LONG_MAX,
		.for_reclaim = 0,
	};
708 709 710 711
	struct blk_plug plug;

	blk_start_plug(&plug);

712
retry_flush_dents:
713
	f2fs_lock_all(sbi);
J
Jaegeuk Kim 已提交
714 715
	/* write all the dirty dentry pages */
	if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
716
		f2fs_unlock_all(sbi);
717 718
		sync_dirty_dir_inodes(sbi);
		goto retry_flush_dents;
J
Jaegeuk Kim 已提交
719 720 721 722 723 724
	}

	/*
	 * POR: we should ensure that there is no dirty node pages
	 * until finishing nat/sit flush.
	 */
725 726
retry_flush_nodes:
	mutex_lock(&sbi->node_write);
J
Jaegeuk Kim 已提交
727 728

	if (get_pages(sbi, F2FS_DIRTY_NODES)) {
729 730 731
		mutex_unlock(&sbi->node_write);
		sync_node_pages(sbi, 0, &wbc);
		goto retry_flush_nodes;
J
Jaegeuk Kim 已提交
732
	}
733
	blk_finish_plug(&plug);
J
Jaegeuk Kim 已提交
734 735 736 737
}

static void unblock_operations(struct f2fs_sb_info *sbi)
{
738
	mutex_unlock(&sbi->node_write);
739
	f2fs_unlock_all(sbi);
J
Jaegeuk Kim 已提交
740 741
}

742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
static void wait_on_all_pages_writeback(struct f2fs_sb_info *sbi)
{
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);

		if (!get_pages(sbi, F2FS_WRITEBACK))
			break;

		io_schedule();
	}
	finish_wait(&sbi->cp_wait, &wait);
}

J
Jaegeuk Kim 已提交
757 758 759 760 761 762 763
static void do_checkpoint(struct f2fs_sb_info *sbi, bool is_umount)
{
	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
	nid_t last_nid = 0;
	block_t start_blk;
	struct page *cp_page;
	unsigned int data_sum_blocks, orphan_blocks;
J
Jaegeuk Kim 已提交
764
	__u32 crc32 = 0;
J
Jaegeuk Kim 已提交
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
	void *kaddr;
	int i;

	/* Flush all the NAT/SIT pages */
	while (get_pages(sbi, F2FS_DIRTY_META))
		sync_meta_pages(sbi, META, LONG_MAX);

	next_free_nid(sbi, &last_nid);

	/*
	 * modify checkpoint
	 * version number is already updated
	 */
	ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi));
	ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
	ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
	for (i = 0; i < 3; i++) {
		ckpt->cur_node_segno[i] =
			cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE));
		ckpt->cur_node_blkoff[i] =
			cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE));
		ckpt->alloc_type[i + CURSEG_HOT_NODE] =
				curseg_alloc_type(sbi, i + CURSEG_HOT_NODE);
	}
	for (i = 0; i < 3; i++) {
		ckpt->cur_data_segno[i] =
			cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA));
		ckpt->cur_data_blkoff[i] =
			cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA));
		ckpt->alloc_type[i + CURSEG_HOT_DATA] =
				curseg_alloc_type(sbi, i + CURSEG_HOT_DATA);
	}

	ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
	ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
	ckpt->next_free_nid = cpu_to_le32(last_nid);

	/* 2 cp  + n data seg summary + orphan inode blocks */
	data_sum_blocks = npages_for_summary_flush(sbi);
	if (data_sum_blocks < 3)
805
		set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
J
Jaegeuk Kim 已提交
806
	else
807
		clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
J
Jaegeuk Kim 已提交
808 809 810

	orphan_blocks = (sbi->n_orphans + F2FS_ORPHANS_PER_BLOCK - 1)
					/ F2FS_ORPHANS_PER_BLOCK;
811
	ckpt->cp_pack_start_sum = cpu_to_le32(1 + orphan_blocks);
J
Jaegeuk Kim 已提交
812 813

	if (is_umount) {
814 815 816
		set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
		ckpt->cp_pack_total_block_count = cpu_to_le32(2 +
			data_sum_blocks + orphan_blocks + NR_CURSEG_NODE_TYPE);
J
Jaegeuk Kim 已提交
817
	} else {
818 819 820
		clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
		ckpt->cp_pack_total_block_count = cpu_to_le32(2 +
			data_sum_blocks + orphan_blocks);
J
Jaegeuk Kim 已提交
821 822 823
	}

	if (sbi->n_orphans)
824
		set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
J
Jaegeuk Kim 已提交
825
	else
826
		clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
J
Jaegeuk Kim 已提交
827 828 829 830 831 832

	/* update SIT/NAT bitmap */
	get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
	get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));

	crc32 = f2fs_crc32(ckpt, le32_to_cpu(ckpt->checksum_offset));
J
Jaegeuk Kim 已提交
833 834
	*((__le32 *)((unsigned char *)ckpt +
				le32_to_cpu(ckpt->checksum_offset)))
J
Jaegeuk Kim 已提交
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
				= cpu_to_le32(crc32);

	start_blk = __start_cp_addr(sbi);

	/* write out checkpoint buffer at block 0 */
	cp_page = grab_meta_page(sbi, start_blk++);
	kaddr = page_address(cp_page);
	memcpy(kaddr, ckpt, (1 << sbi->log_blocksize));
	set_page_dirty(cp_page);
	f2fs_put_page(cp_page, 1);

	if (sbi->n_orphans) {
		write_orphan_inodes(sbi, start_blk);
		start_blk += orphan_blocks;
	}

	write_data_summaries(sbi, start_blk);
	start_blk += data_sum_blocks;
	if (is_umount) {
		write_node_summaries(sbi, start_blk);
		start_blk += NR_CURSEG_NODE_TYPE;
	}

	/* writeout checkpoint block */
	cp_page = grab_meta_page(sbi, start_blk);
	kaddr = page_address(cp_page);
	memcpy(kaddr, ckpt, (1 << sbi->log_blocksize));
	set_page_dirty(cp_page);
	f2fs_put_page(cp_page, 1);

	/* wait for previous submitted node/meta pages writeback */
866
	wait_on_all_pages_writeback(sbi);
J
Jaegeuk Kim 已提交
867

868
	filemap_fdatawait_range(NODE_MAPPING(sbi), 0, LONG_MAX);
G
Gu Zheng 已提交
869
	filemap_fdatawait_range(META_MAPPING(sbi), 0, LONG_MAX);
J
Jaegeuk Kim 已提交
870 871 872 873 874 875

	/* update user_block_counts */
	sbi->last_valid_block_count = sbi->total_valid_block_count;
	sbi->alloc_valid_block_count = 0;

	/* Here, we only have one bio having CP pack */
876
	sync_meta_pages(sbi, META_FLUSH, LONG_MAX);
J
Jaegeuk Kim 已提交
877

878
	if (unlikely(!is_set_ckpt_flags(ckpt, CP_ERROR_FLAG))) {
879 880 881
		clear_prefree_segments(sbi);
		F2FS_RESET_SB_DIRT(sbi);
	}
J
Jaegeuk Kim 已提交
882 883
}

J
Jaegeuk Kim 已提交
884
/*
J
Jaegeuk Kim 已提交
885 886
 * We guarantee that this checkpoint procedure should not fail.
 */
887
void write_checkpoint(struct f2fs_sb_info *sbi, bool is_umount)
J
Jaegeuk Kim 已提交
888 889 890 891
{
	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
	unsigned long long ckpt_ver;

892 893
	trace_f2fs_write_checkpoint(sbi->sb, is_umount, "start block_ops");

894 895
	mutex_lock(&sbi->cp_mutex);
	block_operations(sbi);
J
Jaegeuk Kim 已提交
896

897 898
	trace_f2fs_write_checkpoint(sbi->sb, is_umount, "finish block_ops");

J
Jaegeuk Kim 已提交
899 900 901
	f2fs_submit_merged_bio(sbi, DATA, WRITE);
	f2fs_submit_merged_bio(sbi, NODE, WRITE);
	f2fs_submit_merged_bio(sbi, META, WRITE);
J
Jaegeuk Kim 已提交
902 903 904 905 906 907

	/*
	 * update checkpoint pack index
	 * Increase the version number so that
	 * SIT entries and seg summaries are written at correct place
	 */
908
	ckpt_ver = cur_cp_version(ckpt);
J
Jaegeuk Kim 已提交
909 910 911 912 913 914 915 916 917 918 919
	ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);

	/* write cached NAT/SIT entries to NAT/SIT area */
	flush_nat_entries(sbi);
	flush_sit_entries(sbi);

	/* unlock all the fs_lock[] in do_checkpoint() */
	do_checkpoint(sbi, is_umount);

	unblock_operations(sbi);
	mutex_unlock(&sbi->cp_mutex);
920

921
	stat_inc_cp_count(sbi->stat_info);
922
	trace_f2fs_write_checkpoint(sbi->sb, is_umount, "finish checkpoint");
J
Jaegeuk Kim 已提交
923 924 925 926
}

void init_orphan_info(struct f2fs_sb_info *sbi)
{
927
	spin_lock_init(&sbi->orphan_inode_lock);
J
Jaegeuk Kim 已提交
928 929
	INIT_LIST_HEAD(&sbi->orphan_inode_list);
	sbi->n_orphans = 0;
930 931 932 933 934 935 936 937
	/*
	 * considering 512 blocks in a segment 8 blocks are needed for cp
	 * and log segment summaries. Remaining blocks are used to keep
	 * orphan entries with the limitation one reserved segment
	 * for cp pack we can have max 1020*504 orphan entries
	 */
	sbi->max_orphans = (sbi->blocks_per_seg - 2 - NR_CURSEG_TYPE)
				* F2FS_ORPHANS_PER_BLOCK;
J
Jaegeuk Kim 已提交
938 939
}

940
int __init create_checkpoint_caches(void)
J
Jaegeuk Kim 已提交
941 942
{
	orphan_entry_slab = f2fs_kmem_cache_create("f2fs_orphan_entry",
943
			sizeof(struct orphan_inode_entry));
944
	if (!orphan_entry_slab)
J
Jaegeuk Kim 已提交
945 946
		return -ENOMEM;
	inode_entry_slab = f2fs_kmem_cache_create("f2fs_dirty_dir_entry",
947
			sizeof(struct dir_inode_entry));
948
	if (!inode_entry_slab) {
J
Jaegeuk Kim 已提交
949 950 951 952 953 954 955 956 957 958 959
		kmem_cache_destroy(orphan_entry_slab);
		return -ENOMEM;
	}
	return 0;
}

void destroy_checkpoint_caches(void)
{
	kmem_cache_destroy(orphan_entry_slab);
	kmem_cache_destroy(inode_entry_slab);
}