node.c 46.7 KB
Newer Older
J
Jaegeuk Kim 已提交
1
/*
J
Jaegeuk Kim 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 * fs/f2fs/node.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/mpage.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/pagevec.h>
#include <linux/swap.h>

#include "f2fs.h"
#include "node.h"
#include "segment.h"
22
#include <trace/events/f2fs.h>
J
Jaegeuk Kim 已提交
23

24 25
#define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock)

J
Jaegeuk Kim 已提交
26 27 28
static struct kmem_cache *nat_entry_slab;
static struct kmem_cache *free_nid_slab;

29
bool available_free_memory(struct f2fs_sb_info *sbi, int type)
30
{
31
	struct f2fs_nm_info *nm_i = NM_I(sbi);
32 33
	struct sysinfo val;
	unsigned long mem_size = 0;
34
	bool res = false;
35 36

	si_meminfo(&val);
37 38 39 40 41 42 43 44 45 46 47 48
	/* give 25%, 25%, 50% memory for each components respectively */
	if (type == FREE_NIDS) {
		mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >> 12;
		res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 2);
	} else if (type == NAT_ENTRIES) {
		mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >> 12;
		res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 2);
	} else if (type == DIRTY_DENTS) {
		mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
		res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 1);
	}
	return res;
49 50
}

J
Jaegeuk Kim 已提交
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
static void clear_node_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
	unsigned int long flags;

	if (PageDirty(page)) {
		spin_lock_irqsave(&mapping->tree_lock, flags);
		radix_tree_tag_clear(&mapping->page_tree,
				page_index(page),
				PAGECACHE_TAG_DIRTY);
		spin_unlock_irqrestore(&mapping->tree_lock, flags);

		clear_page_dirty_for_io(page);
		dec_page_count(sbi, F2FS_DIRTY_NODES);
	}
	ClearPageUptodate(page);
}

static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	pgoff_t index = current_nat_addr(sbi, nid);
	return get_meta_page(sbi, index);
}

static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct page *src_page;
	struct page *dst_page;
	pgoff_t src_off;
	pgoff_t dst_off;
	void *src_addr;
	void *dst_addr;
	struct f2fs_nm_info *nm_i = NM_I(sbi);

	src_off = current_nat_addr(sbi, nid);
	dst_off = next_nat_addr(sbi, src_off);

	/* get current nat block page with lock */
	src_page = get_meta_page(sbi, src_off);

	/* Dirty src_page means that it is already the new target NAT page. */
	if (PageDirty(src_page))
		return src_page;

	dst_page = grab_meta_page(sbi, dst_off);

	src_addr = page_address(src_page);
	dst_addr = page_address(dst_page);
	memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
	set_page_dirty(dst_page);
	f2fs_put_page(src_page, 1);

	set_to_next_nat(nm_i, nid);

	return dst_page;
}

static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)
{
	return radix_tree_lookup(&nm_i->nat_root, n);
}

static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
		nid_t start, unsigned int nr, struct nat_entry **ep)
{
	return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
}

static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
{
	list_del(&e->list);
	radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
	nm_i->nat_cnt--;
	kmem_cache_free(nat_entry_slab, e);
}

int is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
	int is_cp = 1;

	read_lock(&nm_i->nat_tree_lock);
	e = __lookup_nat_cache(nm_i, nid);
	if (e && !e->checkpointed)
		is_cp = 0;
	read_unlock(&nm_i->nat_tree_lock);
	return is_cp;
}

142 143 144 145 146 147 148 149 150 151 152 153 154 155
bool fsync_mark_done(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
	bool fsync_done = false;

	read_lock(&nm_i->nat_tree_lock);
	e = __lookup_nat_cache(nm_i, nid);
	if (e)
		fsync_done = e->fsync_done;
	read_unlock(&nm_i->nat_tree_lock);
	return fsync_done;
}

J
Jaegeuk Kim 已提交
156 157 158 159 160 161 162 163 164 165 166 167 168
static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)
{
	struct nat_entry *new;

	new = kmem_cache_alloc(nat_entry_slab, GFP_ATOMIC);
	if (!new)
		return NULL;
	if (radix_tree_insert(&nm_i->nat_root, nid, new)) {
		kmem_cache_free(nat_entry_slab, new);
		return NULL;
	}
	memset(new, 0, sizeof(struct nat_entry));
	nat_set_nid(new, nid);
169
	new->checkpointed = true;
J
Jaegeuk Kim 已提交
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	list_add_tail(&new->list, &nm_i->nat_entries);
	nm_i->nat_cnt++;
	return new;
}

static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,
						struct f2fs_nat_entry *ne)
{
	struct nat_entry *e;
retry:
	write_lock(&nm_i->nat_tree_lock);
	e = __lookup_nat_cache(nm_i, nid);
	if (!e) {
		e = grab_nat_entry(nm_i, nid);
		if (!e) {
			write_unlock(&nm_i->nat_tree_lock);
			goto retry;
		}
188
		node_info_from_raw_nat(&e->ni, ne);
J
Jaegeuk Kim 已提交
189 190 191 192 193
	}
	write_unlock(&nm_i->nat_tree_lock);
}

static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
194
			block_t new_blkaddr, bool fsync_done)
J
Jaegeuk Kim 已提交
195 196 197 198 199 200 201 202 203 204 205 206 207
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
retry:
	write_lock(&nm_i->nat_tree_lock);
	e = __lookup_nat_cache(nm_i, ni->nid);
	if (!e) {
		e = grab_nat_entry(nm_i, ni->nid);
		if (!e) {
			write_unlock(&nm_i->nat_tree_lock);
			goto retry;
		}
		e->ni = *ni;
208
		f2fs_bug_on(ni->blk_addr == NEW_ADDR);
J
Jaegeuk Kim 已提交
209 210 211 212 213 214 215
	} else if (new_blkaddr == NEW_ADDR) {
		/*
		 * when nid is reallocated,
		 * previous nat entry can be remained in nat cache.
		 * So, reinitialize it with new information.
		 */
		e->ni = *ni;
216
		f2fs_bug_on(ni->blk_addr != NULL_ADDR);
J
Jaegeuk Kim 已提交
217 218 219
	}

	/* sanity check */
220 221
	f2fs_bug_on(nat_get_blkaddr(e) != ni->blk_addr);
	f2fs_bug_on(nat_get_blkaddr(e) == NULL_ADDR &&
J
Jaegeuk Kim 已提交
222
			new_blkaddr == NULL_ADDR);
223
	f2fs_bug_on(nat_get_blkaddr(e) == NEW_ADDR &&
J
Jaegeuk Kim 已提交
224
			new_blkaddr == NEW_ADDR);
225
	f2fs_bug_on(nat_get_blkaddr(e) != NEW_ADDR &&
J
Jaegeuk Kim 已提交
226 227 228 229 230 231 232 233 234 235 236 237
			nat_get_blkaddr(e) != NULL_ADDR &&
			new_blkaddr == NEW_ADDR);

	/* increament version no as node is removed */
	if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
		unsigned char version = nat_get_version(e);
		nat_set_version(e, inc_node_version(version));
	}

	/* change address */
	nat_set_blkaddr(e, new_blkaddr);
	__set_nat_cache_dirty(nm_i, e);
238 239 240 241 242

	/* update fsync_mark if its inode nat entry is still alive */
	e = __lookup_nat_cache(nm_i, ni->ino);
	if (e)
		e->fsync_done = fsync_done;
J
Jaegeuk Kim 已提交
243 244 245
	write_unlock(&nm_i->nat_tree_lock);
}

246
int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
J
Jaegeuk Kim 已提交
247 248 249
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);

250
	if (available_free_memory(sbi, NAT_ENTRIES))
J
Jaegeuk Kim 已提交
251 252 253 254 255 256 257 258 259 260 261 262 263 264
		return 0;

	write_lock(&nm_i->nat_tree_lock);
	while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
		struct nat_entry *ne;
		ne = list_first_entry(&nm_i->nat_entries,
					struct nat_entry, list);
		__del_from_nat_cache(nm_i, ne);
		nr_shrink--;
	}
	write_unlock(&nm_i->nat_tree_lock);
	return nr_shrink;
}

J
Jaegeuk Kim 已提交
265
/*
J
Jaegeuk Kim 已提交
266 267 268 269 270 271 272 273 274 275 276 277 278 279
 * This function returns always success
 */
void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
	struct f2fs_summary_block *sum = curseg->sum_blk;
	nid_t start_nid = START_NID(nid);
	struct f2fs_nat_block *nat_blk;
	struct page *page = NULL;
	struct f2fs_nat_entry ne;
	struct nat_entry *e;
	int i;

280
	memset(&ne, 0, sizeof(struct f2fs_nat_entry));
J
Jaegeuk Kim 已提交
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
	ni->nid = nid;

	/* Check nat cache */
	read_lock(&nm_i->nat_tree_lock);
	e = __lookup_nat_cache(nm_i, nid);
	if (e) {
		ni->ino = nat_get_ino(e);
		ni->blk_addr = nat_get_blkaddr(e);
		ni->version = nat_get_version(e);
	}
	read_unlock(&nm_i->nat_tree_lock);
	if (e)
		return;

	/* Check current segment summary */
	mutex_lock(&curseg->curseg_mutex);
	i = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 0);
	if (i >= 0) {
		ne = nat_in_journal(sum, i);
		node_info_from_raw_nat(ni, &ne);
	}
	mutex_unlock(&curseg->curseg_mutex);
	if (i >= 0)
		goto cache;

	/* Fill node_info from nat page */
	page = get_current_nat_page(sbi, start_nid);
	nat_blk = (struct f2fs_nat_block *)page_address(page);
	ne = nat_blk->entries[nid - start_nid];
	node_info_from_raw_nat(ni, &ne);
	f2fs_put_page(page, 1);
cache:
	/* cache nat entry */
	cache_nat_entry(NM_I(sbi), nid, &ne);
}

J
Jaegeuk Kim 已提交
317
/*
J
Jaegeuk Kim 已提交
318 319 320
 * The maximum depth is four.
 * Offset[0] will have raw inode offset.
 */
321 322
static int get_node_path(struct f2fs_inode_info *fi, long block,
				int offset[4], unsigned int noffset[4])
J
Jaegeuk Kim 已提交
323
{
324
	const long direct_index = ADDRS_PER_INODE(fi);
J
Jaegeuk Kim 已提交
325 326 327 328 329 330 331 332 333 334
	const long direct_blks = ADDRS_PER_BLOCK;
	const long dptrs_per_blk = NIDS_PER_BLOCK;
	const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
	const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
	int n = 0;
	int level = 0;

	noffset[0] = 0;

	if (block < direct_index) {
335
		offset[n] = block;
J
Jaegeuk Kim 已提交
336 337 338 339 340 341
		goto got;
	}
	block -= direct_index;
	if (block < direct_blks) {
		offset[n++] = NODE_DIR1_BLOCK;
		noffset[n] = 1;
342
		offset[n] = block;
J
Jaegeuk Kim 已提交
343 344 345 346 347 348 349
		level = 1;
		goto got;
	}
	block -= direct_blks;
	if (block < direct_blks) {
		offset[n++] = NODE_DIR2_BLOCK;
		noffset[n] = 2;
350
		offset[n] = block;
J
Jaegeuk Kim 已提交
351 352 353 354 355 356 357 358 359
		level = 1;
		goto got;
	}
	block -= direct_blks;
	if (block < indirect_blks) {
		offset[n++] = NODE_IND1_BLOCK;
		noffset[n] = 3;
		offset[n++] = block / direct_blks;
		noffset[n] = 4 + offset[n - 1];
360
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
361 362 363 364 365 366 367 368 369
		level = 2;
		goto got;
	}
	block -= indirect_blks;
	if (block < indirect_blks) {
		offset[n++] = NODE_IND2_BLOCK;
		noffset[n] = 4 + dptrs_per_blk;
		offset[n++] = block / direct_blks;
		noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
370
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384
		level = 2;
		goto got;
	}
	block -= indirect_blks;
	if (block < dindirect_blks) {
		offset[n++] = NODE_DIND_BLOCK;
		noffset[n] = 5 + (dptrs_per_blk * 2);
		offset[n++] = block / indirect_blks;
		noffset[n] = 6 + (dptrs_per_blk * 2) +
			      offset[n - 1] * (dptrs_per_blk + 1);
		offset[n++] = (block / direct_blks) % dptrs_per_blk;
		noffset[n] = 7 + (dptrs_per_blk * 2) +
			      offset[n - 2] * (dptrs_per_blk + 1) +
			      offset[n - 1];
385
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
386 387 388 389 390 391 392 393 394 395 396
		level = 3;
		goto got;
	} else {
		BUG();
	}
got:
	return level;
}

/*
 * Caller should call f2fs_put_dnode(dn).
C
Chao Yu 已提交
397 398
 * Also, it should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op() only if ro is not set RDONLY_NODE.
399
 * In the case of RDONLY_NODE, we don't need to care about mutex.
J
Jaegeuk Kim 已提交
400
 */
401
int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
J
Jaegeuk Kim 已提交
402 403 404 405 406 407 408 409 410 411
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct page *npage[4];
	struct page *parent;
	int offset[4];
	unsigned int noffset[4];
	nid_t nids[4];
	int level, i;
	int err = 0;

412
	level = get_node_path(F2FS_I(dn->inode), index, offset, noffset);
J
Jaegeuk Kim 已提交
413 414

	nids[0] = dn->inode->i_ino;
415
	npage[0] = dn->inode_page;
J
Jaegeuk Kim 已提交
416

417 418 419 420 421
	if (!npage[0]) {
		npage[0] = get_node_page(sbi, nids[0]);
		if (IS_ERR(npage[0]))
			return PTR_ERR(npage[0]);
	}
J
Jaegeuk Kim 已提交
422
	parent = npage[0];
423 424
	if (level != 0)
		nids[1] = get_nid(parent, offset[0], true);
J
Jaegeuk Kim 已提交
425 426 427 428 429 430 431
	dn->inode_page = npage[0];
	dn->inode_page_locked = true;

	/* get indirect or direct nodes */
	for (i = 1; i <= level; i++) {
		bool done = false;

432
		if (!nids[i] && mode == ALLOC_NODE) {
J
Jaegeuk Kim 已提交
433 434 435 436 437 438 439
			/* alloc new node */
			if (!alloc_nid(sbi, &(nids[i]))) {
				err = -ENOSPC;
				goto release_pages;
			}

			dn->nid = nids[i];
440
			npage[i] = new_node_page(dn, noffset[i], NULL);
J
Jaegeuk Kim 已提交
441 442 443 444 445 446 447 448 449
			if (IS_ERR(npage[i])) {
				alloc_nid_failed(sbi, nids[i]);
				err = PTR_ERR(npage[i]);
				goto release_pages;
			}

			set_nid(parent, offset[i - 1], nids[i], i == 1);
			alloc_nid_done(sbi, nids[i]);
			done = true;
450
		} else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
J
Jaegeuk Kim 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
			npage[i] = get_node_page_ra(parent, offset[i - 1]);
			if (IS_ERR(npage[i])) {
				err = PTR_ERR(npage[i]);
				goto release_pages;
			}
			done = true;
		}
		if (i == 1) {
			dn->inode_page_locked = false;
			unlock_page(parent);
		} else {
			f2fs_put_page(parent, 1);
		}

		if (!done) {
			npage[i] = get_node_page(sbi, nids[i]);
			if (IS_ERR(npage[i])) {
				err = PTR_ERR(npage[i]);
				f2fs_put_page(npage[0], 0);
				goto release_out;
			}
		}
		if (i < level) {
			parent = npage[i];
			nids[i + 1] = get_nid(parent, offset[i], false);
		}
	}
	dn->nid = nids[level];
	dn->ofs_in_node = offset[level];
	dn->node_page = npage[level];
	dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
	return 0;

release_pages:
	f2fs_put_page(parent, 1);
	if (i > 1)
		f2fs_put_page(npage[0], 0);
release_out:
	dn->inode_page = NULL;
	dn->node_page = NULL;
	return err;
}

static void truncate_node(struct dnode_of_data *dn)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct node_info ni;

	get_node_info(sbi, dn->nid, &ni);
500
	if (dn->inode->i_blocks == 0) {
501
		f2fs_bug_on(ni.blk_addr != NULL_ADDR);
502 503
		goto invalidate;
	}
504
	f2fs_bug_on(ni.blk_addr == NULL_ADDR);
J
Jaegeuk Kim 已提交
505 506

	/* Deallocate node address */
507
	invalidate_blocks(sbi, ni.blk_addr);
508
	dec_valid_node_count(sbi, dn->inode);
509
	set_node_addr(sbi, &ni, NULL_ADDR, false);
J
Jaegeuk Kim 已提交
510 511 512 513 514 515 516

	if (dn->nid == dn->inode->i_ino) {
		remove_orphan_inode(sbi, dn->nid);
		dec_valid_inode_count(sbi);
	} else {
		sync_inode_page(dn);
	}
517
invalidate:
J
Jaegeuk Kim 已提交
518 519 520 521
	clear_node_page_dirty(dn->node_page);
	F2FS_SET_SB_DIRT(sbi);

	f2fs_put_page(dn->node_page, 1);
522 523 524 525

	invalidate_mapping_pages(NODE_MAPPING(sbi),
			dn->node_page->index, dn->node_page->index);

J
Jaegeuk Kim 已提交
526
	dn->node_page = NULL;
527
	trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
J
Jaegeuk Kim 已提交
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
}

static int truncate_dnode(struct dnode_of_data *dn)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct page *page;

	if (dn->nid == 0)
		return 1;

	/* get direct node */
	page = get_node_page(sbi, dn->nid);
	if (IS_ERR(page) && PTR_ERR(page) == -ENOENT)
		return 1;
	else if (IS_ERR(page))
		return PTR_ERR(page);

	/* Make dnode_of_data for parameter */
	dn->node_page = page;
	dn->ofs_in_node = 0;
	truncate_data_blocks(dn);
	truncate_node(dn);
	return 1;
}

static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
						int ofs, int depth)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct dnode_of_data rdn = *dn;
	struct page *page;
	struct f2fs_node *rn;
	nid_t child_nid;
	unsigned int child_nofs;
	int freed = 0;
	int i, ret;

	if (dn->nid == 0)
		return NIDS_PER_BLOCK + 1;

568 569
	trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);

J
Jaegeuk Kim 已提交
570
	page = get_node_page(sbi, dn->nid);
571 572
	if (IS_ERR(page)) {
		trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page));
J
Jaegeuk Kim 已提交
573
		return PTR_ERR(page);
574
	}
J
Jaegeuk Kim 已提交
575

576
	rn = F2FS_NODE(page);
J
Jaegeuk Kim 已提交
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	if (depth < 3) {
		for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
			child_nid = le32_to_cpu(rn->in.nid[i]);
			if (child_nid == 0)
				continue;
			rdn.nid = child_nid;
			ret = truncate_dnode(&rdn);
			if (ret < 0)
				goto out_err;
			set_nid(page, i, 0, false);
		}
	} else {
		child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
		for (i = ofs; i < NIDS_PER_BLOCK; i++) {
			child_nid = le32_to_cpu(rn->in.nid[i]);
			if (child_nid == 0) {
				child_nofs += NIDS_PER_BLOCK + 1;
				continue;
			}
			rdn.nid = child_nid;
			ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
			if (ret == (NIDS_PER_BLOCK + 1)) {
				set_nid(page, i, 0, false);
				child_nofs += ret;
			} else if (ret < 0 && ret != -ENOENT) {
				goto out_err;
			}
		}
		freed = child_nofs;
	}

	if (!ofs) {
		/* remove current indirect node */
		dn->node_page = page;
		truncate_node(dn);
		freed++;
	} else {
		f2fs_put_page(page, 1);
	}
616
	trace_f2fs_truncate_nodes_exit(dn->inode, freed);
J
Jaegeuk Kim 已提交
617 618 619 620
	return freed;

out_err:
	f2fs_put_page(page, 1);
621
	trace_f2fs_truncate_nodes_exit(dn->inode, ret);
J
Jaegeuk Kim 已提交
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
	return ret;
}

static int truncate_partial_nodes(struct dnode_of_data *dn,
			struct f2fs_inode *ri, int *offset, int depth)
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct page *pages[2];
	nid_t nid[3];
	nid_t child_nid;
	int err = 0;
	int i;
	int idx = depth - 2;

	nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
	if (!nid[0])
		return 0;

	/* get indirect nodes in the path */
641
	for (i = 0; i < idx + 1; i++) {
J
Jaegeuk Kim 已提交
642 643 644 645
		/* refernece count'll be increased */
		pages[i] = get_node_page(sbi, nid[i]);
		if (IS_ERR(pages[i])) {
			err = PTR_ERR(pages[i]);
646
			idx = i - 1;
J
Jaegeuk Kim 已提交
647 648 649 650 651 652
			goto fail;
		}
		nid[i + 1] = get_nid(pages[i], offset[i + 1], false);
	}

	/* free direct nodes linked to a partial indirect node */
653
	for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
J
Jaegeuk Kim 已提交
654 655 656 657 658 659 660 661 662 663
		child_nid = get_nid(pages[idx], i, false);
		if (!child_nid)
			continue;
		dn->nid = child_nid;
		err = truncate_dnode(dn);
		if (err < 0)
			goto fail;
		set_nid(pages[idx], i, 0, false);
	}

664
	if (offset[idx + 1] == 0) {
J
Jaegeuk Kim 已提交
665 666 667 668 669 670 671
		dn->node_page = pages[idx];
		dn->nid = nid[idx];
		truncate_node(dn);
	} else {
		f2fs_put_page(pages[idx], 1);
	}
	offset[idx]++;
672 673
	offset[idx + 1] = 0;
	idx--;
J
Jaegeuk Kim 已提交
674
fail:
675
	for (i = idx; i >= 0; i--)
J
Jaegeuk Kim 已提交
676
		f2fs_put_page(pages[i], 1);
677 678 679

	trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);

J
Jaegeuk Kim 已提交
680 681 682
	return err;
}

J
Jaegeuk Kim 已提交
683
/*
J
Jaegeuk Kim 已提交
684 685 686 687 688 689 690
 * All the block addresses of data and nodes should be nullified.
 */
int truncate_inode_blocks(struct inode *inode, pgoff_t from)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	int err = 0, cont = 1;
	int level, offset[4], noffset[4];
J
Jaegeuk Kim 已提交
691
	unsigned int nofs = 0;
692
	struct f2fs_inode *ri;
J
Jaegeuk Kim 已提交
693 694 695
	struct dnode_of_data dn;
	struct page *page;

696 697
	trace_f2fs_truncate_inode_blocks_enter(inode, from);

698
	level = get_node_path(F2FS_I(inode), from, offset, noffset);
699
restart:
J
Jaegeuk Kim 已提交
700
	page = get_node_page(sbi, inode->i_ino);
701 702
	if (IS_ERR(page)) {
		trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page));
J
Jaegeuk Kim 已提交
703
		return PTR_ERR(page);
704
	}
J
Jaegeuk Kim 已提交
705 706 707 708

	set_new_dnode(&dn, inode, page, NULL, 0);
	unlock_page(page);

709
	ri = F2FS_INODE(page);
J
Jaegeuk Kim 已提交
710 711 712 713 714 715 716 717 718
	switch (level) {
	case 0:
	case 1:
		nofs = noffset[1];
		break;
	case 2:
		nofs = noffset[1];
		if (!offset[level - 1])
			goto skip_partial;
719
		err = truncate_partial_nodes(&dn, ri, offset, level);
J
Jaegeuk Kim 已提交
720 721 722 723 724 725 726 727
		if (err < 0 && err != -ENOENT)
			goto fail;
		nofs += 1 + NIDS_PER_BLOCK;
		break;
	case 3:
		nofs = 5 + 2 * NIDS_PER_BLOCK;
		if (!offset[level - 1])
			goto skip_partial;
728
		err = truncate_partial_nodes(&dn, ri, offset, level);
J
Jaegeuk Kim 已提交
729 730 731 732 733 734 735 736 737
		if (err < 0 && err != -ENOENT)
			goto fail;
		break;
	default:
		BUG();
	}

skip_partial:
	while (cont) {
738
		dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
J
Jaegeuk Kim 已提交
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
		switch (offset[0]) {
		case NODE_DIR1_BLOCK:
		case NODE_DIR2_BLOCK:
			err = truncate_dnode(&dn);
			break;

		case NODE_IND1_BLOCK:
		case NODE_IND2_BLOCK:
			err = truncate_nodes(&dn, nofs, offset[1], 2);
			break;

		case NODE_DIND_BLOCK:
			err = truncate_nodes(&dn, nofs, offset[1], 3);
			cont = 0;
			break;

		default:
			BUG();
		}
		if (err < 0 && err != -ENOENT)
			goto fail;
		if (offset[1] == 0 &&
761
				ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) {
J
Jaegeuk Kim 已提交
762
			lock_page(page);
763
			if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
764 765 766
				f2fs_put_page(page, 1);
				goto restart;
			}
767
			f2fs_wait_on_page_writeback(page, NODE);
768
			ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0;
J
Jaegeuk Kim 已提交
769 770 771 772 773 774 775 776 777
			set_page_dirty(page);
			unlock_page(page);
		}
		offset[1] = 0;
		offset[0]++;
		nofs += err;
	}
fail:
	f2fs_put_page(page, 0);
778
	trace_f2fs_truncate_inode_blocks_exit(inode, err);
J
Jaegeuk Kim 已提交
779 780 781
	return err > 0 ? 0 : err;
}

782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
int truncate_xattr_node(struct inode *inode, struct page *page)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	nid_t nid = F2FS_I(inode)->i_xattr_nid;
	struct dnode_of_data dn;
	struct page *npage;

	if (!nid)
		return 0;

	npage = get_node_page(sbi, nid);
	if (IS_ERR(npage))
		return PTR_ERR(npage);

	F2FS_I(inode)->i_xattr_nid = 0;
J
Jaegeuk Kim 已提交
797 798 799 800

	/* need to do checkpoint during fsync */
	F2FS_I(inode)->xattr_ver = cur_cp_version(F2FS_CKPT(sbi));

801 802 803
	set_new_dnode(&dn, inode, page, npage, nid);

	if (page)
804
		dn.inode_page_locked = true;
805 806 807 808
	truncate_node(&dn);
	return 0;
}

809
/*
C
Chao Yu 已提交
810 811
 * Caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
812
 */
G
Gu Zheng 已提交
813
void remove_inode_page(struct inode *inode)
J
Jaegeuk Kim 已提交
814 815 816 817 818 819 820
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	struct page *page;
	nid_t ino = inode->i_ino;
	struct dnode_of_data dn;

	page = get_node_page(sbi, ino);
821
	if (IS_ERR(page))
G
Gu Zheng 已提交
822
		return;
J
Jaegeuk Kim 已提交
823

G
Gu Zheng 已提交
824
	if (truncate_xattr_node(inode, page)) {
825
		f2fs_put_page(page, 1);
G
Gu Zheng 已提交
826
		return;
J
Jaegeuk Kim 已提交
827
	}
828
	/* 0 is possible, after f2fs_new_inode() is failed */
829
	f2fs_bug_on(inode->i_blocks != 0 && inode->i_blocks != 1);
830 831
	set_new_dnode(&dn, inode, page, page, ino);
	truncate_node(&dn);
J
Jaegeuk Kim 已提交
832 833
}

834
struct page *new_inode_page(struct inode *inode, const struct qstr *name)
J
Jaegeuk Kim 已提交
835 836 837 838 839
{
	struct dnode_of_data dn;

	/* allocate inode page for new inode */
	set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
840 841

	/* caller should f2fs_put_page(page, 1); */
842
	return new_node_page(&dn, 0, NULL);
J
Jaegeuk Kim 已提交
843 844
}

845 846
struct page *new_node_page(struct dnode_of_data *dn,
				unsigned int ofs, struct page *ipage)
J
Jaegeuk Kim 已提交
847 848 849 850 851 852
{
	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
	struct node_info old_ni, new_ni;
	struct page *page;
	int err;

853
	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
J
Jaegeuk Kim 已提交
854 855
		return ERR_PTR(-EPERM);

856
	page = grab_cache_page(NODE_MAPPING(sbi), dn->nid);
J
Jaegeuk Kim 已提交
857 858 859
	if (!page)
		return ERR_PTR(-ENOMEM);

860
	if (unlikely(!inc_valid_node_count(sbi, dn->inode))) {
861 862 863
		err = -ENOSPC;
		goto fail;
	}
J
Jaegeuk Kim 已提交
864

865
	get_node_info(sbi, dn->nid, &old_ni);
J
Jaegeuk Kim 已提交
866 867

	/* Reinitialize old_ni with new node page */
868
	f2fs_bug_on(old_ni.blk_addr != NULL_ADDR);
J
Jaegeuk Kim 已提交
869 870
	new_ni = old_ni;
	new_ni.ino = dn->inode->i_ino;
871
	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
872

873
	f2fs_wait_on_page_writeback(page, NODE);
874
	fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true);
875
	set_cold_node(dn->inode, page);
876 877
	SetPageUptodate(page);
	set_page_dirty(page);
J
Jaegeuk Kim 已提交
878

879
	if (f2fs_has_xattr_block(ofs))
880 881
		F2FS_I(dn->inode)->i_xattr_nid = dn->nid;

J
Jaegeuk Kim 已提交
882
	dn->node_page = page;
883 884 885 886
	if (ipage)
		update_inode(dn->inode, ipage);
	else
		sync_inode_page(dn);
J
Jaegeuk Kim 已提交
887 888 889 890 891 892
	if (ofs == 0)
		inc_valid_inode_count(sbi);

	return page;

fail:
893
	clear_node_page_dirty(page);
J
Jaegeuk Kim 已提交
894 895 896 897
	f2fs_put_page(page, 1);
	return ERR_PTR(err);
}

898 899 900 901 902 903
/*
 * Caller should do after getting the following values.
 * 0: f2fs_put_page(page, 0)
 * LOCKED_PAGE: f2fs_put_page(page, 1)
 * error: nothing
 */
904
static int read_node_page(struct page *page, int rw)
J
Jaegeuk Kim 已提交
905 906 907 908 909 910
{
	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
	struct node_info ni;

	get_node_info(sbi, page->index, &ni);

911
	if (unlikely(ni.blk_addr == NULL_ADDR)) {
912
		f2fs_put_page(page, 1);
J
Jaegeuk Kim 已提交
913
		return -ENOENT;
914 915
	}

916 917
	if (PageUptodate(page))
		return LOCKED_PAGE;
918

919
	return f2fs_submit_page_bio(sbi, page, ni.blk_addr, rw);
J
Jaegeuk Kim 已提交
920 921
}

J
Jaegeuk Kim 已提交
922
/*
J
Jaegeuk Kim 已提交
923 924 925 926 927
 * Readahead a node page
 */
void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct page *apage;
928
	int err;
J
Jaegeuk Kim 已提交
929

930
	apage = find_get_page(NODE_MAPPING(sbi), nid);
931 932 933 934
	if (apage && PageUptodate(apage)) {
		f2fs_put_page(apage, 0);
		return;
	}
J
Jaegeuk Kim 已提交
935 936
	f2fs_put_page(apage, 0);

937
	apage = grab_cache_page(NODE_MAPPING(sbi), nid);
J
Jaegeuk Kim 已提交
938 939 940
	if (!apage)
		return;

941 942
	err = read_node_page(apage, READA);
	if (err == 0)
943
		f2fs_put_page(apage, 0);
944 945
	else if (err == LOCKED_PAGE)
		f2fs_put_page(apage, 1);
J
Jaegeuk Kim 已提交
946 947 948 949
}

struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid)
{
950 951
	struct page *page;
	int err;
952
repeat:
953
	page = grab_cache_page(NODE_MAPPING(sbi), nid);
J
Jaegeuk Kim 已提交
954 955 956 957
	if (!page)
		return ERR_PTR(-ENOMEM);

	err = read_node_page(page, READ_SYNC);
958
	if (err < 0)
J
Jaegeuk Kim 已提交
959
		return ERR_PTR(err);
960 961
	else if (err == LOCKED_PAGE)
		goto got_it;
J
Jaegeuk Kim 已提交
962

963
	lock_page(page);
964
	if (unlikely(!PageUptodate(page) || nid != nid_of_node(page))) {
965 966 967
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
	}
968
	if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
969 970 971
		f2fs_put_page(page, 1);
		goto repeat;
	}
972
got_it:
J
Jaegeuk Kim 已提交
973 974 975 976
	mark_page_accessed(page);
	return page;
}

J
Jaegeuk Kim 已提交
977
/*
J
Jaegeuk Kim 已提交
978 979 980 981 982 983
 * Return a locked page for the desired node page.
 * And, readahead MAX_RA_NODE number of node pages.
 */
struct page *get_node_page_ra(struct page *parent, int start)
{
	struct f2fs_sb_info *sbi = F2FS_SB(parent->mapping->host->i_sb);
984
	struct blk_plug plug;
J
Jaegeuk Kim 已提交
985
	struct page *page;
986 987
	int err, i, end;
	nid_t nid;
J
Jaegeuk Kim 已提交
988 989 990 991 992

	/* First, try getting the desired direct node. */
	nid = get_nid(parent, start, false);
	if (!nid)
		return ERR_PTR(-ENOENT);
993
repeat:
994
	page = grab_cache_page(NODE_MAPPING(sbi), nid);
J
Jaegeuk Kim 已提交
995 996 997
	if (!page)
		return ERR_PTR(-ENOMEM);

998
	err = read_node_page(page, READ_SYNC);
999
	if (err < 0)
J
Jaegeuk Kim 已提交
1000
		return ERR_PTR(err);
1001 1002
	else if (err == LOCKED_PAGE)
		goto page_hit;
J
Jaegeuk Kim 已提交
1003

1004 1005
	blk_start_plug(&plug);

J
Jaegeuk Kim 已提交
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
	/* Then, try readahead for siblings of the desired node */
	end = start + MAX_RA_NODE;
	end = min(end, NIDS_PER_BLOCK);
	for (i = start + 1; i < end; i++) {
		nid = get_nid(parent, i, false);
		if (!nid)
			continue;
		ra_node_page(sbi, nid);
	}

1016 1017
	blk_finish_plug(&plug);

J
Jaegeuk Kim 已提交
1018
	lock_page(page);
1019
	if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
1020 1021 1022
		f2fs_put_page(page, 1);
		goto repeat;
	}
1023
page_hit:
1024
	if (unlikely(!PageUptodate(page))) {
J
Jaegeuk Kim 已提交
1025 1026 1027
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
	}
1028
	mark_page_accessed(page);
J
Jaegeuk Kim 已提交
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
	return page;
}

void sync_inode_page(struct dnode_of_data *dn)
{
	if (IS_INODE(dn->node_page) || dn->inode_page == dn->node_page) {
		update_inode(dn->inode, dn->node_page);
	} else if (dn->inode_page) {
		if (!dn->inode_page_locked)
			lock_page(dn->inode_page);
		update_inode(dn->inode, dn->inode_page);
		if (!dn->inode_page_locked)
			unlock_page(dn->inode_page);
	} else {
1043
		update_inode_page(dn->inode);
J
Jaegeuk Kim 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
	}
}

int sync_node_pages(struct f2fs_sb_info *sbi, nid_t ino,
					struct writeback_control *wbc)
{
	pgoff_t index, end;
	struct pagevec pvec;
	int step = ino ? 2 : 0;
	int nwritten = 0, wrote = 0;

	pagevec_init(&pvec, 0);

next_step:
	index = 0;
	end = LONG_MAX;

	while (index <= end) {
		int i, nr_pages;
1063
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
J
Jaegeuk Kim 已提交
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
				PAGECACHE_TAG_DIRTY,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/*
			 * flushing sequence with step:
			 * 0. indirect nodes
			 * 1. dentry dnodes
			 * 2. file dnodes
			 */
			if (step == 0 && IS_DNODE(page))
				continue;
			if (step == 1 && (!IS_DNODE(page) ||
						is_cold_node(page)))
				continue;
			if (step == 2 && (!IS_DNODE(page) ||
						!is_cold_node(page)))
				continue;

			/*
			 * If an fsync mode,
			 * we should not skip writing node pages.
			 */
			if (ino && ino_of_node(page) == ino)
				lock_page(page);
			else if (!trylock_page(page))
				continue;

1096
			if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
J
Jaegeuk Kim 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
continue_unlock:
				unlock_page(page);
				continue;
			}
			if (ino && ino_of_node(page) != ino)
				goto continue_unlock;

			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;

			/* called by fsync() */
			if (ino && IS_DNODE(page)) {
				int mark = !is_checkpointed_node(sbi, ino);
				set_fsync_mark(page, 1);
				if (IS_INODE(page))
					set_dentry_mark(page, mark);
				nwritten++;
			} else {
				set_fsync_mark(page, 0);
				set_dentry_mark(page, 0);
			}
1123
			NODE_MAPPING(sbi)->a_ops->writepage(page, wbc);
J
Jaegeuk Kim 已提交
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
			wrote++;

			if (--wbc->nr_to_write == 0)
				break;
		}
		pagevec_release(&pvec);
		cond_resched();

		if (wbc->nr_to_write == 0) {
			step = 2;
			break;
		}
	}

	if (step < 2) {
		step++;
		goto next_step;
	}

	if (wrote)
J
Jaegeuk Kim 已提交
1144
		f2fs_submit_merged_bio(sbi, NODE, WRITE);
J
Jaegeuk Kim 已提交
1145 1146 1147
	return nwritten;
}

1148 1149 1150 1151 1152 1153 1154
int wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, nid_t ino)
{
	pgoff_t index = 0, end = LONG_MAX;
	struct pagevec pvec;
	int ret2 = 0, ret = 0;

	pagevec_init(&pvec, 0);
1155 1156 1157 1158 1159 1160 1161 1162

	while (index <= end) {
		int i, nr_pages;
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
				PAGECACHE_TAG_WRITEBACK,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;
1163 1164 1165 1166 1167

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
1168
			if (unlikely(page->index > end))
1169 1170
				continue;

1171
			if (ino && ino_of_node(page) == ino) {
1172
				f2fs_wait_on_page_writeback(page, NODE);
1173 1174 1175
				if (TestClearPageError(page))
					ret = -EIO;
			}
1176 1177 1178 1179 1180
		}
		pagevec_release(&pvec);
		cond_resched();
	}

1181
	if (unlikely(test_and_clear_bit(AS_ENOSPC, &NODE_MAPPING(sbi)->flags)))
1182
		ret2 = -ENOSPC;
1183
	if (unlikely(test_and_clear_bit(AS_EIO, &NODE_MAPPING(sbi)->flags)))
1184 1185 1186 1187 1188 1189
		ret2 = -EIO;
	if (!ret)
		ret = ret2;
	return ret;
}

J
Jaegeuk Kim 已提交
1190 1191 1192 1193 1194 1195 1196
static int f2fs_write_node_page(struct page *page,
				struct writeback_control *wbc)
{
	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
	nid_t nid;
	block_t new_addr;
	struct node_info ni;
1197 1198
	struct f2fs_io_info fio = {
		.type = NODE,
C
Chris Fries 已提交
1199
		.rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1200
	};
J
Jaegeuk Kim 已提交
1201

1202
	if (unlikely(sbi->por_doing))
1203 1204
		goto redirty_out;

1205
	f2fs_wait_on_page_writeback(page, NODE);
J
Jaegeuk Kim 已提交
1206 1207 1208

	/* get old block addr of this node page */
	nid = nid_of_node(page);
1209
	f2fs_bug_on(page->index != nid);
J
Jaegeuk Kim 已提交
1210 1211 1212 1213

	get_node_info(sbi, nid, &ni);

	/* This page is already truncated */
1214
	if (unlikely(ni.blk_addr == NULL_ADDR)) {
1215 1216 1217 1218
		dec_page_count(sbi, F2FS_DIRTY_NODES);
		unlock_page(page);
		return 0;
	}
J
Jaegeuk Kim 已提交
1219

1220 1221
	if (wbc->for_reclaim)
		goto redirty_out;
1222

1223
	mutex_lock(&sbi->node_write);
J
Jaegeuk Kim 已提交
1224
	set_page_writeback(page);
1225
	write_node_page(sbi, page, &fio, nid, ni.blk_addr, &new_addr);
1226
	set_node_addr(sbi, &ni, new_addr, is_fsync_dnode(page));
J
Jaegeuk Kim 已提交
1227
	dec_page_count(sbi, F2FS_DIRTY_NODES);
1228
	mutex_unlock(&sbi->node_write);
J
Jaegeuk Kim 已提交
1229 1230
	unlock_page(page);
	return 0;
1231 1232

redirty_out:
1233
	redirty_page_for_writepage(wbc, page);
1234
	return AOP_WRITEPAGE_ACTIVATE;
J
Jaegeuk Kim 已提交
1235 1236 1237 1238 1239 1240
}

static int f2fs_write_node_pages(struct address_space *mapping,
			    struct writeback_control *wbc)
{
	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
1241
	long diff;
J
Jaegeuk Kim 已提交
1242

1243 1244
	/* balancing f2fs's metadata in background */
	f2fs_balance_fs_bg(sbi);
J
Jaegeuk Kim 已提交
1245

1246
	/* collect a number of dirty node pages and write together */
1247
	if (get_pages(sbi, F2FS_DIRTY_NODES) < nr_pages_to_skip(sbi, NODE))
1248
		goto skip_write;
1249

1250
	diff = nr_pages_to_write(sbi, NODE, wbc);
1251
	wbc->sync_mode = WB_SYNC_NONE;
J
Jaegeuk Kim 已提交
1252
	sync_node_pages(sbi, 0, wbc);
1253
	wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
J
Jaegeuk Kim 已提交
1254
	return 0;
1255 1256 1257 1258

skip_write:
	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES);
	return 0;
J
Jaegeuk Kim 已提交
1259 1260 1261 1262 1263 1264 1265
}

static int f2fs_set_node_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);

1266 1267
	trace_f2fs_set_page_dirty(page, NODE);

J
Jaegeuk Kim 已提交
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
	SetPageUptodate(page);
	if (!PageDirty(page)) {
		__set_page_dirty_nobuffers(page);
		inc_page_count(sbi, F2FS_DIRTY_NODES);
		SetPagePrivate(page);
		return 1;
	}
	return 0;
}

1278 1279
static void f2fs_invalidate_node_page(struct page *page, unsigned int offset,
				      unsigned int length)
J
Jaegeuk Kim 已提交
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
{
	struct inode *inode = page->mapping->host;
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	if (PageDirty(page))
		dec_page_count(sbi, F2FS_DIRTY_NODES);
	ClearPagePrivate(page);
}

static int f2fs_release_node_page(struct page *page, gfp_t wait)
{
	ClearPagePrivate(page);
1291
	return 1;
J
Jaegeuk Kim 已提交
1292 1293
}

J
Jaegeuk Kim 已提交
1294
/*
J
Jaegeuk Kim 已提交
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
 * Structure of the f2fs node operations
 */
const struct address_space_operations f2fs_node_aops = {
	.writepage	= f2fs_write_node_page,
	.writepages	= f2fs_write_node_pages,
	.set_page_dirty	= f2fs_set_node_page_dirty,
	.invalidatepage	= f2fs_invalidate_node_page,
	.releasepage	= f2fs_release_node_page,
};

1305 1306
static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
						nid_t n)
J
Jaegeuk Kim 已提交
1307
{
1308
	return radix_tree_lookup(&nm_i->free_nid_root, n);
J
Jaegeuk Kim 已提交
1309 1310
}

1311 1312
static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,
						struct free_nid *i)
J
Jaegeuk Kim 已提交
1313 1314
{
	list_del(&i->list);
1315
	radix_tree_delete(&nm_i->free_nid_root, i->nid);
J
Jaegeuk Kim 已提交
1316 1317
}

1318
static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
J
Jaegeuk Kim 已提交
1319
{
1320
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
1321
	struct free_nid *i;
1322 1323
	struct nat_entry *ne;
	bool allocated = false;
J
Jaegeuk Kim 已提交
1324

1325
	if (!available_free_memory(sbi, FREE_NIDS))
H
Haicheng Li 已提交
1326
		return -1;
1327 1328

	/* 0 nid should not be used */
1329
	if (unlikely(nid == 0))
1330
		return 0;
1331

1332 1333 1334 1335
	if (build) {
		/* do not add allocated nids */
		read_lock(&nm_i->nat_tree_lock);
		ne = __lookup_nat_cache(nm_i, nid);
1336 1337
		if (ne &&
			(!ne->checkpointed || nat_get_blkaddr(ne) != NULL_ADDR))
1338 1339 1340 1341
			allocated = true;
		read_unlock(&nm_i->nat_tree_lock);
		if (allocated)
			return 0;
J
Jaegeuk Kim 已提交
1342
	}
1343 1344

	i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS);
J
Jaegeuk Kim 已提交
1345 1346 1347 1348
	i->nid = nid;
	i->state = NID_NEW;

	spin_lock(&nm_i->free_nid_list_lock);
1349
	if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {
J
Jaegeuk Kim 已提交
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362
		spin_unlock(&nm_i->free_nid_list_lock);
		kmem_cache_free(free_nid_slab, i);
		return 0;
	}
	list_add_tail(&i->list, &nm_i->free_nid_list);
	nm_i->fcnt++;
	spin_unlock(&nm_i->free_nid_list_lock);
	return 1;
}

static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
{
	struct free_nid *i;
1363 1364
	bool need_free = false;

J
Jaegeuk Kim 已提交
1365
	spin_lock(&nm_i->free_nid_list_lock);
1366
	i = __lookup_free_nid_list(nm_i, nid);
J
Jaegeuk Kim 已提交
1367
	if (i && i->state == NID_NEW) {
1368
		__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
1369
		nm_i->fcnt--;
1370
		need_free = true;
J
Jaegeuk Kim 已提交
1371 1372
	}
	spin_unlock(&nm_i->free_nid_list_lock);
1373 1374 1375

	if (need_free)
		kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1376 1377
}

1378
static void scan_nat_page(struct f2fs_sb_info *sbi,
J
Jaegeuk Kim 已提交
1379 1380
			struct page *nat_page, nid_t start_nid)
{
1381
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
1382 1383 1384 1385 1386 1387 1388
	struct f2fs_nat_block *nat_blk = page_address(nat_page);
	block_t blk_addr;
	int i;

	i = start_nid % NAT_ENTRY_PER_BLOCK;

	for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
H
Haicheng Li 已提交
1389

1390
		if (unlikely(start_nid >= nm_i->max_nid))
J
Jaegeuk Kim 已提交
1391
			break;
H
Haicheng Li 已提交
1392 1393

		blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
1394
		f2fs_bug_on(blk_addr == NEW_ADDR);
H
Haicheng Li 已提交
1395
		if (blk_addr == NULL_ADDR) {
1396
			if (add_free_nid(sbi, start_nid, true) < 0)
H
Haicheng Li 已提交
1397 1398
				break;
		}
J
Jaegeuk Kim 已提交
1399 1400 1401 1402 1403 1404 1405 1406
	}
}

static void build_free_nids(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
	struct f2fs_summary_block *sum = curseg->sum_blk;
1407
	int i = 0;
1408
	nid_t nid = nm_i->next_scan_nid;
J
Jaegeuk Kim 已提交
1409

1410 1411 1412
	/* Enough entries */
	if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)
		return;
J
Jaegeuk Kim 已提交
1413

1414
	/* readahead nat pages to be scanned */
1415
	ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, META_NAT);
J
Jaegeuk Kim 已提交
1416 1417 1418 1419

	while (1) {
		struct page *page = get_current_nat_page(sbi, nid);

1420
		scan_nat_page(sbi, page, nid);
J
Jaegeuk Kim 已提交
1421 1422 1423
		f2fs_put_page(page, 1);

		nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
1424
		if (unlikely(nid >= nm_i->max_nid))
J
Jaegeuk Kim 已提交
1425
			nid = 0;
1426 1427

		if (i++ == FREE_NID_PAGES)
J
Jaegeuk Kim 已提交
1428 1429 1430
			break;
	}

1431 1432
	/* go to the next free nat pages to find free nids abundantly */
	nm_i->next_scan_nid = nid;
J
Jaegeuk Kim 已提交
1433 1434 1435 1436 1437 1438 1439

	/* find free nids from current sum_pages */
	mutex_lock(&curseg->curseg_mutex);
	for (i = 0; i < nats_in_cursum(sum); i++) {
		block_t addr = le32_to_cpu(nat_in_journal(sum, i).block_addr);
		nid = le32_to_cpu(nid_in_journal(sum, i));
		if (addr == NULL_ADDR)
1440
			add_free_nid(sbi, nid, true);
J
Jaegeuk Kim 已提交
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
		else
			remove_free_nid(nm_i, nid);
	}
	mutex_unlock(&curseg->curseg_mutex);
}

/*
 * If this function returns success, caller can obtain a new nid
 * from second parameter of this function.
 * The returned nid could be used ino as well as nid when inode is created.
 */
bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i = NULL;
retry:
1457
	if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))
1458
		return false;
J
Jaegeuk Kim 已提交
1459 1460 1461

	spin_lock(&nm_i->free_nid_list_lock);

1462
	/* We should not use stale free nids created by build_free_nids */
1463
	if (nm_i->fcnt && !on_build_free_nids(nm_i)) {
1464
		f2fs_bug_on(list_empty(&nm_i->free_nid_list));
1465
		list_for_each_entry(i, &nm_i->free_nid_list, list)
1466 1467
			if (i->state == NID_NEW)
				break;
J
Jaegeuk Kim 已提交
1468

1469
		f2fs_bug_on(i->state != NID_NEW);
1470 1471 1472 1473 1474 1475
		*nid = i->nid;
		i->state = NID_ALLOC;
		nm_i->fcnt--;
		spin_unlock(&nm_i->free_nid_list_lock);
		return true;
	}
J
Jaegeuk Kim 已提交
1476
	spin_unlock(&nm_i->free_nid_list_lock);
1477 1478 1479 1480 1481 1482

	/* Let's scan nat pages and its caches to get free nids */
	mutex_lock(&nm_i->build_lock);
	build_free_nids(sbi);
	mutex_unlock(&nm_i->build_lock);
	goto retry;
J
Jaegeuk Kim 已提交
1483 1484
}

J
Jaegeuk Kim 已提交
1485
/*
J
Jaegeuk Kim 已提交
1486 1487 1488 1489 1490 1491 1492 1493
 * alloc_nid() should be called prior to this function.
 */
void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i;

	spin_lock(&nm_i->free_nid_list_lock);
1494
	i = __lookup_free_nid_list(nm_i, nid);
1495
	f2fs_bug_on(!i || i->state != NID_ALLOC);
1496
	__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
1497
	spin_unlock(&nm_i->free_nid_list_lock);
1498 1499

	kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1500 1501
}

J
Jaegeuk Kim 已提交
1502
/*
J
Jaegeuk Kim 已提交
1503 1504 1505 1506
 * alloc_nid() should be called prior to this function.
 */
void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
{
1507 1508
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i;
1509
	bool need_free = false;
1510

J
Jaegeuk Kim 已提交
1511 1512 1513
	if (!nid)
		return;

1514
	spin_lock(&nm_i->free_nid_list_lock);
1515
	i = __lookup_free_nid_list(nm_i, nid);
1516
	f2fs_bug_on(!i || i->state != NID_ALLOC);
1517
	if (!available_free_memory(sbi, FREE_NIDS)) {
1518
		__del_from_free_nid_list(nm_i, i);
1519
		need_free = true;
1520 1521 1522 1523
	} else {
		i->state = NID_NEW;
		nm_i->fcnt++;
	}
1524
	spin_unlock(&nm_i->free_nid_list_lock);
1525 1526 1527

	if (need_free)
		kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1528 1529 1530 1531 1532 1533 1534
}

void recover_node_page(struct f2fs_sb_info *sbi, struct page *page,
		struct f2fs_summary *sum, struct node_info *ni,
		block_t new_blkaddr)
{
	rewrite_node_page(sbi, page, sum, ni->blk_addr, new_blkaddr);
1535
	set_node_addr(sbi, ni, new_blkaddr, false);
J
Jaegeuk Kim 已提交
1536 1537 1538
	clear_node_page_dirty(page);
}

1539
static void recover_inline_xattr(struct inode *inode, struct page *page)
1540 1541 1542 1543 1544 1545 1546
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	void *src_addr, *dst_addr;
	size_t inline_size;
	struct page *ipage;
	struct f2fs_inode *ri;

1547
	if (!f2fs_has_inline_xattr(inode))
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
		return;

	if (!IS_INODE(page))
		return;

	ri = F2FS_INODE(page);
	if (!(ri->i_inline & F2FS_INLINE_XATTR))
		return;

	ipage = get_node_page(sbi, inode->i_ino);
	f2fs_bug_on(IS_ERR(ipage));

	dst_addr = inline_xattr_addr(ipage);
	src_addr = inline_xattr_addr(page);
	inline_size = inline_xattr_size(inode);

1564
	f2fs_wait_on_page_writeback(ipage, NODE);
1565 1566 1567 1568 1569 1570
	memcpy(dst_addr, src_addr, inline_size);

	update_inode(inode, ipage);
	f2fs_put_page(ipage, 1);
}

1571 1572 1573 1574 1575 1576 1577
bool recover_xattr_data(struct inode *inode, struct page *page, block_t blkaddr)
{
	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
	nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
	nid_t new_xnid = nid_of_node(page);
	struct node_info ni;

1578 1579
	recover_inline_xattr(inode, page);

1580
	if (!f2fs_has_xattr_block(ofs_of_node(page)))
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
		return false;

	/* 1: invalidate the previous xattr nid */
	if (!prev_xnid)
		goto recover_xnid;

	/* Deallocate node address */
	get_node_info(sbi, prev_xnid, &ni);
	f2fs_bug_on(ni.blk_addr == NULL_ADDR);
	invalidate_blocks(sbi, ni.blk_addr);
	dec_valid_node_count(sbi, inode);
1592
	set_node_addr(sbi, &ni, NULL_ADDR, false);
1593 1594 1595 1596 1597 1598 1599 1600 1601

recover_xnid:
	/* 2: allocate new xattr nid */
	if (unlikely(!inc_valid_node_count(sbi, inode)))
		f2fs_bug_on(1);

	remove_free_nid(NM_I(sbi), new_xnid);
	get_node_info(sbi, new_xnid, &ni);
	ni.ino = inode->i_ino;
1602
	set_node_addr(sbi, &ni, NEW_ADDR, false);
1603 1604 1605 1606
	F2FS_I(inode)->i_xattr_nid = new_xnid;

	/* 3: update xattr blkaddr */
	refresh_sit_entry(sbi, NEW_ADDR, blkaddr);
1607
	set_node_addr(sbi, &ni, blkaddr, false);
1608 1609 1610 1611 1612

	update_inode_page(inode);
	return true;
}

J
Jaegeuk Kim 已提交
1613 1614
int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
{
1615
	struct f2fs_inode *src, *dst;
J
Jaegeuk Kim 已提交
1616 1617 1618 1619
	nid_t ino = ino_of_node(page);
	struct node_info old_ni, new_ni;
	struct page *ipage;

1620 1621 1622 1623 1624
	get_node_info(sbi, ino, &old_ni);

	if (unlikely(old_ni.blk_addr != NULL_ADDR))
		return -EINVAL;

1625
	ipage = grab_cache_page(NODE_MAPPING(sbi), ino);
J
Jaegeuk Kim 已提交
1626 1627 1628 1629 1630 1631 1632 1633 1634
	if (!ipage)
		return -ENOMEM;

	/* Should not use this inode  from free nid list */
	remove_free_nid(NM_I(sbi), ino);

	SetPageUptodate(ipage);
	fill_node_footer(ipage, ino, ino, 0, true);

1635 1636
	src = F2FS_INODE(page);
	dst = F2FS_INODE(ipage);
J
Jaegeuk Kim 已提交
1637

1638 1639 1640 1641 1642
	memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src);
	dst->i_size = 0;
	dst->i_blocks = cpu_to_le64(1);
	dst->i_links = cpu_to_le32(1);
	dst->i_xattr_nid = 0;
J
Jaegeuk Kim 已提交
1643 1644 1645 1646

	new_ni = old_ni;
	new_ni.ino = ino;

1647
	if (unlikely(!inc_valid_node_count(sbi, NULL)))
1648
		WARN_ON(1);
1649
	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
J
Jaegeuk Kim 已提交
1650 1651 1652 1653 1654
	inc_valid_inode_count(sbi);
	f2fs_put_page(ipage, 1);
	return 0;
}

1655 1656 1657 1658 1659 1660 1661 1662 1663
/*
 * ra_sum_pages() merge contiguous pages into one bio and submit.
 * these pre-readed pages are linked in pages list.
 */
static int ra_sum_pages(struct f2fs_sb_info *sbi, struct list_head *pages,
				int start, int nrpages)
{
	struct page *page;
	int page_idx = start;
J
Jaegeuk Kim 已提交
1664 1665
	struct f2fs_io_info fio = {
		.type = META,
1666
		.rw = READ_SYNC | REQ_META | REQ_PRIO
J
Jaegeuk Kim 已提交
1667
	};
1668 1669 1670

	for (; page_idx < start + nrpages; page_idx++) {
		/* alloc temporal page for read node summary info*/
1671
		page = alloc_page(GFP_F2FS_ZERO);
1672 1673
		if (!page)
			break;
1674 1675 1676 1677 1678 1679 1680

		lock_page(page);
		page->index = page_idx;
		list_add_tail(&page->lru, pages);
	}

	list_for_each_entry(page, pages, lru)
J
Jaegeuk Kim 已提交
1681
		f2fs_submit_page_mbio(sbi, page, page->index, &fio);
1682

J
Jaegeuk Kim 已提交
1683
	f2fs_submit_merged_bio(sbi, META, READ);
1684 1685

	return page_idx - start;
1686 1687
}

J
Jaegeuk Kim 已提交
1688 1689 1690 1691 1692
int restore_node_summary(struct f2fs_sb_info *sbi,
			unsigned int segno, struct f2fs_summary_block *sum)
{
	struct f2fs_node *rn;
	struct f2fs_summary *sum_entry;
1693
	struct page *page, *tmp;
J
Jaegeuk Kim 已提交
1694
	block_t addr;
1695 1696 1697
	int bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
	int i, last_offset, nrpages, err = 0;
	LIST_HEAD(page_list);
J
Jaegeuk Kim 已提交
1698 1699 1700 1701 1702 1703

	/* scan the node segment */
	last_offset = sbi->blocks_per_seg;
	addr = START_BLOCK(sbi, segno);
	sum_entry = &sum->entries[0];

1704
	for (i = 0; !err && i < last_offset; i += nrpages, addr += nrpages) {
1705
		nrpages = min(last_offset - i, bio_blocks);
1706

1707
		/* read ahead node pages */
1708 1709 1710
		nrpages = ra_sum_pages(sbi, &page_list, addr, nrpages);
		if (!nrpages)
			return -ENOMEM;
J
Jaegeuk Kim 已提交
1711

1712
		list_for_each_entry_safe(page, tmp, &page_list, lru) {
1713 1714
			if (err)
				goto skip;
1715 1716

			lock_page(page);
1717 1718 1719
			if (unlikely(!PageUptodate(page))) {
				err = -EIO;
			} else {
1720 1721 1722 1723 1724 1725 1726
				rn = F2FS_NODE(page);
				sum_entry->nid = rn->footer.nid;
				sum_entry->version = 0;
				sum_entry->ofs_in_node = 0;
				sum_entry++;
			}
			unlock_page(page);
1727 1728
skip:
			list_del(&page->lru);
1729 1730
			__free_pages(page, 0);
		}
J
Jaegeuk Kim 已提交
1731
	}
1732
	return err;
J
Jaegeuk Kim 已提交
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
}

static bool flush_nats_in_journal(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
	struct f2fs_summary_block *sum = curseg->sum_blk;
	int i;

	mutex_lock(&curseg->curseg_mutex);

	if (nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) {
		mutex_unlock(&curseg->curseg_mutex);
		return false;
	}

	for (i = 0; i < nats_in_cursum(sum); i++) {
		struct nat_entry *ne;
		struct f2fs_nat_entry raw_ne;
		nid_t nid = le32_to_cpu(nid_in_journal(sum, i));

		raw_ne = nat_in_journal(sum, i);
retry:
		write_lock(&nm_i->nat_tree_lock);
		ne = __lookup_nat_cache(nm_i, nid);
		if (ne) {
			__set_nat_cache_dirty(nm_i, ne);
			write_unlock(&nm_i->nat_tree_lock);
			continue;
		}
		ne = grab_nat_entry(nm_i, nid);
		if (!ne) {
			write_unlock(&nm_i->nat_tree_lock);
			goto retry;
		}
1768
		node_info_from_raw_nat(&ne->ni, &raw_ne);
J
Jaegeuk Kim 已提交
1769 1770 1771 1772 1773 1774 1775 1776
		__set_nat_cache_dirty(nm_i, ne);
		write_unlock(&nm_i->nat_tree_lock);
	}
	update_nats_in_cursum(sum, -i);
	mutex_unlock(&curseg->curseg_mutex);
	return true;
}

J
Jaegeuk Kim 已提交
1777
/*
J
Jaegeuk Kim 已提交
1778 1779 1780 1781 1782 1783 1784
 * This function is called during the checkpointing process.
 */
void flush_nat_entries(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
	struct f2fs_summary_block *sum = curseg->sum_blk;
1785
	struct nat_entry *ne, *cur;
J
Jaegeuk Kim 已提交
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796
	struct page *page = NULL;
	struct f2fs_nat_block *nat_blk = NULL;
	nid_t start_nid = 0, end_nid = 0;
	bool flushed;

	flushed = flush_nats_in_journal(sbi);

	if (!flushed)
		mutex_lock(&curseg->curseg_mutex);

	/* 1) flush dirty nat caches */
1797
	list_for_each_entry_safe(ne, cur, &nm_i->dirty_nat_entries, list) {
J
Jaegeuk Kim 已提交
1798 1799 1800 1801 1802 1803
		nid_t nid;
		struct f2fs_nat_entry raw_ne;
		int offset = -1;

		if (nat_get_blkaddr(ne) == NEW_ADDR)
			continue;
1804 1805 1806

		nid = nat_get_nid(ne);

J
Jaegeuk Kim 已提交
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
		if (flushed)
			goto to_nat_page;

		/* if there is room for nat enries in curseg->sumpage */
		offset = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 1);
		if (offset >= 0) {
			raw_ne = nat_in_journal(sum, offset);
			goto flush_now;
		}
to_nat_page:
		if (!page || (start_nid > nid || nid > end_nid)) {
			if (page) {
				f2fs_put_page(page, 1);
				page = NULL;
			}
			start_nid = START_NID(nid);
			end_nid = start_nid + NAT_ENTRY_PER_BLOCK - 1;

			/*
			 * get nat block with dirty flag, increased reference
			 * count, mapped and lock
			 */
			page = get_next_nat_page(sbi, start_nid);
			nat_blk = page_address(page);
		}

1833
		f2fs_bug_on(!nat_blk);
J
Jaegeuk Kim 已提交
1834 1835
		raw_ne = nat_blk->entries[nid - start_nid];
flush_now:
1836
		raw_nat_from_node_info(&raw_ne, &ne->ni);
J
Jaegeuk Kim 已提交
1837 1838 1839 1840 1841 1842 1843 1844

		if (offset < 0) {
			nat_blk->entries[nid - start_nid] = raw_ne;
		} else {
			nat_in_journal(sum, offset) = raw_ne;
			nid_in_journal(sum, offset) = cpu_to_le32(nid);
		}

1845
		if (nat_get_blkaddr(ne) == NULL_ADDR &&
1846
				add_free_nid(sbi, nid, false) <= 0) {
J
Jaegeuk Kim 已提交
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872
			write_lock(&nm_i->nat_tree_lock);
			__del_from_nat_cache(nm_i, ne);
			write_unlock(&nm_i->nat_tree_lock);
		} else {
			write_lock(&nm_i->nat_tree_lock);
			__clear_nat_cache_dirty(nm_i, ne);
			write_unlock(&nm_i->nat_tree_lock);
		}
	}
	if (!flushed)
		mutex_unlock(&curseg->curseg_mutex);
	f2fs_put_page(page, 1);
}

static int init_node_manager(struct f2fs_sb_info *sbi)
{
	struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	unsigned char *version_bitmap;
	unsigned int nat_segs, nat_blocks;

	nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);

	/* segment_count_nat includes pair segment so divide to 2. */
	nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
	nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
1873

1874 1875
	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;

1876
	/* not used nids: 0, node, meta, (and root counted as valid node) */
1877
	nm_i->available_nids = nm_i->max_nid - 3;
J
Jaegeuk Kim 已提交
1878 1879
	nm_i->fcnt = 0;
	nm_i->nat_cnt = 0;
1880
	nm_i->ram_thresh = DEF_RAM_THRESHOLD;
J
Jaegeuk Kim 已提交
1881

1882
	INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
J
Jaegeuk Kim 已提交
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
	INIT_LIST_HEAD(&nm_i->free_nid_list);
	INIT_RADIX_TREE(&nm_i->nat_root, GFP_ATOMIC);
	INIT_LIST_HEAD(&nm_i->nat_entries);
	INIT_LIST_HEAD(&nm_i->dirty_nat_entries);

	mutex_init(&nm_i->build_lock);
	spin_lock_init(&nm_i->free_nid_list_lock);
	rwlock_init(&nm_i->nat_tree_lock);

	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
A
Alexandru Gheorghiu 已提交
1893
	nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
J
Jaegeuk Kim 已提交
1894 1895 1896 1897
	version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
	if (!version_bitmap)
		return -EFAULT;

A
Alexandru Gheorghiu 已提交
1898 1899 1900 1901
	nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
					GFP_KERNEL);
	if (!nm_i->nat_bitmap)
		return -ENOMEM;
J
Jaegeuk Kim 已提交
1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
	return 0;
}

int build_node_manager(struct f2fs_sb_info *sbi)
{
	int err;

	sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL);
	if (!sbi->nm_info)
		return -ENOMEM;

	err = init_node_manager(sbi);
	if (err)
		return err;

	build_free_nids(sbi);
	return 0;
}

void destroy_node_manager(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i, *next_i;
	struct nat_entry *natvec[NATVEC_SIZE];
	nid_t nid = 0;
	unsigned int found;

	if (!nm_i)
		return;

	/* destroy free nid list */
	spin_lock(&nm_i->free_nid_list_lock);
	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
1935
		f2fs_bug_on(i->state == NID_ALLOC);
1936
		__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
1937
		nm_i->fcnt--;
1938 1939 1940
		spin_unlock(&nm_i->free_nid_list_lock);
		kmem_cache_free(free_nid_slab, i);
		spin_lock(&nm_i->free_nid_list_lock);
J
Jaegeuk Kim 已提交
1941
	}
1942
	f2fs_bug_on(nm_i->fcnt);
J
Jaegeuk Kim 已提交
1943 1944 1945 1946 1947 1948 1949
	spin_unlock(&nm_i->free_nid_list_lock);

	/* destroy nat cache */
	write_lock(&nm_i->nat_tree_lock);
	while ((found = __gang_lookup_nat_cache(nm_i,
					nid, NATVEC_SIZE, natvec))) {
		unsigned idx;
1950 1951 1952
		nid = nat_get_nid(natvec[found - 1]) + 1;
		for (idx = 0; idx < found; idx++)
			__del_from_nat_cache(nm_i, natvec[idx]);
J
Jaegeuk Kim 已提交
1953
	}
1954
	f2fs_bug_on(nm_i->nat_cnt);
J
Jaegeuk Kim 已提交
1955 1956 1957 1958 1959 1960 1961
	write_unlock(&nm_i->nat_tree_lock);

	kfree(nm_i->nat_bitmap);
	sbi->nm_info = NULL;
	kfree(nm_i);
}

1962
int __init create_node_manager_caches(void)
J
Jaegeuk Kim 已提交
1963 1964
{
	nat_entry_slab = f2fs_kmem_cache_create("nat_entry",
1965
			sizeof(struct nat_entry));
J
Jaegeuk Kim 已提交
1966 1967 1968 1969
	if (!nat_entry_slab)
		return -ENOMEM;

	free_nid_slab = f2fs_kmem_cache_create("free_nid",
1970
			sizeof(struct free_nid));
J
Jaegeuk Kim 已提交
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982
	if (!free_nid_slab) {
		kmem_cache_destroy(nat_entry_slab);
		return -ENOMEM;
	}
	return 0;
}

void destroy_node_manager_caches(void)
{
	kmem_cache_destroy(free_nid_slab);
	kmem_cache_destroy(nat_entry_slab);
}