node.c 56.6 KB
Newer Older
J
Jaegeuk Kim 已提交
1
/*
J
Jaegeuk Kim 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 * fs/f2fs/node.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/mpage.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/pagevec.h>
#include <linux/swap.h>

#include "f2fs.h"
#include "node.h"
#include "segment.h"
J
Jaegeuk Kim 已提交
22
#include "trace.h"
23
#include <trace/events/f2fs.h>
J
Jaegeuk Kim 已提交
24

25 26
#define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock)

J
Jaegeuk Kim 已提交
27 28
static struct kmem_cache *nat_entry_slab;
static struct kmem_cache *free_nid_slab;
29
static struct kmem_cache *nat_entry_set_slab;
J
Jaegeuk Kim 已提交
30

31
bool available_free_memory(struct f2fs_sb_info *sbi, int type)
32
{
33
	struct f2fs_nm_info *nm_i = NM_I(sbi);
34
	struct sysinfo val;
35
	unsigned long avail_ram;
36
	unsigned long mem_size = 0;
37
	bool res = false;
38 39

	si_meminfo(&val);
40 41 42 43

	/* only uses low memory */
	avail_ram = val.totalram - val.totalhigh;

44 45 46
	/*
	 * give 25%, 25%, 50%, 50%, 50% memory for each components respectively
	 */
47
	if (type == FREE_NIDS) {
48
		mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>
49
							PAGE_SHIFT;
50
		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
51
	} else if (type == NAT_ENTRIES) {
52
		mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>
53
							PAGE_SHIFT;
54
		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
55 56 57 58 59
	} else if (type == DIRTY_DENTS) {
		if (sbi->sb->s_bdi->wb.dirty_exceeded)
			return false;
		mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
60 61 62 63
	} else if (type == INO_ENTRIES) {
		int i;

		for (i = 0; i <= UPDATE_INO; i++)
64
			mem_size += (sbi->im[i].ino_num *
65
				sizeof(struct ino_entry)) >> PAGE_SHIFT;
66
		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
67
	} else if (type == EXTENT_CACHE) {
68 69
		mem_size = (atomic_read(&sbi->total_ext_tree) *
				sizeof(struct extent_tree) +
70
				atomic_read(&sbi->total_ext_node) *
71
				sizeof(struct extent_node)) >> PAGE_SHIFT;
72
		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
73
	} else {
74 75
		if (!sbi->sb->s_bdi->wb.dirty_exceeded)
			return true;
76 77
	}
	return res;
78 79
}

J
Jaegeuk Kim 已提交
80 81 82 83 84 85 86 87 88 89 90 91 92
static void clear_node_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	unsigned int long flags;

	if (PageDirty(page)) {
		spin_lock_irqsave(&mapping->tree_lock, flags);
		radix_tree_tag_clear(&mapping->page_tree,
				page_index(page),
				PAGECACHE_TAG_DIRTY);
		spin_unlock_irqrestore(&mapping->tree_lock, flags);

		clear_page_dirty_for_io(page);
93
		dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES);
J
Jaegeuk Kim 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
	}
	ClearPageUptodate(page);
}

static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	pgoff_t index = current_nat_addr(sbi, nid);
	return get_meta_page(sbi, index);
}

static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct page *src_page;
	struct page *dst_page;
	pgoff_t src_off;
	pgoff_t dst_off;
	void *src_addr;
	void *dst_addr;
	struct f2fs_nm_info *nm_i = NM_I(sbi);

	src_off = current_nat_addr(sbi, nid);
	dst_off = next_nat_addr(sbi, src_off);

	/* get current nat block page with lock */
	src_page = get_meta_page(sbi, src_off);
	dst_page = grab_meta_page(sbi, dst_off);
120
	f2fs_bug_on(sbi, PageDirty(src_page));
J
Jaegeuk Kim 已提交
121 122 123

	src_addr = page_address(src_page);
	dst_addr = page_address(dst_page);
124
	memcpy(dst_addr, src_addr, PAGE_SIZE);
J
Jaegeuk Kim 已提交
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
	set_page_dirty(dst_page);
	f2fs_put_page(src_page, 1);

	set_to_next_nat(nm_i, nid);

	return dst_page;
}

static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)
{
	return radix_tree_lookup(&nm_i->nat_root, n);
}

static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
		nid_t start, unsigned int nr, struct nat_entry **ep)
{
	return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
}

static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
{
	list_del(&e->list);
	radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
	nm_i->nat_cnt--;
	kmem_cache_free(nat_entry_slab, e);
}

152 153 154 155 156 157 158 159
static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,
						struct nat_entry *ne)
{
	nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
	struct nat_entry_set *head;

	if (get_nat_flag(ne, IS_DIRTY))
		return;
160

161 162
	head = radix_tree_lookup(&nm_i->nat_set_root, set);
	if (!head) {
163
		head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS);
164 165 166 167 168

		INIT_LIST_HEAD(&head->entry_list);
		INIT_LIST_HEAD(&head->set_list);
		head->set = set;
		head->entry_cnt = 0;
169
		f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);
170 171 172 173 174 175 176 177 178 179
	}
	list_move_tail(&ne->list, &head->entry_list);
	nm_i->dirty_nat_cnt++;
	head->entry_cnt++;
	set_nat_flag(ne, IS_DIRTY, true);
}

static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,
						struct nat_entry *ne)
{
180
	nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
	struct nat_entry_set *head;

	head = radix_tree_lookup(&nm_i->nat_set_root, set);
	if (head) {
		list_move_tail(&ne->list, &nm_i->nat_entries);
		set_nat_flag(ne, IS_DIRTY, false);
		head->entry_cnt--;
		nm_i->dirty_nat_cnt--;
	}
}

static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,
		nid_t start, unsigned int nr, struct nat_entry_set **ep)
{
	return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,
							start, nr);
}

J
Jaegeuk Kim 已提交
199
int need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)
J
Jaegeuk Kim 已提交
200 201 202
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
J
Jaegeuk Kim 已提交
203
	bool need = false;
J
Jaegeuk Kim 已提交
204

205
	down_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
206
	e = __lookup_nat_cache(nm_i, nid);
J
Jaegeuk Kim 已提交
207 208 209 210 211
	if (e) {
		if (!get_nat_flag(e, IS_CHECKPOINTED) &&
				!get_nat_flag(e, HAS_FSYNCED_INODE))
			need = true;
	}
212
	up_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
213
	return need;
J
Jaegeuk Kim 已提交
214 215
}

J
Jaegeuk Kim 已提交
216
bool is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
217 218 219
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
J
Jaegeuk Kim 已提交
220
	bool is_cp = true;
221

222
	down_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
223 224 225
	e = __lookup_nat_cache(nm_i, nid);
	if (e && !get_nat_flag(e, IS_CHECKPOINTED))
		is_cp = false;
226
	up_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
227
	return is_cp;
228 229
}

230
bool need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
231 232 233
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
234
	bool need_update = true;
235

236
	down_read(&nm_i->nat_tree_lock);
237 238 239 240 241
	e = __lookup_nat_cache(nm_i, ino);
	if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
			(get_nat_flag(e, IS_CHECKPOINTED) ||
			 get_nat_flag(e, HAS_FSYNCED_INODE)))
		need_update = false;
242
	up_read(&nm_i->nat_tree_lock);
243
	return need_update;
244 245
}

J
Jaegeuk Kim 已提交
246 247 248 249
static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)
{
	struct nat_entry *new;

250
	new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_NOFS);
251
	f2fs_radix_tree_insert(&nm_i->nat_root, nid, new);
J
Jaegeuk Kim 已提交
252 253
	memset(new, 0, sizeof(struct nat_entry));
	nat_set_nid(new, nid);
254
	nat_reset_flag(new);
J
Jaegeuk Kim 已提交
255 256 257 258 259
	list_add_tail(&new->list, &nm_i->nat_entries);
	nm_i->nat_cnt++;
	return new;
}

260
static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
J
Jaegeuk Kim 已提交
261 262
						struct f2fs_nat_entry *ne)
{
263
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
264
	struct nat_entry *e;
265

J
Jaegeuk Kim 已提交
266 267 268
	e = __lookup_nat_cache(nm_i, nid);
	if (!e) {
		e = grab_nat_entry(nm_i, nid);
269
		node_info_from_raw_nat(&e->ni, ne);
270 271 272 273
	} else {
		f2fs_bug_on(sbi, nat_get_ino(e) != ne->ino ||
				nat_get_blkaddr(e) != ne->block_addr ||
				nat_get_version(e) != ne->version);
J
Jaegeuk Kim 已提交
274 275 276 277
	}
}

static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
278
			block_t new_blkaddr, bool fsync_done)
J
Jaegeuk Kim 已提交
279 280 281
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct nat_entry *e;
282

283
	down_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
284 285 286
	e = __lookup_nat_cache(nm_i, ni->nid);
	if (!e) {
		e = grab_nat_entry(nm_i, ni->nid);
287
		copy_node_info(&e->ni, ni);
288
		f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
J
Jaegeuk Kim 已提交
289 290 291 292 293 294
	} else if (new_blkaddr == NEW_ADDR) {
		/*
		 * when nid is reallocated,
		 * previous nat entry can be remained in nat cache.
		 * So, reinitialize it with new information.
		 */
295
		copy_node_info(&e->ni, ni);
296
		f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR);
J
Jaegeuk Kim 已提交
297 298 299
	}

	/* sanity check */
300 301
	f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr);
	f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR &&
J
Jaegeuk Kim 已提交
302
			new_blkaddr == NULL_ADDR);
303
	f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR &&
J
Jaegeuk Kim 已提交
304
			new_blkaddr == NEW_ADDR);
305
	f2fs_bug_on(sbi, nat_get_blkaddr(e) != NEW_ADDR &&
J
Jaegeuk Kim 已提交
306 307 308
			nat_get_blkaddr(e) != NULL_ADDR &&
			new_blkaddr == NEW_ADDR);

A
arter97 已提交
309
	/* increment version no as node is removed */
J
Jaegeuk Kim 已提交
310 311 312
	if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
		unsigned char version = nat_get_version(e);
		nat_set_version(e, inc_node_version(version));
J
Jaegeuk Kim 已提交
313 314 315 316

		/* in order to reuse the nid */
		if (nm_i->next_scan_nid > ni->nid)
			nm_i->next_scan_nid = ni->nid;
J
Jaegeuk Kim 已提交
317 318 319 320
	}

	/* change address */
	nat_set_blkaddr(e, new_blkaddr);
321 322
	if (new_blkaddr == NEW_ADDR || new_blkaddr == NULL_ADDR)
		set_nat_flag(e, IS_CHECKPOINTED, false);
J
Jaegeuk Kim 已提交
323
	__set_nat_cache_dirty(nm_i, e);
324 325

	/* update fsync_mark if its inode nat entry is still alive */
326 327
	if (ni->nid != ni->ino)
		e = __lookup_nat_cache(nm_i, ni->ino);
328 329 330 331 332
	if (e) {
		if (fsync_done && ni->nid == ni->ino)
			set_nat_flag(e, HAS_FSYNCED_INODE, true);
		set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
	}
333
	up_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
334 335
}

336
int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
J
Jaegeuk Kim 已提交
337 338
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
339
	int nr = nr_shrink;
J
Jaegeuk Kim 已提交
340

J
Jaegeuk Kim 已提交
341
	if (!down_write_trylock(&nm_i->nat_tree_lock))
J
Jaegeuk Kim 已提交
342 343 344 345 346 347 348 349 350
		return 0;

	while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
		struct nat_entry *ne;
		ne = list_first_entry(&nm_i->nat_entries,
					struct nat_entry, list);
		__del_from_nat_cache(nm_i, ne);
		nr_shrink--;
	}
351
	up_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
352
	return nr - nr_shrink;
J
Jaegeuk Kim 已提交
353 354
}

J
Jaegeuk Kim 已提交
355
/*
A
arter97 已提交
356
 * This function always returns success
J
Jaegeuk Kim 已提交
357 358 359 360 361
 */
void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
362
	struct f2fs_journal *journal = curseg->journal;
J
Jaegeuk Kim 已提交
363 364 365 366 367 368 369 370 371 372
	nid_t start_nid = START_NID(nid);
	struct f2fs_nat_block *nat_blk;
	struct page *page = NULL;
	struct f2fs_nat_entry ne;
	struct nat_entry *e;
	int i;

	ni->nid = nid;

	/* Check nat cache */
373
	down_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
374 375 376 377 378
	e = __lookup_nat_cache(nm_i, nid);
	if (e) {
		ni->ino = nat_get_ino(e);
		ni->blk_addr = nat_get_blkaddr(e);
		ni->version = nat_get_version(e);
379
		up_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
380
		return;
381
	}
J
Jaegeuk Kim 已提交
382

383 384
	memset(&ne, 0, sizeof(struct f2fs_nat_entry));

J
Jaegeuk Kim 已提交
385
	/* Check current segment summary */
386
	down_read(&curseg->journal_rwsem);
387
	i = lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0);
J
Jaegeuk Kim 已提交
388
	if (i >= 0) {
389
		ne = nat_in_journal(journal, i);
J
Jaegeuk Kim 已提交
390 391
		node_info_from_raw_nat(ni, &ne);
	}
392
	up_read(&curseg->journal_rwsem);
J
Jaegeuk Kim 已提交
393 394 395 396 397 398 399 400 401 402
	if (i >= 0)
		goto cache;

	/* Fill node_info from nat page */
	page = get_current_nat_page(sbi, start_nid);
	nat_blk = (struct f2fs_nat_block *)page_address(page);
	ne = nat_blk->entries[nid - start_nid];
	node_info_from_raw_nat(ni, &ne);
	f2fs_put_page(page, 1);
cache:
403
	up_read(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
404
	/* cache nat entry */
405 406
	down_write(&nm_i->nat_tree_lock);
	cache_nat_entry(sbi, nid, &ne);
407
	up_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
408 409
}

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
/*
 * readahead MAX_RA_NODE number of node pages.
 */
static void ra_node_pages(struct page *parent, int start, int n)
{
	struct f2fs_sb_info *sbi = F2FS_P_SB(parent);
	struct blk_plug plug;
	int i, end;
	nid_t nid;

	blk_start_plug(&plug);

	/* Then, try readahead for siblings of the desired node */
	end = start + n;
	end = min(end, NIDS_PER_BLOCK);
	for (i = start; i < end; i++) {
		nid = get_nid(parent, i, false);
		ra_node_page(sbi, nid);
	}

	blk_finish_plug(&plug);
}

433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
pgoff_t get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
{
	const long direct_index = ADDRS_PER_INODE(dn->inode);
	const long direct_blks = ADDRS_PER_BLOCK;
	const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
	unsigned int skipped_unit = ADDRS_PER_BLOCK;
	int cur_level = dn->cur_level;
	int max_level = dn->max_level;
	pgoff_t base = 0;

	if (!dn->max_level)
		return pgofs + 1;

	while (max_level-- > cur_level)
		skipped_unit *= NIDS_PER_BLOCK;

	switch (dn->max_level) {
	case 3:
		base += 2 * indirect_blks;
	case 2:
		base += 2 * direct_blks;
	case 1:
		base += direct_index;
		break;
	default:
		f2fs_bug_on(F2FS_I_SB(dn->inode), 1);
	}

	return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base;
}

J
Jaegeuk Kim 已提交
464
/*
J
Jaegeuk Kim 已提交
465 466 467
 * The maximum depth is four.
 * Offset[0] will have raw inode offset.
 */
468
static int get_node_path(struct inode *inode, long block,
469
				int offset[4], unsigned int noffset[4])
J
Jaegeuk Kim 已提交
470
{
471
	const long direct_index = ADDRS_PER_INODE(inode);
J
Jaegeuk Kim 已提交
472 473 474 475 476 477 478 479 480 481
	const long direct_blks = ADDRS_PER_BLOCK;
	const long dptrs_per_blk = NIDS_PER_BLOCK;
	const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
	const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
	int n = 0;
	int level = 0;

	noffset[0] = 0;

	if (block < direct_index) {
482
		offset[n] = block;
J
Jaegeuk Kim 已提交
483 484 485 486 487 488
		goto got;
	}
	block -= direct_index;
	if (block < direct_blks) {
		offset[n++] = NODE_DIR1_BLOCK;
		noffset[n] = 1;
489
		offset[n] = block;
J
Jaegeuk Kim 已提交
490 491 492 493 494 495 496
		level = 1;
		goto got;
	}
	block -= direct_blks;
	if (block < direct_blks) {
		offset[n++] = NODE_DIR2_BLOCK;
		noffset[n] = 2;
497
		offset[n] = block;
J
Jaegeuk Kim 已提交
498 499 500 501 502 503 504 505 506
		level = 1;
		goto got;
	}
	block -= direct_blks;
	if (block < indirect_blks) {
		offset[n++] = NODE_IND1_BLOCK;
		noffset[n] = 3;
		offset[n++] = block / direct_blks;
		noffset[n] = 4 + offset[n - 1];
507
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
508 509 510 511 512 513 514 515 516
		level = 2;
		goto got;
	}
	block -= indirect_blks;
	if (block < indirect_blks) {
		offset[n++] = NODE_IND2_BLOCK;
		noffset[n] = 4 + dptrs_per_blk;
		offset[n++] = block / direct_blks;
		noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
517
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
518 519 520 521 522 523 524 525 526 527 528 529 530 531
		level = 2;
		goto got;
	}
	block -= indirect_blks;
	if (block < dindirect_blks) {
		offset[n++] = NODE_DIND_BLOCK;
		noffset[n] = 5 + (dptrs_per_blk * 2);
		offset[n++] = block / indirect_blks;
		noffset[n] = 6 + (dptrs_per_blk * 2) +
			      offset[n - 1] * (dptrs_per_blk + 1);
		offset[n++] = (block / direct_blks) % dptrs_per_blk;
		noffset[n] = 7 + (dptrs_per_blk * 2) +
			      offset[n - 2] * (dptrs_per_blk + 1) +
			      offset[n - 1];
532
		offset[n] = block % direct_blks;
J
Jaegeuk Kim 已提交
533 534 535 536 537 538 539 540 541 542 543
		level = 3;
		goto got;
	} else {
		BUG();
	}
got:
	return level;
}

/*
 * Caller should call f2fs_put_dnode(dn).
C
Chao Yu 已提交
544 545
 * Also, it should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op() only if ro is not set RDONLY_NODE.
546
 * In the case of RDONLY_NODE, we don't need to care about mutex.
J
Jaegeuk Kim 已提交
547
 */
548
int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
J
Jaegeuk Kim 已提交
549
{
550
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
J
Jaegeuk Kim 已提交
551
	struct page *npage[4];
552
	struct page *parent = NULL;
J
Jaegeuk Kim 已提交
553 554 555
	int offset[4];
	unsigned int noffset[4];
	nid_t nids[4];
556
	int level, i = 0;
J
Jaegeuk Kim 已提交
557 558
	int err = 0;

559
	level = get_node_path(dn->inode, index, offset, noffset);
J
Jaegeuk Kim 已提交
560 561

	nids[0] = dn->inode->i_ino;
562
	npage[0] = dn->inode_page;
J
Jaegeuk Kim 已提交
563

564 565 566 567 568
	if (!npage[0]) {
		npage[0] = get_node_page(sbi, nids[0]);
		if (IS_ERR(npage[0]))
			return PTR_ERR(npage[0]);
	}
569 570 571

	/* if inline_data is set, should not report any block indices */
	if (f2fs_has_inline_data(dn->inode) && index) {
572
		err = -ENOENT;
573 574 575 576
		f2fs_put_page(npage[0], 1);
		goto release_out;
	}

J
Jaegeuk Kim 已提交
577
	parent = npage[0];
578 579
	if (level != 0)
		nids[1] = get_nid(parent, offset[0], true);
J
Jaegeuk Kim 已提交
580 581 582 583 584 585 586
	dn->inode_page = npage[0];
	dn->inode_page_locked = true;

	/* get indirect or direct nodes */
	for (i = 1; i <= level; i++) {
		bool done = false;

587
		if (!nids[i] && mode == ALLOC_NODE) {
J
Jaegeuk Kim 已提交
588 589 590 591 592 593 594
			/* alloc new node */
			if (!alloc_nid(sbi, &(nids[i]))) {
				err = -ENOSPC;
				goto release_pages;
			}

			dn->nid = nids[i];
595
			npage[i] = new_node_page(dn, noffset[i], NULL);
J
Jaegeuk Kim 已提交
596 597 598 599 600 601 602 603 604
			if (IS_ERR(npage[i])) {
				alloc_nid_failed(sbi, nids[i]);
				err = PTR_ERR(npage[i]);
				goto release_pages;
			}

			set_nid(parent, offset[i - 1], nids[i], i == 1);
			alloc_nid_done(sbi, nids[i]);
			done = true;
605
		} else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
J
Jaegeuk Kim 已提交
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
			npage[i] = get_node_page_ra(parent, offset[i - 1]);
			if (IS_ERR(npage[i])) {
				err = PTR_ERR(npage[i]);
				goto release_pages;
			}
			done = true;
		}
		if (i == 1) {
			dn->inode_page_locked = false;
			unlock_page(parent);
		} else {
			f2fs_put_page(parent, 1);
		}

		if (!done) {
			npage[i] = get_node_page(sbi, nids[i]);
			if (IS_ERR(npage[i])) {
				err = PTR_ERR(npage[i]);
				f2fs_put_page(npage[0], 0);
				goto release_out;
			}
		}
		if (i < level) {
			parent = npage[i];
			nids[i + 1] = get_nid(parent, offset[i], false);
		}
	}
	dn->nid = nids[level];
	dn->ofs_in_node = offset[level];
	dn->node_page = npage[level];
	dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
	return 0;

release_pages:
	f2fs_put_page(parent, 1);
	if (i > 1)
		f2fs_put_page(npage[0], 0);
release_out:
	dn->inode_page = NULL;
	dn->node_page = NULL;
646 647 648 649
	if (err == -ENOENT) {
		dn->cur_level = i;
		dn->max_level = level;
	}
J
Jaegeuk Kim 已提交
650 651 652 653 654
	return err;
}

static void truncate_node(struct dnode_of_data *dn)
{
655
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
J
Jaegeuk Kim 已提交
656 657 658
	struct node_info ni;

	get_node_info(sbi, dn->nid, &ni);
659
	if (dn->inode->i_blocks == 0) {
660
		f2fs_bug_on(sbi, ni.blk_addr != NULL_ADDR);
661 662
		goto invalidate;
	}
663
	f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR);
J
Jaegeuk Kim 已提交
664 665

	/* Deallocate node address */
666
	invalidate_blocks(sbi, ni.blk_addr);
667
	dec_valid_node_count(sbi, dn->inode);
668
	set_node_addr(sbi, &ni, NULL_ADDR, false);
J
Jaegeuk Kim 已提交
669 670 671 672 673 674 675

	if (dn->nid == dn->inode->i_ino) {
		remove_orphan_inode(sbi, dn->nid);
		dec_valid_inode_count(sbi);
	} else {
		sync_inode_page(dn);
	}
676
invalidate:
J
Jaegeuk Kim 已提交
677
	clear_node_page_dirty(dn->node_page);
678
	set_sbi_flag(sbi, SBI_IS_DIRTY);
J
Jaegeuk Kim 已提交
679 680

	f2fs_put_page(dn->node_page, 1);
681 682 683 684

	invalidate_mapping_pages(NODE_MAPPING(sbi),
			dn->node_page->index, dn->node_page->index);

J
Jaegeuk Kim 已提交
685
	dn->node_page = NULL;
686
	trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
J
Jaegeuk Kim 已提交
687 688 689 690 691 692 693 694 695 696
}

static int truncate_dnode(struct dnode_of_data *dn)
{
	struct page *page;

	if (dn->nid == 0)
		return 1;

	/* get direct node */
697
	page = get_node_page(F2FS_I_SB(dn->inode), dn->nid);
J
Jaegeuk Kim 已提交
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
	if (IS_ERR(page) && PTR_ERR(page) == -ENOENT)
		return 1;
	else if (IS_ERR(page))
		return PTR_ERR(page);

	/* Make dnode_of_data for parameter */
	dn->node_page = page;
	dn->ofs_in_node = 0;
	truncate_data_blocks(dn);
	truncate_node(dn);
	return 1;
}

static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
						int ofs, int depth)
{
	struct dnode_of_data rdn = *dn;
	struct page *page;
	struct f2fs_node *rn;
	nid_t child_nid;
	unsigned int child_nofs;
	int freed = 0;
	int i, ret;

	if (dn->nid == 0)
		return NIDS_PER_BLOCK + 1;

725 726
	trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);

727
	page = get_node_page(F2FS_I_SB(dn->inode), dn->nid);
728 729
	if (IS_ERR(page)) {
		trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page));
J
Jaegeuk Kim 已提交
730
		return PTR_ERR(page);
731
	}
J
Jaegeuk Kim 已提交
732

733 734
	ra_node_pages(page, ofs, NIDS_PER_BLOCK);

735
	rn = F2FS_NODE(page);
J
Jaegeuk Kim 已提交
736 737 738 739 740 741 742 743 744
	if (depth < 3) {
		for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
			child_nid = le32_to_cpu(rn->in.nid[i]);
			if (child_nid == 0)
				continue;
			rdn.nid = child_nid;
			ret = truncate_dnode(&rdn);
			if (ret < 0)
				goto out_err;
745 746
			if (set_nid(page, i, 0, false))
				dn->node_changed = true;
J
Jaegeuk Kim 已提交
747 748 749 750 751 752 753 754 755 756 757 758
		}
	} else {
		child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
		for (i = ofs; i < NIDS_PER_BLOCK; i++) {
			child_nid = le32_to_cpu(rn->in.nid[i]);
			if (child_nid == 0) {
				child_nofs += NIDS_PER_BLOCK + 1;
				continue;
			}
			rdn.nid = child_nid;
			ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
			if (ret == (NIDS_PER_BLOCK + 1)) {
759 760
				if (set_nid(page, i, 0, false))
					dn->node_changed = true;
J
Jaegeuk Kim 已提交
761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
				child_nofs += ret;
			} else if (ret < 0 && ret != -ENOENT) {
				goto out_err;
			}
		}
		freed = child_nofs;
	}

	if (!ofs) {
		/* remove current indirect node */
		dn->node_page = page;
		truncate_node(dn);
		freed++;
	} else {
		f2fs_put_page(page, 1);
	}
777
	trace_f2fs_truncate_nodes_exit(dn->inode, freed);
J
Jaegeuk Kim 已提交
778 779 780 781
	return freed;

out_err:
	f2fs_put_page(page, 1);
782
	trace_f2fs_truncate_nodes_exit(dn->inode, ret);
J
Jaegeuk Kim 已提交
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800
	return ret;
}

static int truncate_partial_nodes(struct dnode_of_data *dn,
			struct f2fs_inode *ri, int *offset, int depth)
{
	struct page *pages[2];
	nid_t nid[3];
	nid_t child_nid;
	int err = 0;
	int i;
	int idx = depth - 2;

	nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
	if (!nid[0])
		return 0;

	/* get indirect nodes in the path */
801
	for (i = 0; i < idx + 1; i++) {
A
arter97 已提交
802
		/* reference count'll be increased */
803
		pages[i] = get_node_page(F2FS_I_SB(dn->inode), nid[i]);
J
Jaegeuk Kim 已提交
804 805
		if (IS_ERR(pages[i])) {
			err = PTR_ERR(pages[i]);
806
			idx = i - 1;
J
Jaegeuk Kim 已提交
807 808 809 810 811
			goto fail;
		}
		nid[i + 1] = get_nid(pages[i], offset[i + 1], false);
	}

812 813
	ra_node_pages(pages[idx], offset[idx + 1], NIDS_PER_BLOCK);

J
Jaegeuk Kim 已提交
814
	/* free direct nodes linked to a partial indirect node */
815
	for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
J
Jaegeuk Kim 已提交
816 817 818 819 820 821 822
		child_nid = get_nid(pages[idx], i, false);
		if (!child_nid)
			continue;
		dn->nid = child_nid;
		err = truncate_dnode(dn);
		if (err < 0)
			goto fail;
823 824
		if (set_nid(pages[idx], i, 0, false))
			dn->node_changed = true;
J
Jaegeuk Kim 已提交
825 826
	}

827
	if (offset[idx + 1] == 0) {
J
Jaegeuk Kim 已提交
828 829 830 831 832 833 834
		dn->node_page = pages[idx];
		dn->nid = nid[idx];
		truncate_node(dn);
	} else {
		f2fs_put_page(pages[idx], 1);
	}
	offset[idx]++;
835 836
	offset[idx + 1] = 0;
	idx--;
J
Jaegeuk Kim 已提交
837
fail:
838
	for (i = idx; i >= 0; i--)
J
Jaegeuk Kim 已提交
839
		f2fs_put_page(pages[i], 1);
840 841 842

	trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);

J
Jaegeuk Kim 已提交
843 844 845
	return err;
}

J
Jaegeuk Kim 已提交
846
/*
J
Jaegeuk Kim 已提交
847 848 849 850
 * All the block addresses of data and nodes should be nullified.
 */
int truncate_inode_blocks(struct inode *inode, pgoff_t from)
{
851
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
J
Jaegeuk Kim 已提交
852 853
	int err = 0, cont = 1;
	int level, offset[4], noffset[4];
J
Jaegeuk Kim 已提交
854
	unsigned int nofs = 0;
855
	struct f2fs_inode *ri;
J
Jaegeuk Kim 已提交
856 857 858
	struct dnode_of_data dn;
	struct page *page;

859 860
	trace_f2fs_truncate_inode_blocks_enter(inode, from);

861
	level = get_node_path(inode, from, offset, noffset);
862

J
Jaegeuk Kim 已提交
863
	page = get_node_page(sbi, inode->i_ino);
864 865
	if (IS_ERR(page)) {
		trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page));
J
Jaegeuk Kim 已提交
866
		return PTR_ERR(page);
867
	}
J
Jaegeuk Kim 已提交
868 869 870 871

	set_new_dnode(&dn, inode, page, NULL, 0);
	unlock_page(page);

872
	ri = F2FS_INODE(page);
J
Jaegeuk Kim 已提交
873 874 875 876 877 878 879 880 881
	switch (level) {
	case 0:
	case 1:
		nofs = noffset[1];
		break;
	case 2:
		nofs = noffset[1];
		if (!offset[level - 1])
			goto skip_partial;
882
		err = truncate_partial_nodes(&dn, ri, offset, level);
J
Jaegeuk Kim 已提交
883 884 885 886 887 888 889 890
		if (err < 0 && err != -ENOENT)
			goto fail;
		nofs += 1 + NIDS_PER_BLOCK;
		break;
	case 3:
		nofs = 5 + 2 * NIDS_PER_BLOCK;
		if (!offset[level - 1])
			goto skip_partial;
891
		err = truncate_partial_nodes(&dn, ri, offset, level);
J
Jaegeuk Kim 已提交
892 893 894 895 896 897 898 899 900
		if (err < 0 && err != -ENOENT)
			goto fail;
		break;
	default:
		BUG();
	}

skip_partial:
	while (cont) {
901
		dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
J
Jaegeuk Kim 已提交
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
		switch (offset[0]) {
		case NODE_DIR1_BLOCK:
		case NODE_DIR2_BLOCK:
			err = truncate_dnode(&dn);
			break;

		case NODE_IND1_BLOCK:
		case NODE_IND2_BLOCK:
			err = truncate_nodes(&dn, nofs, offset[1], 2);
			break;

		case NODE_DIND_BLOCK:
			err = truncate_nodes(&dn, nofs, offset[1], 3);
			cont = 0;
			break;

		default:
			BUG();
		}
		if (err < 0 && err != -ENOENT)
			goto fail;
		if (offset[1] == 0 &&
924
				ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) {
J
Jaegeuk Kim 已提交
925
			lock_page(page);
926
			BUG_ON(page->mapping != NODE_MAPPING(sbi));
927
			f2fs_wait_on_page_writeback(page, NODE, true);
928
			ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0;
J
Jaegeuk Kim 已提交
929 930 931 932 933 934 935 936 937
			set_page_dirty(page);
			unlock_page(page);
		}
		offset[1] = 0;
		offset[0]++;
		nofs += err;
	}
fail:
	f2fs_put_page(page, 0);
938
	trace_f2fs_truncate_inode_blocks_exit(inode, err);
J
Jaegeuk Kim 已提交
939 940 941
	return err > 0 ? 0 : err;
}

942 943
int truncate_xattr_node(struct inode *inode, struct page *page)
{
944
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
945 946 947 948 949 950 951 952 953 954 955 956
	nid_t nid = F2FS_I(inode)->i_xattr_nid;
	struct dnode_of_data dn;
	struct page *npage;

	if (!nid)
		return 0;

	npage = get_node_page(sbi, nid);
	if (IS_ERR(npage))
		return PTR_ERR(npage);

	F2FS_I(inode)->i_xattr_nid = 0;
J
Jaegeuk Kim 已提交
957 958 959 960

	/* need to do checkpoint during fsync */
	F2FS_I(inode)->xattr_ver = cur_cp_version(F2FS_CKPT(sbi));

961 962 963
	set_new_dnode(&dn, inode, page, npage, nid);

	if (page)
964
		dn.inode_page_locked = true;
965 966 967 968
	truncate_node(&dn);
	return 0;
}

969
/*
C
Chao Yu 已提交
970 971
 * Caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
972
 */
C
Chao Yu 已提交
973
int remove_inode_page(struct inode *inode)
J
Jaegeuk Kim 已提交
974 975
{
	struct dnode_of_data dn;
C
Chao Yu 已提交
976
	int err;
J
Jaegeuk Kim 已提交
977

978
	set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
C
Chao Yu 已提交
979 980 981
	err = get_dnode_of_data(&dn, 0, LOOKUP_NODE);
	if (err)
		return err;
J
Jaegeuk Kim 已提交
982

C
Chao Yu 已提交
983 984
	err = truncate_xattr_node(inode, dn.inode_page);
	if (err) {
985
		f2fs_put_dnode(&dn);
C
Chao Yu 已提交
986
		return err;
J
Jaegeuk Kim 已提交
987
	}
988 989 990 991 992 993

	/* remove potential inline_data blocks */
	if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
				S_ISLNK(inode->i_mode))
		truncate_data_blocks_range(&dn, 1);

A
arter97 已提交
994
	/* 0 is possible, after f2fs_new_inode() has failed */
995 996
	f2fs_bug_on(F2FS_I_SB(inode),
			inode->i_blocks != 0 && inode->i_blocks != 1);
997 998

	/* will put inode & node pages */
999
	truncate_node(&dn);
C
Chao Yu 已提交
1000
	return 0;
J
Jaegeuk Kim 已提交
1001 1002
}

1003
struct page *new_inode_page(struct inode *inode)
J
Jaegeuk Kim 已提交
1004 1005 1006 1007 1008
{
	struct dnode_of_data dn;

	/* allocate inode page for new inode */
	set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
1009 1010

	/* caller should f2fs_put_page(page, 1); */
1011
	return new_node_page(&dn, 0, NULL);
J
Jaegeuk Kim 已提交
1012 1013
}

1014 1015
struct page *new_node_page(struct dnode_of_data *dn,
				unsigned int ofs, struct page *ipage)
J
Jaegeuk Kim 已提交
1016
{
1017
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
J
Jaegeuk Kim 已提交
1018 1019 1020 1021
	struct node_info old_ni, new_ni;
	struct page *page;
	int err;

1022
	if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
J
Jaegeuk Kim 已提交
1023 1024
		return ERR_PTR(-EPERM);

1025
	page = f2fs_grab_cache_page(NODE_MAPPING(sbi), dn->nid, false);
J
Jaegeuk Kim 已提交
1026 1027 1028
	if (!page)
		return ERR_PTR(-ENOMEM);

1029
	if (unlikely(!inc_valid_node_count(sbi, dn->inode))) {
1030 1031 1032
		err = -ENOSPC;
		goto fail;
	}
J
Jaegeuk Kim 已提交
1033

1034
	get_node_info(sbi, dn->nid, &old_ni);
J
Jaegeuk Kim 已提交
1035 1036

	/* Reinitialize old_ni with new node page */
1037
	f2fs_bug_on(sbi, old_ni.blk_addr != NULL_ADDR);
J
Jaegeuk Kim 已提交
1038 1039
	new_ni = old_ni;
	new_ni.ino = dn->inode->i_ino;
1040
	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
1041

1042
	f2fs_wait_on_page_writeback(page, NODE, true);
1043
	fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true);
1044
	set_cold_node(dn->inode, page);
1045
	SetPageUptodate(page);
1046 1047
	if (set_page_dirty(page))
		dn->node_changed = true;
J
Jaegeuk Kim 已提交
1048

1049
	if (f2fs_has_xattr_block(ofs))
1050 1051
		F2FS_I(dn->inode)->i_xattr_nid = dn->nid;

J
Jaegeuk Kim 已提交
1052
	dn->node_page = page;
1053 1054 1055 1056
	if (ipage)
		update_inode(dn->inode, ipage);
	else
		sync_inode_page(dn);
J
Jaegeuk Kim 已提交
1057 1058 1059 1060 1061 1062
	if (ofs == 0)
		inc_valid_inode_count(sbi);

	return page;

fail:
1063
	clear_node_page_dirty(page);
J
Jaegeuk Kim 已提交
1064 1065 1066 1067
	f2fs_put_page(page, 1);
	return ERR_PTR(err);
}

1068 1069 1070
/*
 * Caller should do after getting the following values.
 * 0: f2fs_put_page(page, 0)
1071
 * LOCKED_PAGE or error: f2fs_put_page(page, 1)
1072
 */
1073
static int read_node_page(struct page *page, int rw)
J
Jaegeuk Kim 已提交
1074
{
1075
	struct f2fs_sb_info *sbi = F2FS_P_SB(page);
J
Jaegeuk Kim 已提交
1076
	struct node_info ni;
1077
	struct f2fs_io_info fio = {
1078
		.sbi = sbi,
1079 1080
		.type = NODE,
		.rw = rw,
1081
		.page = page,
1082
		.encrypted_page = NULL,
1083
	};
J
Jaegeuk Kim 已提交
1084 1085 1086

	get_node_info(sbi, page->index, &ni);

1087
	if (unlikely(ni.blk_addr == NULL_ADDR)) {
1088
		ClearPageUptodate(page);
J
Jaegeuk Kim 已提交
1089
		return -ENOENT;
1090 1091
	}

1092 1093
	if (PageUptodate(page))
		return LOCKED_PAGE;
1094

1095
	fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr;
1096
	return f2fs_submit_page_bio(&fio);
J
Jaegeuk Kim 已提交
1097 1098
}

J
Jaegeuk Kim 已提交
1099
/*
J
Jaegeuk Kim 已提交
1100 1101 1102 1103 1104
 * Readahead a node page
 */
void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct page *apage;
1105
	int err;
J
Jaegeuk Kim 已提交
1106

1107 1108 1109 1110
	if (!nid)
		return;
	f2fs_bug_on(sbi, check_nid_range(sbi, nid));

1111 1112 1113 1114
	rcu_read_lock();
	apage = radix_tree_lookup(&NODE_MAPPING(sbi)->page_tree, nid);
	rcu_read_unlock();
	if (apage)
1115
		return;
J
Jaegeuk Kim 已提交
1116

1117
	apage = f2fs_grab_cache_page(NODE_MAPPING(sbi), nid, false);
J
Jaegeuk Kim 已提交
1118 1119 1120
	if (!apage)
		return;

1121
	err = read_node_page(apage, READA);
1122
	f2fs_put_page(apage, err ? 1 : 0);
J
Jaegeuk Kim 已提交
1123 1124
}

J
Jaegeuk Kim 已提交
1125
static struct page *__get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid,
1126
					struct page *parent, int start)
J
Jaegeuk Kim 已提交
1127 1128
{
	struct page *page;
1129
	int err;
J
Jaegeuk Kim 已提交
1130 1131 1132

	if (!nid)
		return ERR_PTR(-ENOENT);
1133
	f2fs_bug_on(sbi, check_nid_range(sbi, nid));
1134
repeat:
1135
	page = f2fs_grab_cache_page(NODE_MAPPING(sbi), nid, false);
J
Jaegeuk Kim 已提交
1136 1137 1138
	if (!page)
		return ERR_PTR(-ENOMEM);

1139
	err = read_node_page(page, READ_SYNC);
1140 1141
	if (err < 0) {
		f2fs_put_page(page, 1);
J
Jaegeuk Kim 已提交
1142
		return ERR_PTR(err);
1143
	} else if (err == LOCKED_PAGE) {
1144
		goto page_hit;
1145
	}
J
Jaegeuk Kim 已提交
1146

1147
	if (parent)
1148
		ra_node_pages(parent, start + 1, MAX_RA_NODE);
1149

J
Jaegeuk Kim 已提交
1150
	lock_page(page);
1151

C
Chao Yu 已提交
1152 1153 1154 1155
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
	}
1156
	if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
1157 1158 1159
		f2fs_put_page(page, 1);
		goto repeat;
	}
1160
page_hit:
C
Chao Yu 已提交
1161
	f2fs_bug_on(sbi, nid != nid_of_node(page));
J
Jaegeuk Kim 已提交
1162 1163 1164
	return page;
}

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid)
{
	return __get_node_page(sbi, nid, NULL, 0);
}

struct page *get_node_page_ra(struct page *parent, int start)
{
	struct f2fs_sb_info *sbi = F2FS_P_SB(parent);
	nid_t nid = get_nid(parent, start, false);

	return __get_node_page(sbi, nid, parent, start);
}

J
Jaegeuk Kim 已提交
1178 1179
void sync_inode_page(struct dnode_of_data *dn)
{
1180 1181
	int ret = 0;

J
Jaegeuk Kim 已提交
1182
	if (IS_INODE(dn->node_page) || dn->inode_page == dn->node_page) {
1183
		ret = update_inode(dn->inode, dn->node_page);
J
Jaegeuk Kim 已提交
1184 1185 1186
	} else if (dn->inode_page) {
		if (!dn->inode_page_locked)
			lock_page(dn->inode_page);
1187
		ret = update_inode(dn->inode, dn->inode_page);
J
Jaegeuk Kim 已提交
1188 1189 1190
		if (!dn->inode_page_locked)
			unlock_page(dn->inode_page);
	} else {
1191
		ret = update_inode_page(dn->inode);
J
Jaegeuk Kim 已提交
1192
	}
1193
	dn->node_changed = ret ? true: false;
J
Jaegeuk Kim 已提交
1194 1195
}

1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
{
	struct inode *inode;
	struct page *page;

	/* should flush inline_data before evict_inode */
	inode = ilookup(sbi->sb, ino);
	if (!inode)
		return;

1206
	page = pagecache_get_page(inode->i_mapping, 0, FGP_LOCK|FGP_NOWAIT, 0);
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
	if (!page)
		goto iput_out;

	if (!PageUptodate(page))
		goto page_out;

	if (!PageDirty(page))
		goto page_out;

	if (!clear_page_dirty_for_io(page))
		goto page_out;

	if (!f2fs_write_inline_data(inode, page))
		inode_dec_dirty_pages(inode);
	else
		set_page_dirty(page);
page_out:
1224
	f2fs_put_page(page, 1);
1225 1226 1227 1228
iput_out:
	iput(inode);
}

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
void move_node_page(struct page *node_page, int gc_type)
{
	if (gc_type == FG_GC) {
		struct f2fs_sb_info *sbi = F2FS_P_SB(node_page);
		struct writeback_control wbc = {
			.sync_mode = WB_SYNC_ALL,
			.nr_to_write = 1,
			.for_reclaim = 0,
		};

		set_page_dirty(node_page);
		f2fs_wait_on_page_writeback(node_page, NODE, true);

		f2fs_bug_on(sbi, PageWriteback(node_page));
		if (!clear_page_dirty_for_io(node_page))
			goto out_page;

		if (NODE_MAPPING(sbi)->a_ops->writepage(node_page, &wbc))
			unlock_page(node_page);
		goto release_page;
	} else {
		/* set page dirty and write it */
		if (!PageWriteback(node_page))
			set_page_dirty(node_page);
	}
out_page:
	unlock_page(node_page);
release_page:
	f2fs_put_page(node_page, 0);
}

1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino)
{
	pgoff_t index, end;
	struct pagevec pvec;
	struct page *last_page = NULL;

	pagevec_init(&pvec, 0);
	index = 0;
	end = ULONG_MAX;

	while (index <= end) {
		int i, nr_pages;
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
				PAGECACHE_TAG_DIRTY,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			if (unlikely(f2fs_cp_error(sbi))) {
				f2fs_put_page(last_page, 0);
				pagevec_release(&pvec);
				return ERR_PTR(-EIO);
			}

			if (!IS_DNODE(page) || !is_cold_node(page))
				continue;
			if (ino_of_node(page) != ino)
				continue;

			lock_page(page);

			if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
continue_unlock:
				unlock_page(page);
				continue;
			}
			if (ino_of_node(page) != ino)
				goto continue_unlock;

			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

			if (last_page)
				f2fs_put_page(last_page, 0);

			get_page(page);
			last_page = page;
			unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
	}
	return last_page;
}

1320
int fsync_node_pages(struct f2fs_sb_info *sbi, nid_t ino,
1321
			struct writeback_control *wbc, bool atomic)
J
Jaegeuk Kim 已提交
1322 1323 1324
{
	pgoff_t index, end;
	struct pagevec pvec;
1325
	int ret = 0;
1326 1327
	struct page *last_page = NULL;
	bool marked = false;
1328

1329 1330 1331 1332 1333 1334
	if (atomic) {
		last_page = last_fsync_dnode(sbi, ino);
		if (IS_ERR_OR_NULL(last_page))
			return PTR_ERR_OR_ZERO(last_page);
	}
retry:
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
	pagevec_init(&pvec, 0);
	index = 0;
	end = ULONG_MAX;

	while (index <= end) {
		int i, nr_pages;
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
				PAGECACHE_TAG_DIRTY,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			if (unlikely(f2fs_cp_error(sbi))) {
1351
				f2fs_put_page(last_page, 0);
1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
				pagevec_release(&pvec);
				return -EIO;
			}

			if (!IS_DNODE(page) || !is_cold_node(page))
				continue;
			if (ino_of_node(page) != ino)
				continue;

			lock_page(page);

			if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
continue_unlock:
				unlock_page(page);
				continue;
			}
			if (ino_of_node(page) != ino)
				goto continue_unlock;

1371
			if (!PageDirty(page) && page != last_page) {
1372 1373 1374 1375 1376 1377 1378
				/* someone wrote it for us */
				goto continue_unlock;
			}

			f2fs_wait_on_page_writeback(page, NODE, true);
			BUG_ON(PageWriteback(page));

1379 1380 1381 1382
			if (!atomic || page == last_page) {
				set_fsync_mark(page, 1);
				if (IS_INODE(page))
					set_dentry_mark(page,
1383
						need_dentry_mark(sbi, ino));
1384 1385 1386 1387 1388 1389 1390
				/*  may be written by other thread */
				if (!PageDirty(page))
					set_page_dirty(page);
			}

			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;
1391

1392 1393
			ret = NODE_MAPPING(sbi)->a_ops->writepage(page, wbc);
			if (ret) {
1394
				unlock_page(page);
1395 1396 1397 1398 1399 1400
				f2fs_put_page(last_page, 0);
				break;
			}
			if (page == last_page) {
				f2fs_put_page(page, 0);
				marked = true;
1401
				break;
1402
			}
1403 1404 1405 1406
		}
		pagevec_release(&pvec);
		cond_resched();

1407
		if (ret || marked)
1408 1409
			break;
	}
1410 1411 1412 1413 1414 1415 1416 1417 1418
	if (!ret && atomic && !marked) {
		f2fs_msg(sbi->sb, KERN_DEBUG,
			"Retry to write fsync mark: ino=%u, idx=%lx",
					ino, last_page->index);
		lock_page(last_page);
		set_page_dirty(last_page);
		unlock_page(last_page);
		goto retry;
	}
1419
	return ret ? -EIO: 0;
1420 1421 1422 1423 1424 1425 1426
}

int sync_node_pages(struct f2fs_sb_info *sbi, struct writeback_control *wbc)
{
	pgoff_t index, end;
	struct pagevec pvec;
	int step = 0;
1427
	int nwritten = 0;
J
Jaegeuk Kim 已提交
1428 1429 1430 1431 1432

	pagevec_init(&pvec, 0);

next_step:
	index = 0;
1433
	end = ULONG_MAX;
J
Jaegeuk Kim 已提交
1434 1435 1436

	while (index <= end) {
		int i, nr_pages;
1437
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
J
Jaegeuk Kim 已提交
1438 1439 1440 1441 1442 1443 1444 1445
				PAGECACHE_TAG_DIRTY,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

C
Chao Yu 已提交
1446 1447 1448 1449 1450
			if (unlikely(f2fs_cp_error(sbi))) {
				pagevec_release(&pvec);
				return -EIO;
			}

J
Jaegeuk Kim 已提交
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
			/*
			 * flushing sequence with step:
			 * 0. indirect nodes
			 * 1. dentry dnodes
			 * 2. file dnodes
			 */
			if (step == 0 && IS_DNODE(page))
				continue;
			if (step == 1 && (!IS_DNODE(page) ||
						is_cold_node(page)))
				continue;
			if (step == 2 && (!IS_DNODE(page) ||
						!is_cold_node(page)))
				continue;
1465
lock_node:
1466
			if (!trylock_page(page))
J
Jaegeuk Kim 已提交
1467 1468
				continue;

1469
			if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
J
Jaegeuk Kim 已提交
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
continue_unlock:
				unlock_page(page);
				continue;
			}

			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

1480
			/* flush inline_data */
1481
			if (is_inline_node(page)) {
1482 1483 1484
				clear_inline_node(page);
				unlock_page(page);
				flush_inline_data(sbi, ino_of_node(page));
1485
				goto lock_node;
1486 1487
			}

1488 1489 1490
			f2fs_wait_on_page_writeback(page, NODE, true);

			BUG_ON(PageWriteback(page));
J
Jaegeuk Kim 已提交
1491 1492 1493
			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;

1494 1495
			set_fsync_mark(page, 0);
			set_dentry_mark(page, 0);
1496 1497 1498

			if (NODE_MAPPING(sbi)->a_ops->writepage(page, wbc))
				unlock_page(page);
J
Jaegeuk Kim 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518

			if (--wbc->nr_to_write == 0)
				break;
		}
		pagevec_release(&pvec);
		cond_resched();

		if (wbc->nr_to_write == 0) {
			step = 2;
			break;
		}
	}

	if (step < 2) {
		step++;
		goto next_step;
	}
	return nwritten;
}

1519 1520
int wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, nid_t ino)
{
1521
	pgoff_t index = 0, end = ULONG_MAX;
1522 1523 1524 1525
	struct pagevec pvec;
	int ret2 = 0, ret = 0;

	pagevec_init(&pvec, 0);
1526 1527 1528 1529 1530 1531 1532 1533

	while (index <= end) {
		int i, nr_pages;
		nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
				PAGECACHE_TAG_WRITEBACK,
				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
			break;
1534 1535 1536 1537 1538

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
1539
			if (unlikely(page->index > end))
1540 1541
				continue;

1542
			if (ino && ino_of_node(page) == ino) {
1543
				f2fs_wait_on_page_writeback(page, NODE, true);
1544 1545 1546
				if (TestClearPageError(page))
					ret = -EIO;
			}
1547 1548 1549 1550 1551
		}
		pagevec_release(&pvec);
		cond_resched();
	}

1552
	if (unlikely(test_and_clear_bit(AS_ENOSPC, &NODE_MAPPING(sbi)->flags)))
1553
		ret2 = -ENOSPC;
1554
	if (unlikely(test_and_clear_bit(AS_EIO, &NODE_MAPPING(sbi)->flags)))
1555 1556 1557 1558 1559 1560
		ret2 = -EIO;
	if (!ret)
		ret = ret2;
	return ret;
}

J
Jaegeuk Kim 已提交
1561 1562 1563
static int f2fs_write_node_page(struct page *page,
				struct writeback_control *wbc)
{
1564
	struct f2fs_sb_info *sbi = F2FS_P_SB(page);
J
Jaegeuk Kim 已提交
1565 1566
	nid_t nid;
	struct node_info ni;
1567
	struct f2fs_io_info fio = {
1568
		.sbi = sbi,
1569
		.type = NODE,
C
Chris Fries 已提交
1570
		.rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1571
		.page = page,
1572
		.encrypted_page = NULL,
1573
	};
J
Jaegeuk Kim 已提交
1574

1575 1576
	trace_f2fs_writepage(page, NODE);

1577
	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1578
		goto redirty_out;
1579 1580
	if (unlikely(f2fs_cp_error(sbi)))
		goto redirty_out;
1581

J
Jaegeuk Kim 已提交
1582 1583
	/* get old block addr of this node page */
	nid = nid_of_node(page);
1584
	f2fs_bug_on(sbi, page->index != nid);
J
Jaegeuk Kim 已提交
1585

1586 1587 1588 1589 1590 1591 1592
	if (wbc->for_reclaim) {
		if (!down_read_trylock(&sbi->node_write))
			goto redirty_out;
	} else {
		down_read(&sbi->node_write);
	}

J
Jaegeuk Kim 已提交
1593 1594 1595
	get_node_info(sbi, nid, &ni);

	/* This page is already truncated */
1596
	if (unlikely(ni.blk_addr == NULL_ADDR)) {
1597
		ClearPageUptodate(page);
1598
		dec_page_count(sbi, F2FS_DIRTY_NODES);
1599
		up_read(&sbi->node_write);
1600 1601 1602
		unlock_page(page);
		return 0;
	}
J
Jaegeuk Kim 已提交
1603 1604

	set_page_writeback(page);
1605
	fio.old_blkaddr = ni.blk_addr;
1606
	write_node_page(nid, &fio);
1607
	set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(page));
J
Jaegeuk Kim 已提交
1608
	dec_page_count(sbi, F2FS_DIRTY_NODES);
1609
	up_read(&sbi->node_write);
1610 1611 1612 1613

	if (wbc->for_reclaim)
		f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, NODE, WRITE);

J
Jaegeuk Kim 已提交
1614
	unlock_page(page);
1615

1616
	if (unlikely(f2fs_cp_error(sbi)))
1617 1618
		f2fs_submit_merged_bio(sbi, NODE, WRITE);

J
Jaegeuk Kim 已提交
1619
	return 0;
1620 1621

redirty_out:
1622
	redirty_page_for_writepage(wbc, page);
1623
	return AOP_WRITEPAGE_ACTIVATE;
J
Jaegeuk Kim 已提交
1624 1625 1626 1627 1628
}

static int f2fs_write_node_pages(struct address_space *mapping,
			    struct writeback_control *wbc)
{
1629
	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
1630
	long diff;
J
Jaegeuk Kim 已提交
1631

1632 1633
	/* balancing f2fs's metadata in background */
	f2fs_balance_fs_bg(sbi);
J
Jaegeuk Kim 已提交
1634

1635
	/* collect a number of dirty node pages and write together */
1636
	if (get_pages(sbi, F2FS_DIRTY_NODES) < nr_pages_to_skip(sbi, NODE))
1637
		goto skip_write;
1638

Y
Yunlei He 已提交
1639 1640
	trace_f2fs_writepages(mapping->host, wbc, NODE);

1641
	diff = nr_pages_to_write(sbi, NODE, wbc);
1642
	wbc->sync_mode = WB_SYNC_NONE;
1643
	sync_node_pages(sbi, wbc);
1644
	wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
J
Jaegeuk Kim 已提交
1645
	return 0;
1646 1647 1648

skip_write:
	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES);
Y
Yunlei He 已提交
1649
	trace_f2fs_writepages(mapping->host, wbc, NODE);
1650
	return 0;
J
Jaegeuk Kim 已提交
1651 1652 1653 1654
}

static int f2fs_set_node_page_dirty(struct page *page)
{
1655 1656
	trace_f2fs_set_page_dirty(page, NODE);

J
Jaegeuk Kim 已提交
1657 1658 1659
	SetPageUptodate(page);
	if (!PageDirty(page)) {
		__set_page_dirty_nobuffers(page);
1660
		inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES);
J
Jaegeuk Kim 已提交
1661
		SetPagePrivate(page);
J
Jaegeuk Kim 已提交
1662
		f2fs_trace_pid(page);
J
Jaegeuk Kim 已提交
1663 1664 1665 1666 1667
		return 1;
	}
	return 0;
}

J
Jaegeuk Kim 已提交
1668
/*
J
Jaegeuk Kim 已提交
1669 1670 1671 1672 1673 1674
 * Structure of the f2fs node operations
 */
const struct address_space_operations f2fs_node_aops = {
	.writepage	= f2fs_write_node_page,
	.writepages	= f2fs_write_node_pages,
	.set_page_dirty	= f2fs_set_node_page_dirty,
1675 1676
	.invalidatepage	= f2fs_invalidate_page,
	.releasepage	= f2fs_release_page,
J
Jaegeuk Kim 已提交
1677 1678
};

1679 1680
static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
						nid_t n)
J
Jaegeuk Kim 已提交
1681
{
1682
	return radix_tree_lookup(&nm_i->free_nid_root, n);
J
Jaegeuk Kim 已提交
1683 1684
}

1685 1686
static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,
						struct free_nid *i)
J
Jaegeuk Kim 已提交
1687 1688
{
	list_del(&i->list);
1689
	radix_tree_delete(&nm_i->free_nid_root, i->nid);
J
Jaegeuk Kim 已提交
1690 1691
}

1692
static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
J
Jaegeuk Kim 已提交
1693
{
1694
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
1695
	struct free_nid *i;
1696
	struct nat_entry *ne;
J
Jaegeuk Kim 已提交
1697

1698
	if (!available_free_memory(sbi, FREE_NIDS))
H
Haicheng Li 已提交
1699
		return -1;
1700 1701

	/* 0 nid should not be used */
1702
	if (unlikely(nid == 0))
1703
		return 0;
1704

1705 1706 1707
	if (build) {
		/* do not add allocated nids */
		ne = __lookup_nat_cache(nm_i, nid);
1708
		if (ne && (!get_nat_flag(ne, IS_CHECKPOINTED) ||
1709
				nat_get_blkaddr(ne) != NULL_ADDR))
1710
			return 0;
J
Jaegeuk Kim 已提交
1711
	}
1712 1713

	i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS);
J
Jaegeuk Kim 已提交
1714 1715 1716
	i->nid = nid;
	i->state = NID_NEW;

1717 1718 1719 1720 1721
	if (radix_tree_preload(GFP_NOFS)) {
		kmem_cache_free(free_nid_slab, i);
		return 0;
	}

J
Jaegeuk Kim 已提交
1722
	spin_lock(&nm_i->free_nid_list_lock);
1723
	if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {
J
Jaegeuk Kim 已提交
1724
		spin_unlock(&nm_i->free_nid_list_lock);
1725
		radix_tree_preload_end();
J
Jaegeuk Kim 已提交
1726 1727 1728 1729 1730 1731
		kmem_cache_free(free_nid_slab, i);
		return 0;
	}
	list_add_tail(&i->list, &nm_i->free_nid_list);
	nm_i->fcnt++;
	spin_unlock(&nm_i->free_nid_list_lock);
1732
	radix_tree_preload_end();
J
Jaegeuk Kim 已提交
1733 1734 1735 1736 1737 1738
	return 1;
}

static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
{
	struct free_nid *i;
1739 1740
	bool need_free = false;

J
Jaegeuk Kim 已提交
1741
	spin_lock(&nm_i->free_nid_list_lock);
1742
	i = __lookup_free_nid_list(nm_i, nid);
J
Jaegeuk Kim 已提交
1743
	if (i && i->state == NID_NEW) {
1744
		__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
1745
		nm_i->fcnt--;
1746
		need_free = true;
J
Jaegeuk Kim 已提交
1747 1748
	}
	spin_unlock(&nm_i->free_nid_list_lock);
1749 1750 1751

	if (need_free)
		kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1752 1753
}

1754
static void scan_nat_page(struct f2fs_sb_info *sbi,
J
Jaegeuk Kim 已提交
1755 1756
			struct page *nat_page, nid_t start_nid)
{
1757
	struct f2fs_nm_info *nm_i = NM_I(sbi);
J
Jaegeuk Kim 已提交
1758 1759 1760 1761 1762 1763 1764
	struct f2fs_nat_block *nat_blk = page_address(nat_page);
	block_t blk_addr;
	int i;

	i = start_nid % NAT_ENTRY_PER_BLOCK;

	for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
H
Haicheng Li 已提交
1765

1766
		if (unlikely(start_nid >= nm_i->max_nid))
J
Jaegeuk Kim 已提交
1767
			break;
H
Haicheng Li 已提交
1768 1769

		blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
1770
		f2fs_bug_on(sbi, blk_addr == NEW_ADDR);
H
Haicheng Li 已提交
1771
		if (blk_addr == NULL_ADDR) {
1772
			if (add_free_nid(sbi, start_nid, true) < 0)
H
Haicheng Li 已提交
1773 1774
				break;
		}
J
Jaegeuk Kim 已提交
1775 1776 1777 1778 1779 1780 1781
	}
}

static void build_free_nids(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
1782
	struct f2fs_journal *journal = curseg->journal;
1783
	int i = 0;
1784
	nid_t nid = nm_i->next_scan_nid;
J
Jaegeuk Kim 已提交
1785

1786 1787 1788
	/* Enough entries */
	if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)
		return;
J
Jaegeuk Kim 已提交
1789

1790
	/* readahead nat pages to be scanned */
1791 1792
	ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
							META_NAT, true);
J
Jaegeuk Kim 已提交
1793

1794 1795
	down_read(&nm_i->nat_tree_lock);

J
Jaegeuk Kim 已提交
1796 1797 1798
	while (1) {
		struct page *page = get_current_nat_page(sbi, nid);

1799
		scan_nat_page(sbi, page, nid);
J
Jaegeuk Kim 已提交
1800 1801 1802
		f2fs_put_page(page, 1);

		nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
1803
		if (unlikely(nid >= nm_i->max_nid))
J
Jaegeuk Kim 已提交
1804
			nid = 0;
1805

1806
		if (++i >= FREE_NID_PAGES)
J
Jaegeuk Kim 已提交
1807 1808 1809
			break;
	}

1810 1811
	/* go to the next free nat pages to find free nids abundantly */
	nm_i->next_scan_nid = nid;
J
Jaegeuk Kim 已提交
1812 1813

	/* find free nids from current sum_pages */
1814
	down_read(&curseg->journal_rwsem);
1815 1816 1817 1818 1819
	for (i = 0; i < nats_in_cursum(journal); i++) {
		block_t addr;

		addr = le32_to_cpu(nat_in_journal(journal, i).block_addr);
		nid = le32_to_cpu(nid_in_journal(journal, i));
J
Jaegeuk Kim 已提交
1820
		if (addr == NULL_ADDR)
1821
			add_free_nid(sbi, nid, true);
J
Jaegeuk Kim 已提交
1822 1823 1824
		else
			remove_free_nid(nm_i, nid);
	}
1825
	up_read(&curseg->journal_rwsem);
1826
	up_read(&nm_i->nat_tree_lock);
C
Chao Yu 已提交
1827 1828

	ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
C
Chao Yu 已提交
1829
					nm_i->ra_nid_pages, META_NAT, false);
J
Jaegeuk Kim 已提交
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
}

/*
 * If this function returns success, caller can obtain a new nid
 * from second parameter of this function.
 * The returned nid could be used ino as well as nid when inode is created.
 */
bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i = NULL;
retry:
J
Jaegeuk Kim 已提交
1842 1843 1844 1845
#ifdef CONFIG_F2FS_FAULT_INJECTION
	if (time_to_inject(FAULT_ALLOC_NID))
		return false;
#endif
1846
	if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))
1847
		return false;
J
Jaegeuk Kim 已提交
1848 1849 1850

	spin_lock(&nm_i->free_nid_list_lock);

1851
	/* We should not use stale free nids created by build_free_nids */
1852
	if (nm_i->fcnt && !on_build_free_nids(nm_i)) {
1853
		f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));
1854
		list_for_each_entry(i, &nm_i->free_nid_list, list)
1855 1856
			if (i->state == NID_NEW)
				break;
J
Jaegeuk Kim 已提交
1857

1858
		f2fs_bug_on(sbi, i->state != NID_NEW);
1859 1860 1861 1862 1863 1864
		*nid = i->nid;
		i->state = NID_ALLOC;
		nm_i->fcnt--;
		spin_unlock(&nm_i->free_nid_list_lock);
		return true;
	}
J
Jaegeuk Kim 已提交
1865
	spin_unlock(&nm_i->free_nid_list_lock);
1866 1867 1868 1869 1870 1871

	/* Let's scan nat pages and its caches to get free nids */
	mutex_lock(&nm_i->build_lock);
	build_free_nids(sbi);
	mutex_unlock(&nm_i->build_lock);
	goto retry;
J
Jaegeuk Kim 已提交
1872 1873
}

J
Jaegeuk Kim 已提交
1874
/*
J
Jaegeuk Kim 已提交
1875 1876 1877 1878 1879 1880 1881 1882
 * alloc_nid() should be called prior to this function.
 */
void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i;

	spin_lock(&nm_i->free_nid_list_lock);
1883
	i = __lookup_free_nid_list(nm_i, nid);
1884
	f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
1885
	__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
1886
	spin_unlock(&nm_i->free_nid_list_lock);
1887 1888

	kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1889 1890
}

J
Jaegeuk Kim 已提交
1891
/*
J
Jaegeuk Kim 已提交
1892 1893 1894 1895
 * alloc_nid() should be called prior to this function.
 */
void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
{
1896 1897
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i;
1898
	bool need_free = false;
1899

J
Jaegeuk Kim 已提交
1900 1901 1902
	if (!nid)
		return;

1903
	spin_lock(&nm_i->free_nid_list_lock);
1904
	i = __lookup_free_nid_list(nm_i, nid);
1905
	f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
1906
	if (!available_free_memory(sbi, FREE_NIDS)) {
1907
		__del_from_free_nid_list(nm_i, i);
1908
		need_free = true;
1909 1910 1911 1912
	} else {
		i->state = NID_NEW;
		nm_i->fcnt++;
	}
1913
	spin_unlock(&nm_i->free_nid_list_lock);
1914 1915 1916

	if (need_free)
		kmem_cache_free(free_nid_slab, i);
J
Jaegeuk Kim 已提交
1917 1918
}

C
Chao Yu 已提交
1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i, *next;
	int nr = nr_shrink;

	if (!mutex_trylock(&nm_i->build_lock))
		return 0;

	spin_lock(&nm_i->free_nid_list_lock);
	list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {
		if (nr_shrink <= 0 || nm_i->fcnt <= NAT_ENTRY_PER_BLOCK)
			break;
		if (i->state == NID_ALLOC)
			continue;
		__del_from_free_nid_list(nm_i, i);
		kmem_cache_free(free_nid_slab, i);
1936
		nm_i->fcnt--;
C
Chao Yu 已提交
1937 1938 1939 1940 1941 1942 1943 1944
		nr_shrink--;
	}
	spin_unlock(&nm_i->free_nid_list_lock);
	mutex_unlock(&nm_i->build_lock);

	return nr - nr_shrink;
}

1945
void recover_inline_xattr(struct inode *inode, struct page *page)
1946 1947 1948 1949 1950 1951
{
	void *src_addr, *dst_addr;
	size_t inline_size;
	struct page *ipage;
	struct f2fs_inode *ri;

1952
	ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino);
1953
	f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(ipage));
1954

1955 1956 1957 1958 1959 1960
	ri = F2FS_INODE(page);
	if (!(ri->i_inline & F2FS_INLINE_XATTR)) {
		clear_inode_flag(F2FS_I(inode), FI_INLINE_XATTR);
		goto update_inode;
	}

1961 1962 1963 1964
	dst_addr = inline_xattr_addr(ipage);
	src_addr = inline_xattr_addr(page);
	inline_size = inline_xattr_size(inode);

1965
	f2fs_wait_on_page_writeback(ipage, NODE, true);
1966
	memcpy(dst_addr, src_addr, inline_size);
1967
update_inode:
1968 1969 1970 1971
	update_inode(inode, ipage);
	f2fs_put_page(ipage, 1);
}

1972
void recover_xattr_data(struct inode *inode, struct page *page, block_t blkaddr)
1973
{
1974
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984
	nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
	nid_t new_xnid = nid_of_node(page);
	struct node_info ni;

	/* 1: invalidate the previous xattr nid */
	if (!prev_xnid)
		goto recover_xnid;

	/* Deallocate node address */
	get_node_info(sbi, prev_xnid, &ni);
1985
	f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR);
1986 1987
	invalidate_blocks(sbi, ni.blk_addr);
	dec_valid_node_count(sbi, inode);
1988
	set_node_addr(sbi, &ni, NULL_ADDR, false);
1989 1990 1991 1992

recover_xnid:
	/* 2: allocate new xattr nid */
	if (unlikely(!inc_valid_node_count(sbi, inode)))
1993
		f2fs_bug_on(sbi, 1);
1994 1995 1996 1997

	remove_free_nid(NM_I(sbi), new_xnid);
	get_node_info(sbi, new_xnid, &ni);
	ni.ino = inode->i_ino;
1998
	set_node_addr(sbi, &ni, NEW_ADDR, false);
1999 2000 2001 2002
	F2FS_I(inode)->i_xattr_nid = new_xnid;

	/* 3: update xattr blkaddr */
	refresh_sit_entry(sbi, NEW_ADDR, blkaddr);
2003
	set_node_addr(sbi, &ni, blkaddr, false);
2004 2005 2006 2007

	update_inode_page(inode);
}

J
Jaegeuk Kim 已提交
2008 2009
int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
{
2010
	struct f2fs_inode *src, *dst;
J
Jaegeuk Kim 已提交
2011 2012 2013 2014
	nid_t ino = ino_of_node(page);
	struct node_info old_ni, new_ni;
	struct page *ipage;

2015 2016 2017 2018 2019
	get_node_info(sbi, ino, &old_ni);

	if (unlikely(old_ni.blk_addr != NULL_ADDR))
		return -EINVAL;

2020
	ipage = f2fs_grab_cache_page(NODE_MAPPING(sbi), ino, false);
J
Jaegeuk Kim 已提交
2021 2022 2023
	if (!ipage)
		return -ENOMEM;

A
arter97 已提交
2024
	/* Should not use this inode from free nid list */
J
Jaegeuk Kim 已提交
2025 2026 2027 2028 2029
	remove_free_nid(NM_I(sbi), ino);

	SetPageUptodate(ipage);
	fill_node_footer(ipage, ino, ino, 0, true);

2030 2031
	src = F2FS_INODE(page);
	dst = F2FS_INODE(ipage);
J
Jaegeuk Kim 已提交
2032

2033 2034 2035 2036 2037
	memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src);
	dst->i_size = 0;
	dst->i_blocks = cpu_to_le64(1);
	dst->i_links = cpu_to_le32(1);
	dst->i_xattr_nid = 0;
2038
	dst->i_inline = src->i_inline & F2FS_INLINE_XATTR;
J
Jaegeuk Kim 已提交
2039 2040 2041 2042

	new_ni = old_ni;
	new_ni.ino = ino;

2043
	if (unlikely(!inc_valid_node_count(sbi, NULL)))
2044
		WARN_ON(1);
2045
	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
J
Jaegeuk Kim 已提交
2046
	inc_valid_inode_count(sbi);
2047
	set_page_dirty(ipage);
J
Jaegeuk Kim 已提交
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
	f2fs_put_page(ipage, 1);
	return 0;
}

int restore_node_summary(struct f2fs_sb_info *sbi,
			unsigned int segno, struct f2fs_summary_block *sum)
{
	struct f2fs_node *rn;
	struct f2fs_summary *sum_entry;
	block_t addr;
J
Jaegeuk Kim 已提交
2058
	int bio_blocks = MAX_BIO_BLOCKS(sbi);
2059
	int i, idx, last_offset, nrpages;
J
Jaegeuk Kim 已提交
2060 2061 2062 2063 2064 2065

	/* scan the node segment */
	last_offset = sbi->blocks_per_seg;
	addr = START_BLOCK(sbi, segno);
	sum_entry = &sum->entries[0];

2066
	for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
2067
		nrpages = min(last_offset - i, bio_blocks);
2068

A
arter97 已提交
2069
		/* readahead node pages */
2070
		ra_meta_pages(sbi, addr, nrpages, META_POR, true);
J
Jaegeuk Kim 已提交
2071

2072
		for (idx = addr; idx < addr + nrpages; idx++) {
2073
			struct page *page = get_tmp_page(sbi, idx);
2074

2075 2076 2077 2078 2079 2080
			rn = F2FS_NODE(page);
			sum_entry->nid = rn->footer.nid;
			sum_entry->version = 0;
			sum_entry->ofs_in_node = 0;
			sum_entry++;
			f2fs_put_page(page, 1);
2081
		}
2082

2083
		invalidate_mapping_pages(META_MAPPING(sbi), addr,
2084
							addr + nrpages);
J
Jaegeuk Kim 已提交
2085
	}
2086
	return 0;
J
Jaegeuk Kim 已提交
2087 2088
}

2089
static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
J
Jaegeuk Kim 已提交
2090 2091 2092
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2093
	struct f2fs_journal *journal = curseg->journal;
J
Jaegeuk Kim 已提交
2094 2095
	int i;

2096
	down_write(&curseg->journal_rwsem);
2097
	for (i = 0; i < nats_in_cursum(journal); i++) {
J
Jaegeuk Kim 已提交
2098 2099
		struct nat_entry *ne;
		struct f2fs_nat_entry raw_ne;
2100
		nid_t nid = le32_to_cpu(nid_in_journal(journal, i));
J
Jaegeuk Kim 已提交
2101

2102
		raw_ne = nat_in_journal(journal, i);
2103

J
Jaegeuk Kim 已提交
2104 2105
		ne = __lookup_nat_cache(nm_i, nid);
		if (!ne) {
2106 2107
			ne = grab_nat_entry(nm_i, nid);
			node_info_from_raw_nat(&ne->ni, &raw_ne);
J
Jaegeuk Kim 已提交
2108 2109 2110
		}
		__set_nat_cache_dirty(nm_i, ne);
	}
2111
	update_nats_in_cursum(journal, -i);
2112
	up_write(&curseg->journal_rwsem);
J
Jaegeuk Kim 已提交
2113 2114
}

2115 2116
static void __adjust_nat_entry_set(struct nat_entry_set *nes,
						struct list_head *head, int max)
J
Jaegeuk Kim 已提交
2117
{
2118
	struct nat_entry_set *cur;
J
Jaegeuk Kim 已提交
2119

2120 2121
	if (nes->entry_cnt >= max)
		goto add_out;
J
Jaegeuk Kim 已提交
2122

2123 2124 2125 2126 2127
	list_for_each_entry(cur, head, set_list) {
		if (cur->entry_cnt >= nes->entry_cnt) {
			list_add(&nes->set_list, cur->set_list.prev);
			return;
		}
2128
	}
2129 2130 2131
add_out:
	list_add_tail(&nes->set_list, head);
}
J
Jaegeuk Kim 已提交
2132

2133 2134 2135 2136
static void __flush_nat_entry_set(struct f2fs_sb_info *sbi,
					struct nat_entry_set *set)
{
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2137
	struct f2fs_journal *journal = curseg->journal;
2138 2139 2140 2141 2142
	nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
	bool to_journal = true;
	struct f2fs_nat_block *nat_blk;
	struct nat_entry *ne, *cur;
	struct page *page = NULL;
J
Jaegeuk Kim 已提交
2143

2144 2145 2146 2147 2148
	/*
	 * there are two steps to flush nat entries:
	 * #1, flush nat entries to journal in current hot data summary block.
	 * #2, flush nat entries to nat page.
	 */
2149
	if (!__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL))
2150 2151 2152
		to_journal = false;

	if (to_journal) {
2153
		down_write(&curseg->journal_rwsem);
2154 2155 2156 2157 2158
	} else {
		page = get_next_nat_page(sbi, start_nid);
		nat_blk = page_address(page);
		f2fs_bug_on(sbi, !nat_blk);
	}
2159

2160 2161 2162 2163 2164 2165 2166 2167
	/* flush dirty nats in nat entry set */
	list_for_each_entry_safe(ne, cur, &set->entry_list, list) {
		struct f2fs_nat_entry *raw_ne;
		nid_t nid = nat_get_nid(ne);
		int offset;

		if (nat_get_blkaddr(ne) == NEW_ADDR)
			continue;
2168 2169

		if (to_journal) {
2170
			offset = lookup_journal_in_cursum(journal,
2171 2172
							NAT_JOURNAL, nid, 1);
			f2fs_bug_on(sbi, offset < 0);
2173 2174
			raw_ne = &nat_in_journal(journal, offset);
			nid_in_journal(journal, offset) = cpu_to_le32(nid);
2175
		} else {
2176
			raw_ne = &nat_blk->entries[nid - start_nid];
J
Jaegeuk Kim 已提交
2177
		}
2178 2179 2180 2181 2182 2183
		raw_nat_from_node_info(raw_ne, &ne->ni);
		nat_reset_flag(ne);
		__clear_nat_cache_dirty(NM_I(sbi), ne);
		if (nat_get_blkaddr(ne) == NULL_ADDR)
			add_free_nid(sbi, nid, false);
	}
J
Jaegeuk Kim 已提交
2184

2185
	if (to_journal)
2186
		up_write(&curseg->journal_rwsem);
2187 2188
	else
		f2fs_put_page(page, 1);
2189

2190 2191 2192 2193
	f2fs_bug_on(sbi, set->entry_cnt);

	radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
	kmem_cache_free(nat_entry_set_slab, set);
2194
}
2195

2196 2197 2198 2199 2200 2201 2202
/*
 * This function is called during the checkpointing process.
 */
void flush_nat_entries(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2203
	struct f2fs_journal *journal = curseg->journal;
2204
	struct nat_entry_set *setvec[SETVEC_SIZE];
2205 2206 2207 2208 2209
	struct nat_entry_set *set, *tmp;
	unsigned int found;
	nid_t set_idx = 0;
	LIST_HEAD(sets);

2210 2211
	if (!nm_i->dirty_nat_cnt)
		return;
2212 2213 2214

	down_write(&nm_i->nat_tree_lock);

2215 2216 2217 2218 2219
	/*
	 * if there are no enough space in journal to store dirty nat
	 * entries, remove all entries from journal and merge them
	 * into nat entry set.
	 */
2220
	if (!__has_cursum_space(journal, nm_i->dirty_nat_cnt, NAT_JOURNAL))
2221 2222 2223
		remove_nats_in_journal(sbi);

	while ((found = __gang_lookup_nat_set(nm_i,
2224
					set_idx, SETVEC_SIZE, setvec))) {
2225 2226 2227 2228
		unsigned idx;
		set_idx = setvec[found - 1]->set + 1;
		for (idx = 0; idx < found; idx++)
			__adjust_nat_entry_set(setvec[idx], &sets,
2229
						MAX_NAT_JENTRIES(journal));
J
Jaegeuk Kim 已提交
2230
	}
2231

2232 2233 2234 2235
	/* flush dirty nats in nat entry set */
	list_for_each_entry_safe(set, tmp, &sets, set_list)
		__flush_nat_entry_set(sbi, set);

2236 2237
	up_write(&nm_i->nat_tree_lock);

2238
	f2fs_bug_on(sbi, nm_i->dirty_nat_cnt);
J
Jaegeuk Kim 已提交
2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
}

static int init_node_manager(struct f2fs_sb_info *sbi)
{
	struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	unsigned char *version_bitmap;
	unsigned int nat_segs, nat_blocks;

	nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);

	/* segment_count_nat includes pair segment so divide to 2. */
	nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
	nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
2253

2254 2255
	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;

2256
	/* not used nids: 0, node, meta, (and root counted as valid node) */
2257
	nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM;
J
Jaegeuk Kim 已提交
2258 2259
	nm_i->fcnt = 0;
	nm_i->nat_cnt = 0;
2260
	nm_i->ram_thresh = DEF_RAM_THRESHOLD;
C
Chao Yu 已提交
2261
	nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
C
Chao Yu 已提交
2262
	nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
J
Jaegeuk Kim 已提交
2263

2264
	INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
J
Jaegeuk Kim 已提交
2265
	INIT_LIST_HEAD(&nm_i->free_nid_list);
2266 2267
	INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
	INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
J
Jaegeuk Kim 已提交
2268 2269 2270 2271
	INIT_LIST_HEAD(&nm_i->nat_entries);

	mutex_init(&nm_i->build_lock);
	spin_lock_init(&nm_i->free_nid_list_lock);
2272
	init_rwsem(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
2273 2274

	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
A
Alexandru Gheorghiu 已提交
2275
	nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
J
Jaegeuk Kim 已提交
2276 2277 2278 2279
	version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
	if (!version_bitmap)
		return -EFAULT;

A
Alexandru Gheorghiu 已提交
2280 2281 2282 2283
	nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
					GFP_KERNEL);
	if (!nm_i->nat_bitmap)
		return -ENOMEM;
J
Jaegeuk Kim 已提交
2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307
	return 0;
}

int build_node_manager(struct f2fs_sb_info *sbi)
{
	int err;

	sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL);
	if (!sbi->nm_info)
		return -ENOMEM;

	err = init_node_manager(sbi);
	if (err)
		return err;

	build_free_nids(sbi);
	return 0;
}

void destroy_node_manager(struct f2fs_sb_info *sbi)
{
	struct f2fs_nm_info *nm_i = NM_I(sbi);
	struct free_nid *i, *next_i;
	struct nat_entry *natvec[NATVEC_SIZE];
2308
	struct nat_entry_set *setvec[SETVEC_SIZE];
J
Jaegeuk Kim 已提交
2309 2310 2311 2312 2313 2314 2315 2316 2317
	nid_t nid = 0;
	unsigned int found;

	if (!nm_i)
		return;

	/* destroy free nid list */
	spin_lock(&nm_i->free_nid_list_lock);
	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
2318
		f2fs_bug_on(sbi, i->state == NID_ALLOC);
2319
		__del_from_free_nid_list(nm_i, i);
J
Jaegeuk Kim 已提交
2320
		nm_i->fcnt--;
2321 2322 2323
		spin_unlock(&nm_i->free_nid_list_lock);
		kmem_cache_free(free_nid_slab, i);
		spin_lock(&nm_i->free_nid_list_lock);
J
Jaegeuk Kim 已提交
2324
	}
2325
	f2fs_bug_on(sbi, nm_i->fcnt);
J
Jaegeuk Kim 已提交
2326 2327 2328
	spin_unlock(&nm_i->free_nid_list_lock);

	/* destroy nat cache */
2329
	down_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
2330 2331 2332
	while ((found = __gang_lookup_nat_cache(nm_i,
					nid, NATVEC_SIZE, natvec))) {
		unsigned idx;
2333

2334 2335 2336
		nid = nat_get_nid(natvec[found - 1]) + 1;
		for (idx = 0; idx < found; idx++)
			__del_from_nat_cache(nm_i, natvec[idx]);
J
Jaegeuk Kim 已提交
2337
	}
2338
	f2fs_bug_on(sbi, nm_i->nat_cnt);
2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353

	/* destroy nat set cache */
	nid = 0;
	while ((found = __gang_lookup_nat_set(nm_i,
					nid, SETVEC_SIZE, setvec))) {
		unsigned idx;

		nid = setvec[found - 1]->set + 1;
		for (idx = 0; idx < found; idx++) {
			/* entry_cnt is not zero, when cp_error was occurred */
			f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list));
			radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);
			kmem_cache_free(nat_entry_set_slab, setvec[idx]);
		}
	}
2354
	up_write(&nm_i->nat_tree_lock);
J
Jaegeuk Kim 已提交
2355 2356 2357 2358 2359 2360

	kfree(nm_i->nat_bitmap);
	sbi->nm_info = NULL;
	kfree(nm_i);
}

2361
int __init create_node_manager_caches(void)
J
Jaegeuk Kim 已提交
2362 2363
{
	nat_entry_slab = f2fs_kmem_cache_create("nat_entry",
2364
			sizeof(struct nat_entry));
J
Jaegeuk Kim 已提交
2365
	if (!nat_entry_slab)
2366
		goto fail;
J
Jaegeuk Kim 已提交
2367 2368

	free_nid_slab = f2fs_kmem_cache_create("free_nid",
2369
			sizeof(struct free_nid));
2370
	if (!free_nid_slab)
2371
		goto destroy_nat_entry;
2372 2373 2374 2375

	nat_entry_set_slab = f2fs_kmem_cache_create("nat_entry_set",
			sizeof(struct nat_entry_set));
	if (!nat_entry_set_slab)
2376
		goto destroy_free_nid;
J
Jaegeuk Kim 已提交
2377
	return 0;
2378

2379
destroy_free_nid:
2380
	kmem_cache_destroy(free_nid_slab);
2381
destroy_nat_entry:
2382 2383 2384
	kmem_cache_destroy(nat_entry_slab);
fail:
	return -ENOMEM;
J
Jaegeuk Kim 已提交
2385 2386 2387 2388
}

void destroy_node_manager_caches(void)
{
2389
	kmem_cache_destroy(nat_entry_set_slab);
J
Jaegeuk Kim 已提交
2390 2391 2392
	kmem_cache_destroy(free_nid_slab);
	kmem_cache_destroy(nat_entry_slab);
}