gc.c 25.9 KB
Newer Older
J
Jaegeuk Kim 已提交
1
/*
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 * fs/f2fs/gc.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/init.h>
#include <linux/f2fs_fs.h>
#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/freezer.h>

#include "f2fs.h"
#include "node.h"
#include "segment.h"
#include "gc.h"
24
#include <trace/events/f2fs.h>
25 26 27 28

static int gc_thread_func(void *data)
{
	struct f2fs_sb_info *sbi = data;
29
	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
30 31 32
	wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head;
	long wait_ms;

33
	wait_ms = gc_th->min_sleep_time;
34

35
	set_freezable();
36
	do {
37 38 39 40
		wait_event_interruptible_timeout(*wq,
				kthread_should_stop() || freezing(current),
				msecs_to_jiffies(wait_ms));

41 42 43 44 45
		if (try_to_freeze())
			continue;
		if (kthread_should_stop())
			break;

46
		if (sbi->sb->s_writers.frozen >= SB_FREEZE_WRITE) {
47
			increase_sleep_time(gc_th, &wait_ms);
48 49 50
			continue;
		}

51
#ifdef CONFIG_F2FS_FAULT_INJECTION
52 53
		if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
			f2fs_show_injection_info(FAULT_CHECKPOINT);
54
			f2fs_stop_checkpoint(sbi, false);
55
		}
56 57
#endif

58 59 60 61 62 63 64 65
		/*
		 * [GC triggering condition]
		 * 0. GC is not conducted currently.
		 * 1. There are enough dirty segments.
		 * 2. IO subsystem is idle by checking the # of writeback pages.
		 * 3. IO subsystem is idle by checking the # of requests in
		 *    bdev's request list.
		 *
A
arter97 已提交
66
		 * Note) We have to avoid triggering GCs frequently.
67 68 69 70 71 72 73 74
		 * Because it is possible that some segments can be
		 * invalidated soon after by user update or deletion.
		 * So, I'd like to wait some time to collect dirty segments.
		 */
		if (!mutex_trylock(&sbi->gc_mutex))
			continue;

		if (!is_idle(sbi)) {
75
			increase_sleep_time(gc_th, &wait_ms);
76 77 78 79 80
			mutex_unlock(&sbi->gc_mutex);
			continue;
		}

		if (has_enough_invalid_blocks(sbi))
81
			decrease_sleep_time(gc_th, &wait_ms);
82
		else
83
			increase_sleep_time(gc_th, &wait_ms);
84

85
		stat_inc_bggc_count(sbi);
86

87
		/* if return value is not zero, no victim was selected */
88
		if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC), true, NULL_SEGNO))
89
			wait_ms = gc_th->no_gc_sleep_time;
90

91 92 93
		trace_f2fs_background_gc(sbi->sb, wait_ms,
				prefree_segments(sbi), free_segments(sbi));

94 95
		/* balancing f2fs's metadata periodically */
		f2fs_balance_fs_bg(sbi);
96

97 98 99 100 101 102
	} while (!kthread_should_stop());
	return 0;
}

int start_gc_thread(struct f2fs_sb_info *sbi)
{
N
Namjae Jeon 已提交
103
	struct f2fs_gc_kthread *gc_th;
104
	dev_t dev = sbi->sb->s_bdev->bd_dev;
105
	int err = 0;
106

107
	gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL);
108 109 110 111
	if (!gc_th) {
		err = -ENOMEM;
		goto out;
	}
112

113 114 115 116
	gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME;
	gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME;
	gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME;

117 118
	gc_th->gc_idle = 0;

119 120 121
	sbi->gc_thread = gc_th;
	init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head);
	sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
122
			"f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev));
123
	if (IS_ERR(gc_th->f2fs_gc_task)) {
124
		err = PTR_ERR(gc_th->f2fs_gc_task);
125
		kfree(gc_th);
126
		sbi->gc_thread = NULL;
127
	}
128 129
out:
	return err;
130 131 132 133 134 135 136 137 138 139 140 141
}

void stop_gc_thread(struct f2fs_sb_info *sbi)
{
	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
	if (!gc_th)
		return;
	kthread_stop(gc_th->f2fs_gc_task);
	kfree(gc_th);
	sbi->gc_thread = NULL;
}

142
static int select_gc_type(struct f2fs_gc_kthread *gc_th, int gc_type)
143
{
144 145 146 147 148 149 150 151 152
	int gc_mode = (gc_type == BG_GC) ? GC_CB : GC_GREEDY;

	if (gc_th && gc_th->gc_idle) {
		if (gc_th->gc_idle == 1)
			gc_mode = GC_CB;
		else if (gc_th->gc_idle == 2)
			gc_mode = GC_GREEDY;
	}
	return gc_mode;
153 154 155 156 157 158 159
}

static void select_policy(struct f2fs_sb_info *sbi, int gc_type,
			int type, struct victim_sel_policy *p)
{
	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);

160
	if (p->alloc_mode == SSR) {
161 162
		p->gc_mode = GC_GREEDY;
		p->dirty_segmap = dirty_i->dirty_segmap[type];
163
		p->max_search = dirty_i->nr_dirty[type];
164 165
		p->ofs_unit = 1;
	} else {
166
		p->gc_mode = select_gc_type(sbi->gc_thread, gc_type);
167
		p->dirty_segmap = dirty_i->dirty_segmap[DIRTY];
168
		p->max_search = dirty_i->nr_dirty[DIRTY];
169 170
		p->ofs_unit = sbi->segs_per_sec;
	}
171

172 173
	/* we need to check every dirty segments in the FG_GC case */
	if (gc_type != FG_GC && p->max_search > sbi->max_victim_search)
174
		p->max_search = sbi->max_victim_search;
175

176 177 178 179
	/* let's select beginning hot/small space first */
	if (type == CURSEG_HOT_DATA || IS_NODESEG(type))
		p->offset = 0;
	else
180
		p->offset = SIT_I(sbi)->last_victim[p->gc_mode];
181 182 183 184 185
}

static unsigned int get_max_cost(struct f2fs_sb_info *sbi,
				struct victim_sel_policy *p)
{
186 187
	/* SSR allocates in a segment unit */
	if (p->alloc_mode == SSR)
188
		return sbi->blocks_per_seg;
189
	if (p->gc_mode == GC_GREEDY)
190
		return 2 * sbi->blocks_per_seg * p->ofs_unit;
191 192 193 194 195 196 197 198 199
	else if (p->gc_mode == GC_CB)
		return UINT_MAX;
	else /* No other gc_mode */
		return 0;
}

static unsigned int check_bg_victims(struct f2fs_sb_info *sbi)
{
	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
200
	unsigned int secno;
201 202 203 204 205 206

	/*
	 * If the gc_type is FG_GC, we can select victim segments
	 * selected by background GC before.
	 * Those segments guarantee they have small valid blocks.
	 */
207
	for_each_set_bit(secno, dirty_i->victim_secmap, MAIN_SECS(sbi)) {
208
		if (sec_usage_check(sbi, secno))
209
			continue;
210 211 212 213

		if (no_fggc_candidate(sbi, secno))
			continue;

214
		clear_bit(secno, dirty_i->victim_secmap);
215
		return GET_SEG_FROM_SEC(sbi, secno);
216 217 218 219 220 221 222
	}
	return NULL_SEGNO;
}

static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno)
{
	struct sit_info *sit_i = SIT_I(sbi);
223 224
	unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
	unsigned int start = GET_SEG_FROM_SEC(sbi, secno);
225 226 227 228 229 230 231 232
	unsigned long long mtime = 0;
	unsigned int vblocks;
	unsigned char age = 0;
	unsigned char u;
	unsigned int i;

	for (i = 0; i < sbi->segs_per_sec; i++)
		mtime += get_seg_entry(sbi, start + i)->mtime;
233
	vblocks = get_valid_blocks(sbi, segno, true);
234 235 236 237 238 239

	mtime = div_u64(mtime, sbi->segs_per_sec);
	vblocks = div_u64(vblocks, sbi->segs_per_sec);

	u = (vblocks * 100) >> sbi->log_blocks_per_seg;

A
arter97 已提交
240
	/* Handle if the system time has changed by the user */
241 242 243 244 245 246 247 248 249 250 251
	if (mtime < sit_i->min_mtime)
		sit_i->min_mtime = mtime;
	if (mtime > sit_i->max_mtime)
		sit_i->max_mtime = mtime;
	if (sit_i->max_mtime != sit_i->min_mtime)
		age = 100 - div64_u64(100 * (mtime - sit_i->min_mtime),
				sit_i->max_mtime - sit_i->min_mtime);

	return UINT_MAX - ((100 * (100 - u) * age) / (100 + u));
}

252 253 254 255
static unsigned int get_greedy_cost(struct f2fs_sb_info *sbi,
						unsigned int segno)
{
	unsigned int valid_blocks =
256
			get_valid_blocks(sbi, segno, true);
257 258 259 260 261

	return IS_DATASEG(get_seg_entry(sbi, segno)->type) ?
				valid_blocks * 2 : valid_blocks;
}

262 263 264 265 266 267 268 269 270
static unsigned int get_ssr_cost(struct f2fs_sb_info *sbi,
						unsigned int segno)
{
	struct seg_entry *se = get_seg_entry(sbi, segno);

	return se->ckpt_valid_blocks > se->valid_blocks ?
				se->ckpt_valid_blocks : se->valid_blocks;
}

271 272
static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi,
			unsigned int segno, struct victim_sel_policy *p)
273 274
{
	if (p->alloc_mode == SSR)
275
		return get_ssr_cost(sbi, segno);
276 277 278

	/* alloc_mode == LFS */
	if (p->gc_mode == GC_GREEDY)
279
		return get_greedy_cost(sbi, segno);
280 281 282 283
	else
		return get_cb_cost(sbi, segno);
}

284 285 286 287 288 289 290 291 292 293 294 295
static unsigned int count_bits(const unsigned long *addr,
				unsigned int offset, unsigned int len)
{
	unsigned int end = offset + len, sum = 0;

	while (offset < end) {
		if (test_bit(offset++, addr))
			++sum;
	}
	return sum;
}

J
Jaegeuk Kim 已提交
296
/*
M
Masanari Iida 已提交
297
 * This function is called from two paths.
298 299 300 301 302 303 304 305 306 307
 * One is garbage collection and the other is SSR segment selection.
 * When it is called during GC, it just gets a victim segment
 * and it does not remove it from dirty seglist.
 * When it is called from SSR segment selection, it finds a segment
 * which has minimum valid blocks and removes it from dirty seglist.
 */
static int get_victim_by_default(struct f2fs_sb_info *sbi,
		unsigned int *result, int gc_type, int type, char alloc_mode)
{
	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
308
	struct sit_info *sm = SIT_I(sbi);
309
	struct victim_sel_policy p;
S
Sheng Yong 已提交
310
	unsigned int secno, last_victim;
311
	unsigned int last_segment = MAIN_SEGS(sbi);
312
	unsigned int nsearched = 0;
313

314 315
	mutex_lock(&dirty_i->seglist_lock);

316 317 318 319
	p.alloc_mode = alloc_mode;
	select_policy(sbi, gc_type, type, &p);

	p.min_segno = NULL_SEGNO;
S
Sheng Yong 已提交
320
	p.min_cost = get_max_cost(sbi, &p);
321

322 323 324 325 326 327 328 329
	if (*result != NULL_SEGNO) {
		if (IS_DATASEG(get_seg_entry(sbi, *result)->type) &&
			get_valid_blocks(sbi, *result, false) &&
			!sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result)))
			p.min_segno = *result;
		goto out;
	}

330 331 332
	if (p.max_search == 0)
		goto out;

333
	last_victim = sm->last_victim[p.gc_mode];
334 335 336 337 338 339 340 341
	if (p.alloc_mode == LFS && gc_type == FG_GC) {
		p.min_segno = check_bg_victims(sbi);
		if (p.min_segno != NULL_SEGNO)
			goto got_it;
	}

	while (1) {
		unsigned long cost;
342
		unsigned int segno;
343

344 345
		segno = find_next_bit(p.dirty_segmap, last_segment, p.offset);
		if (segno >= last_segment) {
346 347 348 349
			if (sm->last_victim[p.gc_mode]) {
				last_segment =
					sm->last_victim[p.gc_mode];
				sm->last_victim[p.gc_mode] = 0;
350 351 352 353 354
				p.offset = 0;
				continue;
			}
			break;
		}
355 356

		p.offset = segno + p.ofs_unit;
357
		if (p.ofs_unit > 1) {
358
			p.offset -= segno % p.ofs_unit;
359 360 361 362 363 364 365
			nsearched += count_bits(p.dirty_segmap,
						p.offset - p.ofs_unit,
						p.ofs_unit);
		} else {
			nsearched++;
		}

366
		secno = GET_SEC_FROM_SEG(sbi, segno);
367

368
		if (sec_usage_check(sbi, secno))
369
			goto next;
370
		if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap))
371
			goto next;
372 373 374
		if (gc_type == FG_GC && p.alloc_mode == LFS &&
					no_fggc_candidate(sbi, secno))
			goto next;
375 376 377 378 379 380

		cost = get_gc_cost(sbi, segno, &p);

		if (p.min_cost > cost) {
			p.min_segno = segno;
			p.min_cost = cost;
381
		}
382 383
next:
		if (nsearched >= p.max_search) {
384 385
			if (!sm->last_victim[p.gc_mode] && segno <= last_victim)
				sm->last_victim[p.gc_mode] = last_victim + 1;
386
			else
387 388
				sm->last_victim[p.gc_mode] = segno + 1;
			sm->last_victim[p.gc_mode] %= MAIN_SEGS(sbi);
389 390 391 392
			break;
		}
	}
	if (p.min_segno != NULL_SEGNO) {
393
got_it:
394
		if (p.alloc_mode == LFS) {
395
			secno = GET_SEC_FROM_SEG(sbi, p.min_segno);
396 397 398 399
			if (gc_type == FG_GC)
				sbi->cur_victim_sec = secno;
			else
				set_bit(secno, dirty_i->victim_secmap);
400
		}
401
		*result = (p.min_segno / p.ofs_unit) * p.ofs_unit;
402 403 404 405

		trace_f2fs_get_victim(sbi->sb, type, gc_type, &p,
				sbi->cur_victim_sec,
				prefree_segments(sbi), free_segments(sbi));
406
	}
407
out:
408 409 410 411 412 413 414 415 416
	mutex_unlock(&dirty_i->seglist_lock);

	return (p.min_segno == NULL_SEGNO) ? 0 : 1;
}

static const struct victim_selection default_v_ops = {
	.get_victim = get_victim_by_default,
};

417
static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino)
418 419 420
{
	struct inode_entry *ie;

421 422 423
	ie = radix_tree_lookup(&gc_list->iroot, ino);
	if (ie)
		return ie->inode;
424 425 426
	return NULL;
}

427
static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode)
428
{
429 430
	struct inode_entry *new_ie;

431
	if (inode == find_gc_inode(gc_list, inode->i_ino)) {
432 433
		iput(inode);
		return;
434
	}
435
	new_ie = f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS);
436
	new_ie->inode = inode;
437 438

	f2fs_radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie);
439
	list_add_tail(&new_ie->list, &gc_list->ilist);
440 441
}

442
static void put_gc_inode(struct gc_inode_list *gc_list)
443 444
{
	struct inode_entry *ie, *next_ie;
445 446
	list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) {
		radix_tree_delete(&gc_list->iroot, ie->inode->i_ino);
447 448
		iput(ie->inode);
		list_del(&ie->list);
449
		kmem_cache_free(inode_entry_slab, ie);
450 451 452 453 454 455 456 457 458 459 460 461 462 463
	}
}

static int check_valid_map(struct f2fs_sb_info *sbi,
				unsigned int segno, int offset)
{
	struct sit_info *sit_i = SIT_I(sbi);
	struct seg_entry *sentry;
	int ret;

	mutex_lock(&sit_i->sentry_lock);
	sentry = get_seg_entry(sbi, segno);
	ret = f2fs_test_bit(offset, sentry->cur_valid_map);
	mutex_unlock(&sit_i->sentry_lock);
464
	return ret;
465 466
}

J
Jaegeuk Kim 已提交
467
/*
468 469 470 471
 * This function compares node address got in summary with that in NAT.
 * On validity, copy that node with cold status, otherwise (invalid node)
 * ignore that.
 */
C
Chao Yu 已提交
472
static void gc_node_segment(struct f2fs_sb_info *sbi,
473 474 475
		struct f2fs_summary *sum, unsigned int segno, int gc_type)
{
	struct f2fs_summary *entry;
476
	block_t start_addr;
477
	int off;
478
	int phase = 0;
479

480 481
	start_addr = START_BLOCK(sbi, segno);

482 483
next_step:
	entry = sum;
484

485 486 487
	for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
		nid_t nid = le32_to_cpu(entry->nid);
		struct page *node_page;
488
		struct node_info ni;
489

490
		/* stop BG_GC if there is not enough free sections. */
491
		if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0))
C
Chao Yu 已提交
492
			return;
493

494
		if (check_valid_map(sbi, segno, off) == 0)
495 496
			continue;

497 498 499 500 501 502 503
		if (phase == 0) {
			ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
							META_NAT, true);
			continue;
		}

		if (phase == 1) {
504 505 506
			ra_node_page(sbi, nid);
			continue;
		}
507 508

		/* phase == 2 */
509 510 511 512
		node_page = get_node_page(sbi, nid);
		if (IS_ERR(node_page))
			continue;

513 514 515 516
		/* block may become invalid during get_node_page */
		if (check_valid_map(sbi, segno, off) == 0) {
			f2fs_put_page(node_page, 1);
			continue;
517 518 519 520 521 522
		}

		get_node_info(sbi, nid, &ni);
		if (ni.blk_addr != start_addr + off) {
			f2fs_put_page(node_page, 1);
			continue;
523 524
		}

525
		move_node_page(node_page, gc_type);
526
		stat_inc_node_blk_count(sbi, 1, gc_type);
527
	}
528

529
	if (++phase < 3)
530 531 532
		goto next_step;
}

J
Jaegeuk Kim 已提交
533
/*
534 535 536 537 538
 * Calculate start block index indicating the given node offset.
 * Be careful, caller should give this node offset only indicating direct node
 * blocks. If any node offsets, which point the other types of node blocks such
 * as indirect or double indirect node blocks, are given, it must be a caller's
 * bug.
539
 */
540
block_t start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
541
{
542 543
	unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4;
	unsigned int bidx;
544

545 546
	if (node_ofs == 0)
		return 0;
547

548
	if (node_ofs <= 2) {
549 550
		bidx = node_ofs - 1;
	} else if (node_ofs <= indirect_blks) {
551
		int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1);
552 553
		bidx = node_ofs - 2 - dec;
	} else {
554
		int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1);
555 556
		bidx = node_ofs - 5 - dec;
	}
557
	return bidx * ADDRS_PER_BLOCK + ADDRS_PER_INODE(inode);
558 559
}

560
static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
561 562 563 564 565 566 567 568 569 570 571 572
		struct node_info *dni, block_t blkaddr, unsigned int *nofs)
{
	struct page *node_page;
	nid_t nid;
	unsigned int ofs_in_node;
	block_t source_blkaddr;

	nid = le32_to_cpu(sum->nid);
	ofs_in_node = le16_to_cpu(sum->ofs_in_node);

	node_page = get_node_page(sbi, nid);
	if (IS_ERR(node_page))
573
		return false;
574 575 576 577

	get_node_info(sbi, nid, dni);

	if (sum->version != dni->version) {
578 579 580 581
		f2fs_msg(sbi->sb, KERN_WARNING,
				"%s: valid data with mismatched node version.",
				__func__);
		set_sbi_flag(sbi, SBI_NEED_FSCK);
582 583 584 585 586 587 588
	}

	*nofs = ofs_of_node(node_page);
	source_blkaddr = datablock_addr(node_page, ofs_in_node);
	f2fs_put_page(node_page, 1);

	if (source_blkaddr != blkaddr)
589 590
		return false;
	return true;
591 592
}

593 594
static void move_encrypted_block(struct inode *inode, block_t bidx,
							unsigned int segno, int off)
595 596 597 598
{
	struct f2fs_io_info fio = {
		.sbi = F2FS_I_SB(inode),
		.type = DATA,
J
Jaegeuk Kim 已提交
599
		.temp = COLD,
M
Mike Christie 已提交
600
		.op = REQ_OP_READ,
601
		.op_flags = 0,
602
		.encrypted_page = NULL,
603
		.in_list = false,
604 605 606 607 608
	};
	struct dnode_of_data dn;
	struct f2fs_summary sum;
	struct node_info ni;
	struct page *page;
609
	block_t newaddr;
610 611 612
	int err;

	/* do not read out */
613
	page = f2fs_grab_cache_page(inode->i_mapping, bidx, false);
614 615 616
	if (!page)
		return;

617 618 619
	if (!check_valid_map(F2FS_I_SB(inode), segno, off))
		goto out;

C
Chao Yu 已提交
620 621 622
	if (f2fs_is_atomic_file(inode))
		goto out;

623 624 625 626 627
	set_new_dnode(&dn, inode, NULL, NULL, 0);
	err = get_dnode_of_data(&dn, bidx, LOOKUP_NODE);
	if (err)
		goto out;

628 629
	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
		ClearPageUptodate(page);
630
		goto put_out;
631 632 633 634 635 636
	}

	/*
	 * don't cache encrypted data into meta inode until previous dirty
	 * data were writebacked to avoid racing between GC and flush.
	 */
637
	f2fs_wait_on_page_writeback(page, DATA, true);
638 639 640 641 642 643

	get_node_info(fio.sbi, dn.nid, &ni);
	set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);

	/* read page */
	fio.page = page;
644
	fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
645

646
	allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
647
					&sum, CURSEG_COLD_DATA, NULL, false);
648 649 650 651 652 653 654

	fio.encrypted_page = pagecache_get_page(META_MAPPING(fio.sbi), newaddr,
					FGP_LOCK | FGP_CREAT, GFP_NOFS);
	if (!fio.encrypted_page) {
		err = -ENOMEM;
		goto recover_block;
	}
655

656 657 658 659 660 661 662
	err = f2fs_submit_page_bio(&fio);
	if (err)
		goto put_page_out;

	/* write page */
	lock_page(fio.encrypted_page);

663
	if (unlikely(fio.encrypted_page->mapping != META_MAPPING(fio.sbi))) {
664
		err = -EIO;
665
		goto put_page_out;
666
	}
667
	if (unlikely(!PageUptodate(fio.encrypted_page))) {
668
		err = -EIO;
669
		goto put_page_out;
670
	}
671

672
	set_page_dirty(fio.encrypted_page);
673
	f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true);
674 675 676
	if (clear_page_dirty_for_io(fio.encrypted_page))
		dec_page_count(fio.sbi, F2FS_DIRTY_META);

677
	set_page_writeback(fio.encrypted_page);
678 679

	/* allocate block address */
680
	f2fs_wait_on_page_writeback(dn.node_page, NODE, true);
681

M
Mike Christie 已提交
682
	fio.op = REQ_OP_WRITE;
683
	fio.op_flags = REQ_SYNC;
684
	fio.new_blkaddr = newaddr;
685
	f2fs_submit_page_write(&fio);
686

687
	f2fs_update_data_blkaddr(&dn, newaddr);
688
	set_inode_flag(inode, FI_APPEND_WRITE);
689
	if (page->index == 0)
690
		set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
691
put_page_out:
692
	f2fs_put_page(fio.encrypted_page, 1);
693 694 695 696
recover_block:
	if (err)
		__f2fs_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr,
								true, true);
697 698 699 700 701 702
put_out:
	f2fs_put_dnode(&dn);
out:
	f2fs_put_page(page, 1);
}

703 704
static void move_data_page(struct inode *inode, block_t bidx, int gc_type,
							unsigned int segno, int off)
705
{
J
Jaegeuk Kim 已提交
706 707
	struct page *page;

708
	page = get_lock_data_page(inode, bidx, true);
J
Jaegeuk Kim 已提交
709 710
	if (IS_ERR(page))
		return;
711

712 713 714
	if (!check_valid_map(F2FS_I_SB(inode), segno, off))
		goto out;

C
Chao Yu 已提交
715 716 717
	if (f2fs_is_atomic_file(inode))
		goto out;

718
	if (gc_type == BG_GC) {
719 720
		if (PageWriteback(page))
			goto out;
721 722 723
		set_page_dirty(page);
		set_cold_data(page);
	} else {
J
Jaegeuk Kim 已提交
724 725 726
		struct f2fs_io_info fio = {
			.sbi = F2FS_I_SB(inode),
			.type = DATA,
J
Jaegeuk Kim 已提交
727
			.temp = COLD,
M
Mike Christie 已提交
728
			.op = REQ_OP_WRITE,
729
			.op_flags = REQ_SYNC,
730
			.old_blkaddr = NULL_ADDR,
J
Jaegeuk Kim 已提交
731
			.page = page,
732
			.encrypted_page = NULL,
733
			.need_lock = LOCK_REQ,
J
Jaegeuk Kim 已提交
734
		};
735 736 737 738
		bool is_dirty = PageDirty(page);
		int err;

retry:
739
		set_page_dirty(page);
740
		f2fs_wait_on_page_writeback(page, DATA, true);
741
		if (clear_page_dirty_for_io(page)) {
742
			inode_dec_dirty_pages(inode);
743 744
			remove_dirty_inode(inode);
		}
745

746
		set_cold_data(page);
747 748 749 750 751 752

		err = do_write_data_page(&fio);
		if (err == -ENOMEM && is_dirty) {
			congestion_wait(BLK_RW_ASYNC, HZ/50);
			goto retry;
		}
753 754 755 756 757
	}
out:
	f2fs_put_page(page, 1);
}

J
Jaegeuk Kim 已提交
758
/*
759 760 761 762 763 764
 * This function tries to get parent node of victim data block, and identifies
 * data block validity. If the block is valid, copy that with cold status and
 * modify parent node.
 * If the parent node is not valid or the data block address is different,
 * the victim data block is ignored.
 */
C
Chao Yu 已提交
765
static void gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
766
		struct gc_inode_list *gc_list, unsigned int segno, int gc_type)
767 768 769 770
{
	struct super_block *sb = sbi->sb;
	struct f2fs_summary *entry;
	block_t start_addr;
771
	int off;
772 773 774 775 776 777
	int phase = 0;

	start_addr = START_BLOCK(sbi, segno);

next_step:
	entry = sum;
778

779 780 781 782 783 784
	for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
		struct page *data_page;
		struct inode *inode;
		struct node_info dni; /* dnode info for the data */
		unsigned int ofs_in_node, nofs;
		block_t start_bidx;
785
		nid_t nid = le32_to_cpu(entry->nid);
786

787
		/* stop BG_GC if there is not enough free sections. */
788
		if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0))
C
Chao Yu 已提交
789
			return;
790

791
		if (check_valid_map(sbi, segno, off) == 0)
792 793 794
			continue;

		if (phase == 0) {
795 796 797 798 799 800 801
			ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
							META_NAT, true);
			continue;
		}

		if (phase == 1) {
			ra_node_page(sbi, nid);
802 803 804 805
			continue;
		}

		/* Get an inode by ino with checking validity */
806
		if (!is_alive(sbi, entry, &dni, start_addr + off, &nofs))
807 808
			continue;

809
		if (phase == 2) {
810 811 812 813 814 815
			ra_node_page(sbi, dni.ino);
			continue;
		}

		ofs_in_node = le16_to_cpu(entry->ofs_in_node);

816
		if (phase == 3) {
817
			inode = f2fs_iget(sb, dni.ino);
818
			if (IS_ERR(inode) || is_bad_inode(inode))
819 820
				continue;

821 822 823 824 825 826 827
			/* if encrypted inode, let's go phase 3 */
			if (f2fs_encrypted_inode(inode) &&
						S_ISREG(inode->i_mode)) {
				add_gc_inode(gc_list, inode);
				continue;
			}

828
			start_bidx = start_bidx_of_node(nofs, inode);
829
			data_page = get_read_data_page(inode,
830 831
					start_bidx + ofs_in_node, REQ_RAHEAD,
					true);
832 833 834 835
			if (IS_ERR(data_page)) {
				iput(inode);
				continue;
			}
836 837

			f2fs_put_page(data_page, 0);
838
			add_gc_inode(gc_list, inode);
839 840 841
			continue;
		}

842
		/* phase 4 */
843
		inode = find_gc_inode(gc_list, dni.ino);
844
		if (inode) {
845 846 847 848 849 850 851 852 853 854 855 856 857 858
			struct f2fs_inode_info *fi = F2FS_I(inode);
			bool locked = false;

			if (S_ISREG(inode->i_mode)) {
				if (!down_write_trylock(&fi->dio_rwsem[READ]))
					continue;
				if (!down_write_trylock(
						&fi->dio_rwsem[WRITE])) {
					up_write(&fi->dio_rwsem[READ]);
					continue;
				}
				locked = true;
			}

859
			start_bidx = start_bidx_of_node(nofs, inode)
J
Jaegeuk Kim 已提交
860
								+ ofs_in_node;
861
			if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
862
				move_encrypted_block(inode, start_bidx, segno, off);
863
			else
864
				move_data_page(inode, start_bidx, gc_type, segno, off);
865 866 867 868 869 870

			if (locked) {
				up_write(&fi->dio_rwsem[WRITE]);
				up_write(&fi->dio_rwsem[READ]);
			}

871
			stat_inc_data_blk_count(sbi, 1, gc_type);
872 873
		}
	}
874

875
	if (++phase < 5)
876 877 878 879
		goto next_step;
}

static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim,
880
			int gc_type)
881 882 883
{
	struct sit_info *sit_i = SIT_I(sbi);
	int ret;
884

885
	mutex_lock(&sit_i->sentry_lock);
886 887
	ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type,
					      NO_CHECK_TYPE, LFS);
888 889 890 891
	mutex_unlock(&sit_i->sentry_lock);
	return ret;
}

C
Chao Yu 已提交
892 893
static int do_garbage_collect(struct f2fs_sb_info *sbi,
				unsigned int start_segno,
894
				struct gc_inode_list *gc_list, int gc_type)
895 896 897
{
	struct page *sum_page;
	struct f2fs_summary_block *sum;
898
	struct blk_plug plug;
C
Chao Yu 已提交
899 900
	unsigned int segno = start_segno;
	unsigned int end_segno = start_segno + sbi->segs_per_sec;
C
Chao Yu 已提交
901
	int sec_freed = 0;
C
Chao Yu 已提交
902 903
	unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ?
						SUM_TYPE_DATA : SUM_TYPE_NODE;
904

C
Chao Yu 已提交
905 906 907 908 909 910 911 912 913 914
	/* readahead multi ssa blocks those have contiguous address */
	if (sbi->segs_per_sec > 1)
		ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno),
					sbi->segs_per_sec, META_SSA, true);

	/* reference all summary page */
	while (segno < end_segno) {
		sum_page = get_sum_page(sbi, segno++);
		unlock_page(sum_page);
	}
915

916 917
	blk_start_plug(&plug);

C
Chao Yu 已提交
918
	for (segno = start_segno; segno < end_segno; segno++) {
J
Jaegeuk Kim 已提交
919

C
Chao Yu 已提交
920 921 922 923
		/* find segment summary of victim */
		sum_page = find_get_page(META_MAPPING(sbi),
					GET_SUM_BLOCK(sbi, segno));
		f2fs_put_page(sum_page, 0);
924

925
		if (get_valid_blocks(sbi, segno, false) == 0 ||
926 927 928 929
				!PageUptodate(sum_page) ||
				unlikely(f2fs_cp_error(sbi)))
			goto next;

C
Chao Yu 已提交
930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946
		sum = page_address(sum_page);
		f2fs_bug_on(sbi, type != GET_SUM_TYPE((&sum->footer)));

		/*
		 * this is to avoid deadlock:
		 * - lock_page(sum_page)         - f2fs_replace_block
		 *  - check_valid_map()            - mutex_lock(sentry_lock)
		 *   - mutex_lock(sentry_lock)     - change_curseg()
		 *                                  - lock_page(sum_page)
		 */
		if (type == SUM_TYPE_NODE)
			gc_node_segment(sbi, sum->entries, segno, gc_type);
		else
			gc_data_segment(sbi, sum->entries, gc_list, segno,
								gc_type);

		stat_inc_seg_count(sbi, type, gc_type);
947
next:
C
Chao Yu 已提交
948 949 950
		f2fs_put_page(sum_page, 0);
	}

951
	if (gc_type == FG_GC)
952 953
		f2fs_submit_merged_write(sbi,
				(type == SUM_TYPE_NODE) ? NODE : DATA);
954

C
Chao Yu 已提交
955
	blk_finish_plug(&plug);
956

C
Chao Yu 已提交
957
	if (gc_type == FG_GC &&
958
		get_valid_blocks(sbi, start_segno, true) == 0)
C
Chao Yu 已提交
959
		sec_freed = 1;
960 961 962

	stat_inc_call_count(sbi->stat_info);

C
Chao Yu 已提交
963
	return sec_freed;
964 965
}

966 967
int f2fs_gc(struct f2fs_sb_info *sbi, bool sync,
			bool background, unsigned int segno)
968
{
C
Chao Yu 已提交
969
	int gc_type = sync ? FG_GC : BG_GC;
C
Chao Yu 已提交
970
	int sec_freed = 0;
971
	int ret;
972
	struct cp_control cpc;
973
	unsigned int init_segno = segno;
974 975
	struct gc_inode_list gc_list = {
		.ilist = LIST_HEAD_INIT(gc_list.ilist),
976
		.iroot = RADIX_TREE_INIT(GFP_NOFS),
977
	};
978

979
	cpc.reason = __get_cp_reason(sbi);
980
gc_more:
981 982
	if (unlikely(!(sbi->sb->s_flags & MS_ACTIVE))) {
		ret = -EINVAL;
J
Jaegeuk Kim 已提交
983
		goto stop;
984
	}
C
Chao Yu 已提交
985 986
	if (unlikely(f2fs_cp_error(sbi))) {
		ret = -EIO;
987
		goto stop;
C
Chao Yu 已提交
988
	}
989

990
	if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) {
991
		/*
992 993 994
		 * For example, if there are many prefree_segments below given
		 * threshold, we can make them free by checkpoint. Then, we
		 * secure free segments which doesn't need fggc any more.
995
		 */
996 997 998 999 1000
		if (prefree_segments(sbi)) {
			ret = write_checkpoint(sbi, &cpc);
			if (ret)
				goto stop;
		}
1001 1002
		if (has_not_enough_free_secs(sbi, 0, 0))
			gc_type = FG_GC;
1003
	}
1004

1005
	ret = -EINVAL;
1006 1007 1008
	/* f2fs_balance_fs doesn't need to do BG_GC in critical path. */
	if (gc_type == BG_GC && !background)
		goto stop;
1009
	if (!__get_victim(sbi, &segno, gc_type))
J
Jaegeuk Kim 已提交
1010
		goto stop;
1011
	ret = 0;
1012

C
Chao Yu 已提交
1013 1014
	if (do_garbage_collect(sbi, segno, &gc_list, gc_type) &&
			gc_type == FG_GC)
1015
		sec_freed++;
1016

1017
	if (gc_type == FG_GC)
1018
		sbi->cur_victim_sec = NULL_SEGNO;
1019

C
Chao Yu 已提交
1020
	if (!sync) {
1021 1022
		if (has_not_enough_free_secs(sbi, sec_freed, 0)) {
			segno = NULL_SEGNO;
C
Chao Yu 已提交
1023
			goto gc_more;
1024
		}
1025

C
Chao Yu 已提交
1026
		if (gc_type == FG_GC)
1027
			ret = write_checkpoint(sbi, &cpc);
C
Chao Yu 已提交
1028
	}
J
Jaegeuk Kim 已提交
1029
stop:
1030 1031
	SIT_I(sbi)->last_victim[ALLOC_NEXT] = 0;
	SIT_I(sbi)->last_victim[FLUSH_DEVICE] = init_segno;
1032 1033
	mutex_unlock(&sbi->gc_mutex);

1034
	put_gc_inode(&gc_list);
C
Chao Yu 已提交
1035 1036 1037

	if (sync)
		ret = sec_freed ? 0 : -EAGAIN;
1038
	return ret;
1039 1040 1041 1042
}

void build_gc_manager(struct f2fs_sb_info *sbi)
{
1043
	u64 main_count, resv_count, ovp_count;
1044

1045
	DIRTY_I(sbi)->v_ops = &default_v_ops;
1046 1047 1048 1049 1050 1051

	/* threshold of # of valid blocks in a section for victims of FG_GC */
	main_count = SM_I(sbi)->main_segments << sbi->log_blocks_per_seg;
	resv_count = SM_I(sbi)->reserved_segments << sbi->log_blocks_per_seg;
	ovp_count = SM_I(sbi)->ovp_segments << sbi->log_blocks_per_seg;

1052 1053
	sbi->fggc_threshold = div64_u64((main_count - ovp_count) *
				BLKS_PER_SEC(sbi), (main_count - resv_count));
1054 1055 1056 1057 1058

	/* give warm/cold data area from slower device */
	if (sbi->s_ndevs && sbi->segs_per_sec == 1)
		SIT_I(sbi)->last_victim[ALLOC_NEXT] =
				GET_SEGNO(sbi, FDEV(0).end_blk) + 1;
1059
}