fs-writeback.c 31.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 * fs/fs-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * Contains all the functions related to writing back and waiting
 * upon dirty inodes against superblocks, and writing back dirty
 * pages against inodes.  ie: data writeback.  Writeout of the
 * inode itself is not handled here.
 *
11
 * 10Apr2002	Andrew Morton
L
Linus Torvalds 已提交
12 13 14 15 16
 *		Split out of fs/inode.c
 *		Additions for address_space-based writeback
 */

#include <linux/kernel.h>
J
Jens Axboe 已提交
17
#include <linux/module.h>
L
Linus Torvalds 已提交
18
#include <linux/spinlock.h>
19
#include <linux/slab.h>
L
Linus Torvalds 已提交
20 21 22
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
23 24
#include <linux/kthread.h>
#include <linux/freezer.h>
L
Linus Torvalds 已提交
25 26 27 28
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/buffer_head.h>
29
#include <linux/tracepoint.h>
30
#include "internal.h"
L
Linus Torvalds 已提交
31

32 33 34
/*
 * Passed into wb_writeback(), essentially a subset of writeback_control
 */
35
struct wb_writeback_work {
36 37 38
	long nr_pages;
	struct super_block *sb;
	enum writeback_sync_modes sync_mode;
39 40 41
	unsigned int for_kupdate:1;
	unsigned int range_cyclic:1;
	unsigned int for_background:1;
42

43
	struct list_head list;		/* pending work list */
44
	struct completion *done;	/* set if the caller waits */
45 46
};

47 48 49 50 51 52 53 54 55 56 57 58 59
/*
 * Include the creation of the trace points after defining the
 * wb_writeback_work structure so that the definition remains local to this
 * file.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/writeback.h>

/*
 * We don't actually have pdflush, but this one is exported though /proc...
 */
int nr_pdflush_threads;

60 61 62 63
/**
 * writeback_in_progress - determine whether there is writeback in progress
 * @bdi: the device's backing_dev_info structure.
 *
64 65
 * Determine whether there is writeback waiting to be handled against a
 * backing device.
66 67 68
 */
int writeback_in_progress(struct backing_dev_info *bdi)
{
J
Jan Kara 已提交
69
	return test_bit(BDI_writeback_running, &bdi->state);
70 71
}

72 73 74 75
static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
{
	struct super_block *sb = inode->i_sb;

76 77 78 79
	if (strcmp(sb->s_type->name, "bdev") == 0)
		return inode->i_mapping->backing_dev_info;

	return sb->s_bdi;
80 81
}

82 83
static void bdi_queue_work(struct backing_dev_info *bdi,
		struct wb_writeback_work *work)
84
{
85
	trace_writeback_queue(bdi, work);
86

87
	spin_lock_bh(&bdi->wb_lock);
88
	list_add_tail(&work->list, &bdi->work_list);
89 90 91 92 93 94 95
	if (bdi->wb.task) {
		wake_up_process(bdi->wb.task);
	} else {
		/*
		 * The bdi thread isn't there, wake up the forker thread which
		 * will create and run it.
		 */
96
		trace_writeback_nothread(bdi, work);
97
		wake_up_process(default_backing_dev_info.wb.task);
L
Linus Torvalds 已提交
98
	}
99
	spin_unlock_bh(&bdi->wb_lock);
L
Linus Torvalds 已提交
100 101
}

102 103 104
static void
__bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
		bool range_cyclic, bool for_background)
L
Linus Torvalds 已提交
105
{
106
	struct wb_writeback_work *work;
107

108 109 110 111
	/*
	 * This is WB_SYNC_NONE writeback, so if allocation fails just
	 * wakeup the thread for old dirty data writeback
	 */
112 113
	work = kzalloc(sizeof(*work), GFP_ATOMIC);
	if (!work) {
114 115
		if (bdi->wb.task) {
			trace_writeback_nowork(bdi);
116
			wake_up_process(bdi->wb.task);
117
		}
118
		return;
119
	}
120

121 122 123 124
	work->sync_mode	= WB_SYNC_NONE;
	work->nr_pages	= nr_pages;
	work->range_cyclic = range_cyclic;
	work->for_background = for_background;
125

126
	bdi_queue_work(bdi, work);
127 128 129 130 131 132 133 134 135 136
}

/**
 * bdi_start_writeback - start writeback
 * @bdi: the backing device to write from
 * @nr_pages: the number of pages to write
 *
 * Description:
 *   This does WB_SYNC_NONE opportunistic writeback. The IO is only
 *   started when this function returns, we make no guarentees on
137
 *   completion. Caller need not hold sb s_umount semaphore.
138 139
 *
 */
140
void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages)
141
{
142
	__bdi_start_writeback(bdi, nr_pages, true, false);
143
}
144

145 146 147 148 149 150 151 152 153 154 155
/**
 * bdi_start_background_writeback - start background writeback
 * @bdi: the backing device to write from
 *
 * Description:
 *   This does WB_SYNC_NONE background writeback. The IO is only
 *   started when this function returns, we make no guarentees on
 *   completion. Caller need not hold sb s_umount semaphore.
 */
void bdi_start_background_writeback(struct backing_dev_info *bdi)
{
156
	__bdi_start_writeback(bdi, LONG_MAX, true, true);
L
Linus Torvalds 已提交
157 158
}

159 160 161 162 163
/*
 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
 * furthest end of its superblock's dirty-inode list.
 *
 * Before stamping the inode's ->dirtied_when, we check to see whether it is
164
 * already the most-recently-dirtied inode on the b_dirty list.  If that is
165 166 167 168 169
 * the case then the inode must have been redirtied while it was being written
 * out and we don't reset its dirtied_when.
 */
static void redirty_tail(struct inode *inode)
{
170
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
171

172
	if (!list_empty(&wb->b_dirty)) {
173
		struct inode *tail;
174

175
		tail = list_entry(wb->b_dirty.next, struct inode, i_list);
176
		if (time_before(inode->dirtied_when, tail->dirtied_when))
177 178
			inode->dirtied_when = jiffies;
	}
179
	list_move(&inode->i_list, &wb->b_dirty);
180 181
}

182
/*
183
 * requeue inode for re-scanning after bdi->b_io list is exhausted.
184
 */
185
static void requeue_io(struct inode *inode)
186
{
187 188 189
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;

	list_move(&inode->i_list, &wb->b_more_io);
190 191
}

J
Joern Engel 已提交
192 193 194 195 196 197 198 199 200
static void inode_sync_complete(struct inode *inode)
{
	/*
	 * Prevent speculative execution through spin_unlock(&inode_lock);
	 */
	smp_mb();
	wake_up_bit(&inode->i_state, __I_SYNC);
}

201 202 203 204 205 206 207 208
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
	bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
	/*
	 * For inodes being constantly redirtied, dirtied_when can get stuck.
	 * It _appears_ to be in the future, but is actually in distant past.
	 * This test is necessary to prevent such wrapped-around relative times
209
	 * from permanently stopping the whole bdi writeback.
210 211 212 213 214 215
	 */
	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
	return ret;
}

216 217 218 219 220 221 222
/*
 * Move expired dirty inodes from @delaying_queue to @dispatch_queue.
 */
static void move_expired_inodes(struct list_head *delaying_queue,
			       struct list_head *dispatch_queue,
				unsigned long *older_than_this)
{
223 224
	LIST_HEAD(tmp);
	struct list_head *pos, *node;
225
	struct super_block *sb = NULL;
226
	struct inode *inode;
227
	int do_sb_sort = 0;
228

229
	while (!list_empty(delaying_queue)) {
230
		inode = list_entry(delaying_queue->prev, struct inode, i_list);
231
		if (older_than_this &&
232
		    inode_dirtied_after(inode, *older_than_this))
233
			break;
234 235 236
		if (sb && sb != inode->i_sb)
			do_sb_sort = 1;
		sb = inode->i_sb;
237 238 239
		list_move(&inode->i_list, &tmp);
	}

240 241 242 243 244 245
	/* just one sb in list, splice to dispatch_queue and we're done */
	if (!do_sb_sort) {
		list_splice(&tmp, dispatch_queue);
		return;
	}

246 247 248 249 250 251 252 253 254
	/* Move inodes from one superblock together */
	while (!list_empty(&tmp)) {
		inode = list_entry(tmp.prev, struct inode, i_list);
		sb = inode->i_sb;
		list_for_each_prev_safe(pos, node, &tmp) {
			inode = list_entry(pos, struct inode, i_list);
			if (inode->i_sb == sb)
				list_move(&inode->i_list, dispatch_queue);
		}
255 256 257 258 259
	}
}

/*
 * Queue all expired dirty inodes for io, eldest first.
260 261 262 263 264 265 266 267
 * Before
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    gf         edc     BA
 * After
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    g          fBAedc
 *                                           |
 *                                           +--> dequeue for IO
268
 */
269
static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this)
270
{
271
	list_splice_init(&wb->b_more_io, &wb->b_io);
272
	move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this);
273 274
}

275
static int write_inode(struct inode *inode, struct writeback_control *wbc)
276
{
277
	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
278
		return inode->i_sb->s_op->write_inode(inode, wbc);
279
	return 0;
280 281
}

L
Linus Torvalds 已提交
282
/*
283 284 285 286 287 288 289 290
 * Wait for writeback on an inode to complete.
 */
static void inode_wait_for_writeback(struct inode *inode)
{
	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
	wait_queue_head_t *wqh;

	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
291
	 while (inode->i_state & I_SYNC) {
292 293 294
		spin_unlock(&inode_lock);
		__wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
		spin_lock(&inode_lock);
295
	}
296 297 298 299 300 301 302
}

/*
 * Write out an inode's dirty pages.  Called under inode_lock.  Either the
 * caller has ref on the inode (either via __iget or via syscall against an fd)
 * or the inode has I_WILL_FREE set (via generic_forget_inode)
 *
L
Linus Torvalds 已提交
303 304 305 306 307 308 309 310 311
 * If `wait' is set, wait on the writeout.
 *
 * The whole writeout design is quite complex and fragile.  We want to avoid
 * starvation of particular inodes when others are being redirtied, prevent
 * livelocks, etc.
 *
 * Called under inode_lock.
 */
static int
312
writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
L
Linus Torvalds 已提交
313 314
{
	struct address_space *mapping = inode->i_mapping;
315
	unsigned dirty;
L
Linus Torvalds 已提交
316 317
	int ret;

318 319 320 321 322 323 324 325
	if (!atomic_read(&inode->i_count))
		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
	else
		WARN_ON(inode->i_state & I_WILL_FREE);

	if (inode->i_state & I_SYNC) {
		/*
		 * If this inode is locked for writeback and we are not doing
326
		 * writeback-for-data-integrity, move it to b_more_io so that
327 328 329
		 * writeback can proceed with the other inodes on s_io.
		 *
		 * We'll have another go at writing back this inode when we
330
		 * completed a full scan of b_io.
331
		 */
332
		if (wbc->sync_mode != WB_SYNC_ALL) {
333 334 335 336 337 338 339 340 341 342
			requeue_io(inode);
			return 0;
		}

		/*
		 * It's a data-integrity sync.  We must wait.
		 */
		inode_wait_for_writeback(inode);
	}

J
Joern Engel 已提交
343
	BUG_ON(inode->i_state & I_SYNC);
L
Linus Torvalds 已提交
344

345
	/* Set I_SYNC, reset I_DIRTY_PAGES */
J
Joern Engel 已提交
346
	inode->i_state |= I_SYNC;
347
	inode->i_state &= ~I_DIRTY_PAGES;
L
Linus Torvalds 已提交
348 349 350 351
	spin_unlock(&inode_lock);

	ret = do_writepages(mapping, wbc);

352 353 354 355 356
	/*
	 * Make sure to wait on the data before writing out the metadata.
	 * This is important for filesystems that modify metadata on data
	 * I/O completion.
	 */
357
	if (wbc->sync_mode == WB_SYNC_ALL) {
358
		int err = filemap_fdatawait(mapping);
L
Linus Torvalds 已提交
359 360 361 362
		if (ret == 0)
			ret = err;
	}

363 364 365 366 367 368 369 370 371
	/*
	 * Some filesystems may redirty the inode during the writeback
	 * due to delalloc, clear dirty metadata flags right before
	 * write_inode()
	 */
	spin_lock(&inode_lock);
	dirty = inode->i_state & I_DIRTY;
	inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC);
	spin_unlock(&inode_lock);
372 373
	/* Don't write the inode if only I_DIRTY_PAGES was set */
	if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
374
		int err = write_inode(inode, wbc);
L
Linus Torvalds 已提交
375 376 377 378 379
		if (ret == 0)
			ret = err;
	}

	spin_lock(&inode_lock);
J
Joern Engel 已提交
380
	inode->i_state &= ~I_SYNC;
A
Al Viro 已提交
381
	if (!(inode->i_state & I_FREEING)) {
382
		if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
L
Linus Torvalds 已提交
383 384
			/*
			 * We didn't write back all the pages.  nfs_writepages()
385
			 * sometimes bales out without doing anything.
386
			 */
387 388
			inode->i_state |= I_DIRTY_PAGES;
			if (wbc->nr_to_write <= 0) {
L
Linus Torvalds 已提交
389
				/*
390
				 * slice used up: queue for next turn
L
Linus Torvalds 已提交
391
				 */
392
				requeue_io(inode);
L
Linus Torvalds 已提交
393 394
			} else {
				/*
395 396 397 398 399
				 * Writeback blocked by something other than
				 * congestion. Delay the inode for some time to
				 * avoid spinning on the CPU (100% iowait)
				 * retrying writeback of the dirty page/inode
				 * that cannot be performed immediately.
L
Linus Torvalds 已提交
400
				 */
401
				redirty_tail(inode);
L
Linus Torvalds 已提交
402
			}
403 404 405 406 407 408 409 410
		} else if (inode->i_state & I_DIRTY) {
			/*
			 * Filesystems can dirty the inode during writeback
			 * operations, such as delayed allocation during
			 * submission or metadata updates after data IO
			 * completion.
			 */
			redirty_tail(inode);
L
Linus Torvalds 已提交
411 412 413 414 415 416 417 418 419 420 421 422
		} else if (atomic_read(&inode->i_count)) {
			/*
			 * The inode is clean, inuse
			 */
			list_move(&inode->i_list, &inode_in_use);
		} else {
			/*
			 * The inode is clean, unused
			 */
			list_move(&inode->i_list, &inode_unused);
		}
	}
J
Joern Engel 已提交
423
	inode_sync_complete(inode);
L
Linus Torvalds 已提交
424 425 426
	return ret;
}

427
/*
428
 * For background writeback the caller does not have the sb pinned
429 430 431
 * before calling writeback. So make sure that we do pin it, so it doesn't
 * go away while we are writing inodes from it.
 */
432
static bool pin_sb_for_writeback(struct super_block *sb)
433 434
{
	spin_lock(&sb_lock);
435 436 437 438 439
	if (list_empty(&sb->s_instances)) {
		spin_unlock(&sb_lock);
		return false;
	}

440
	sb->s_count++;
441 442
	spin_unlock(&sb_lock);

443
	if (down_read_trylock(&sb->s_umount)) {
444
		if (sb->s_root)
445
			return true;
446 447
		up_read(&sb->s_umount);
	}
448 449

	put_super(sb);
450
	return false;
451 452
}

453 454
/*
 * Write a portion of b_io inodes which belong to @sb.
455 456
 *
 * If @only_this_sb is true, then find and write all such
457 458
 * inodes. Otherwise write only ones which go sequentially
 * in reverse order.
459
 *
460 461 462
 * Return 1, if the caller writeback routine should be
 * interrupted. Otherwise return 0.
 */
463 464
static int writeback_sb_inodes(struct super_block *sb, struct bdi_writeback *wb,
		struct writeback_control *wbc, bool only_this_sb)
L
Linus Torvalds 已提交
465
{
466
	while (!list_empty(&wb->b_io)) {
L
Linus Torvalds 已提交
467
		long pages_skipped;
468 469
		struct inode *inode = list_entry(wb->b_io.prev,
						 struct inode, i_list);
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486

		if (inode->i_sb != sb) {
			if (only_this_sb) {
				/*
				 * We only want to write back data for this
				 * superblock, move all inodes not belonging
				 * to it back onto the dirty list.
				 */
				redirty_tail(inode);
				continue;
			}

			/*
			 * The inode belongs to a different superblock.
			 * Bounce back to the caller to unpin this and
			 * pin the next superblock.
			 */
487
			return 0;
488 489
		}

490
		if (inode->i_state & (I_NEW | I_WILL_FREE)) {
491 492 493
			requeue_io(inode);
			continue;
		}
494 495 496 497
		/*
		 * Was this inode dirtied after sync_sb_inodes was called?
		 * This keeps sync from extra jobs and livelock.
		 */
498 499
		if (inode_dirtied_after(inode, wbc->wb_start))
			return 1;
L
Linus Torvalds 已提交
500

A
Al Viro 已提交
501
		BUG_ON(inode->i_state & I_FREEING);
L
Linus Torvalds 已提交
502 503
		__iget(inode);
		pages_skipped = wbc->pages_skipped;
504
		writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
505 506 507 508 509
		if (wbc->pages_skipped != pages_skipped) {
			/*
			 * writeback is not making progress due to locked
			 * buffers.  Skip this inode for now.
			 */
510
			redirty_tail(inode);
L
Linus Torvalds 已提交
511 512 513
		}
		spin_unlock(&inode_lock);
		iput(inode);
514
		cond_resched();
L
Linus Torvalds 已提交
515
		spin_lock(&inode_lock);
516 517
		if (wbc->nr_to_write <= 0) {
			wbc->more_io = 1;
518
			return 1;
519
		}
520
		if (!list_empty(&wb->b_more_io))
521
			wbc->more_io = 1;
L
Linus Torvalds 已提交
522
	}
523 524 525 526
	/* b_io is empty */
	return 1;
}

527 528
void writeback_inodes_wb(struct bdi_writeback *wb,
		struct writeback_control *wbc)
529 530 531
{
	int ret = 0;

532 533
	if (!wbc->wb_start)
		wbc->wb_start = jiffies; /* livelock avoidance */
534 535 536
	spin_lock(&inode_lock);
	if (!wbc->for_kupdate || list_empty(&wb->b_io))
		queue_io(wb, wbc->older_than_this);
N
Nick Piggin 已提交
537

538 539 540 541
	while (!list_empty(&wb->b_io)) {
		struct inode *inode = list_entry(wb->b_io.prev,
						 struct inode, i_list);
		struct super_block *sb = inode->i_sb;
542

543 544 545
		if (!pin_sb_for_writeback(sb)) {
			requeue_io(inode);
			continue;
546
		}
547 548
		ret = writeback_sb_inodes(sb, wb, wbc, false);
		drop_super(sb);
549 550 551 552

		if (ret)
			break;
	}
553 554 555 556
	spin_unlock(&inode_lock);
	/* Leave any unwritten inodes on b_io */
}

557 558 559 560 561 562 563 564 565 566 567 568
static void __writeback_inodes_sb(struct super_block *sb,
		struct bdi_writeback *wb, struct writeback_control *wbc)
{
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

	spin_lock(&inode_lock);
	if (!wbc->for_kupdate || list_empty(&wb->b_io))
		queue_io(wb, wbc->older_than_this);
	writeback_sb_inodes(sb, wb, wbc, true);
	spin_unlock(&inode_lock);
}

569
/*
570 571 572 573 574 575 576 577 578 579 580 581
 * The maximum number of pages to writeout in a single bdi flush/kupdate
 * operation.  We do this so we don't hold I_SYNC against an inode for
 * enormous amounts of time, which would block a userspace task which has
 * been forced to throttle against that inode.  Also, the code reevaluates
 * the dirty each time it has written this many pages.
 */
#define MAX_WRITEBACK_PAGES     1024

static inline bool over_bground_thresh(void)
{
	unsigned long background_thresh, dirty_thresh;

582
	global_dirty_limits(&background_thresh, &dirty_thresh);
583 584 585 586 587 588 589

	return (global_page_state(NR_FILE_DIRTY) +
		global_page_state(NR_UNSTABLE_NFS) >= background_thresh);
}

/*
 * Explicit flushing or periodic writeback of "old" data.
590
 *
591 592 593 594
 * Define "old": the first time one of an inode's pages is dirtied, we mark the
 * dirtying-time in the inode's address_space.  So this periodic writeback code
 * just walks the superblock inode list, writing back any inodes which are
 * older than a specific point in time.
595
 *
596 597 598
 * Try to run once per dirty_writeback_interval.  But if a writeback event
 * takes longer than a dirty_writeback_interval interval, then leave a
 * one-second gap.
599
 *
600 601
 * older_than_this takes precedence over nr_to_write.  So we'll only write back
 * all dirty pages if they are all attached to "old" mappings.
602
 */
603
static long wb_writeback(struct bdi_writeback *wb,
604
			 struct wb_writeback_work *work)
605
{
606
	struct writeback_control wbc = {
607
		.sync_mode		= work->sync_mode,
608
		.older_than_this	= NULL,
609 610 611
		.for_kupdate		= work->for_kupdate,
		.for_background		= work->for_background,
		.range_cyclic		= work->range_cyclic,
612 613 614
	};
	unsigned long oldest_jif;
	long wrote = 0;
J
Jan Kara 已提交
615
	struct inode *inode;
616

617 618 619 620 621
	if (wbc.for_kupdate) {
		wbc.older_than_this = &oldest_jif;
		oldest_jif = jiffies -
				msecs_to_jiffies(dirty_expire_interval * 10);
	}
622 623 624 625
	if (!wbc.range_cyclic) {
		wbc.range_start = 0;
		wbc.range_end = LLONG_MAX;
	}
N
Nick Piggin 已提交
626

627
	wbc.wb_start = jiffies; /* livelock avoidance */
628 629
	for (;;) {
		/*
630
		 * Stop writeback when nr_pages has been consumed
631
		 */
632
		if (work->nr_pages <= 0)
633
			break;
634

N
Nick Piggin 已提交
635
		/*
636 637
		 * For background writeout, stop when we are below the
		 * background dirty threshold
N
Nick Piggin 已提交
638
		 */
639
		if (work->for_background && !over_bground_thresh())
640
			break;
N
Nick Piggin 已提交
641

642 643 644
		wbc.more_io = 0;
		wbc.nr_to_write = MAX_WRITEBACK_PAGES;
		wbc.pages_skipped = 0;
645 646

		trace_wbc_writeback_start(&wbc, wb->bdi);
647 648
		if (work->sb)
			__writeback_inodes_sb(work->sb, wb, &wbc);
649 650
		else
			writeback_inodes_wb(wb, &wbc);
651 652
		trace_wbc_writeback_written(&wbc, wb->bdi);

653
		work->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
654 655 656
		wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write;

		/*
657
		 * If we consumed everything, see if we have more
658
		 */
659 660 661 662 663 664
		if (wbc.nr_to_write <= 0)
			continue;
		/*
		 * Didn't write everything and we don't have more IO, bail
		 */
		if (!wbc.more_io)
665
			break;
666 667 668 669 670 671 672 673 674 675 676 677 678 679
		/*
		 * Did we write something? Try for more
		 */
		if (wbc.nr_to_write < MAX_WRITEBACK_PAGES)
			continue;
		/*
		 * Nothing written. Wait for some inode to
		 * become available for writeback. Otherwise
		 * we'll just busyloop.
		 */
		spin_lock(&inode_lock);
		if (!list_empty(&wb->b_more_io))  {
			inode = list_entry(wb->b_more_io.prev,
						struct inode, i_list);
680
			trace_wbc_writeback_wait(&wbc, wb->bdi);
681
			inode_wait_for_writeback(inode);
682
		}
683
		spin_unlock(&inode_lock);
684 685 686 687 688 689
	}

	return wrote;
}

/*
690
 * Return the next wb_writeback_work struct that hasn't been processed yet.
691
 */
692
static struct wb_writeback_work *
693
get_next_work_item(struct backing_dev_info *bdi)
694
{
695
	struct wb_writeback_work *work = NULL;
696

697
	spin_lock_bh(&bdi->wb_lock);
698 699 700 701
	if (!list_empty(&bdi->work_list)) {
		work = list_entry(bdi->work_list.next,
				  struct wb_writeback_work, list);
		list_del_init(&work->list);
702
	}
703
	spin_unlock_bh(&bdi->wb_lock);
704
	return work;
705 706 707 708 709 710 711
}

static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
	unsigned long expired;
	long nr_pages;

712 713 714 715 716 717
	/*
	 * When set to zero, disable periodic writeback
	 */
	if (!dirty_writeback_interval)
		return 0;

718 719 720 721 722 723 724 725
	expired = wb->last_old_flush +
			msecs_to_jiffies(dirty_writeback_interval * 10);
	if (time_before(jiffies, expired))
		return 0;

	wb->last_old_flush = jiffies;
	nr_pages = global_page_state(NR_FILE_DIRTY) +
			global_page_state(NR_UNSTABLE_NFS) +
726
			get_nr_dirty_inodes();
727

728
	if (nr_pages) {
729
		struct wb_writeback_work work = {
730 731 732 733 734 735
			.nr_pages	= nr_pages,
			.sync_mode	= WB_SYNC_NONE,
			.for_kupdate	= 1,
			.range_cyclic	= 1,
		};

736
		return wb_writeback(wb, &work);
737
	}
738 739 740 741 742 743 744 745 746 747

	return 0;
}

/*
 * Retrieve work items and do the writeback they describe
 */
long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
{
	struct backing_dev_info *bdi = wb->bdi;
748
	struct wb_writeback_work *work;
749
	long wrote = 0;
750

J
Jan Kara 已提交
751
	set_bit(BDI_writeback_running, &wb->bdi->state);
752
	while ((work = get_next_work_item(bdi)) != NULL) {
753 754
		/*
		 * Override sync mode, in case we must wait for completion
755
		 * because this thread is exiting now.
756 757
		 */
		if (force_wait)
758
			work->sync_mode = WB_SYNC_ALL;
759

760 761
		trace_writeback_exec(bdi, work);

762
		wrote += wb_writeback(wb, work);
763 764

		/*
765 766
		 * Notify the caller of completion if this is a synchronous
		 * work item, otherwise just free it.
767
		 */
768 769 770 771
		if (work->done)
			complete(work->done);
		else
			kfree(work);
772 773 774 775 776 777
	}

	/*
	 * Check for periodic writeback, kupdated() style
	 */
	wrote += wb_check_old_data_flush(wb);
J
Jan Kara 已提交
778
	clear_bit(BDI_writeback_running, &wb->bdi->state);
779 780 781 782 783 784 785 786

	return wrote;
}

/*
 * Handle writeback of dirty data for the device backed by this bdi. Also
 * wakes up periodically and does kupdated style flushing.
 */
787
int bdi_writeback_thread(void *data)
788
{
789 790
	struct bdi_writeback *wb = data;
	struct backing_dev_info *bdi = wb->bdi;
791 792
	long pages_written;

793 794
	current->flags |= PF_FLUSHER | PF_SWAPWRITE;
	set_freezable();
795
	wb->last_active = jiffies;
796 797 798 799 800 801

	/*
	 * Our parent may run at a different priority, just set us to normal
	 */
	set_user_nice(current, 0);

802 803
	trace_writeback_thread_start(bdi);

804
	while (!kthread_should_stop()) {
805 806 807 808 809 810
		/*
		 * Remove own delayed wake-up timer, since we are already awake
		 * and we'll take care of the preriodic write-back.
		 */
		del_timer(&wb->wakeup_timer);

811 812
		pages_written = wb_do_writeback(wb, 0);

813 814
		trace_writeback_pages_written(pages_written);

815
		if (pages_written)
816
			wb->last_active = jiffies;
817

818
		set_current_state(TASK_INTERRUPTIBLE);
819
		if (!list_empty(&bdi->work_list) || kthread_should_stop()) {
820
			__set_current_state(TASK_RUNNING);
821
			continue;
822 823
		}

824
		if (wb_has_dirty_io(wb) && dirty_writeback_interval)
825
			schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10));
826 827 828 829 830 831
		else {
			/*
			 * We have nothing to do, so can go sleep without any
			 * timeout and save power. When a work is queued or
			 * something is made dirty - we will be woken up.
			 */
832
			schedule();
833
		}
834

835 836 837
		try_to_freeze();
	}

838
	/* Flush any work that raced with us exiting */
839 840
	if (!list_empty(&bdi->work_list))
		wb_do_writeback(wb, 1);
841 842

	trace_writeback_thread_stop(bdi);
843 844 845
	return 0;
}

846

847
/*
848 849
 * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
 * the whole world.
850
 */
851
void wakeup_flusher_threads(long nr_pages)
852
{
853
	struct backing_dev_info *bdi;
854

855 856
	if (!nr_pages) {
		nr_pages = global_page_state(NR_FILE_DIRTY) +
857 858
				global_page_state(NR_UNSTABLE_NFS);
	}
859

860
	rcu_read_lock();
861
	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
862 863
		if (!bdi_has_dirty_io(bdi))
			continue;
864
		__bdi_start_writeback(bdi, nr_pages, false, false);
865
	}
866
	rcu_read_unlock();
L
Linus Torvalds 已提交
867 868
}

869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896
static noinline void block_dump___mark_inode_dirty(struct inode *inode)
{
	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
		struct dentry *dentry;
		const char *name = "?";

		dentry = d_find_alias(inode);
		if (dentry) {
			spin_lock(&dentry->d_lock);
			name = (const char *) dentry->d_name.name;
		}
		printk(KERN_DEBUG
		       "%s(%d): dirtied inode %lu (%s) on %s\n",
		       current->comm, task_pid_nr(current), inode->i_ino,
		       name, inode->i_sb->s_id);
		if (dentry) {
			spin_unlock(&dentry->d_lock);
			dput(dentry);
		}
	}
}

/**
 *	__mark_inode_dirty -	internal function
 *	@inode: inode to mark
 *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
 *	Mark an inode as dirty. Callers should use mark_inode_dirty or
 *  	mark_inode_dirty_sync.
L
Linus Torvalds 已提交
897
 *
898 899 900 901 902 903 904 905 906
 * Put the inode on the super block's dirty list.
 *
 * CAREFUL! We mark it dirty unconditionally, but move it onto the
 * dirty list only if it is hashed or if it refers to a blockdev.
 * If it was not hashed, it will never be added to the dirty list
 * even if it is later hashed, as it will have been marked dirty already.
 *
 * In short, make sure you hash any inodes _before_ you start marking
 * them dirty.
L
Linus Torvalds 已提交
907
 *
908 909
 * This function *must* be atomic for the I_DIRTY_PAGES case -
 * set_page_dirty() is called under spinlock in several places.
L
Linus Torvalds 已提交
910
 *
911 912 913 914 915 916
 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
 * the kernel-internal blockdev inode represents the dirtying time of the
 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
 * page->mapping->host, so the page-dirtying time is recorded in the internal
 * blockdev inode.
L
Linus Torvalds 已提交
917
 */
918
void __mark_inode_dirty(struct inode *inode, int flags)
L
Linus Torvalds 已提交
919
{
920
	struct super_block *sb = inode->i_sb;
921 922
	struct backing_dev_info *bdi = NULL;
	bool wakeup_bdi = false;
L
Linus Torvalds 已提交
923

924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964
	/*
	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
	 * dirty the inode itself
	 */
	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
		if (sb->s_op->dirty_inode)
			sb->s_op->dirty_inode(inode);
	}

	/*
	 * make sure that changes are seen by all cpus before we test i_state
	 * -- mikulas
	 */
	smp_mb();

	/* avoid the locking if we can */
	if ((inode->i_state & flags) == flags)
		return;

	if (unlikely(block_dump))
		block_dump___mark_inode_dirty(inode);

	spin_lock(&inode_lock);
	if ((inode->i_state & flags) != flags) {
		const int was_dirty = inode->i_state & I_DIRTY;

		inode->i_state |= flags;

		/*
		 * If the inode is being synced, just update its dirty state.
		 * The unlocker will place the inode on the appropriate
		 * superblock list, based upon its state.
		 */
		if (inode->i_state & I_SYNC)
			goto out;

		/*
		 * Only add valid (hashed) inodes to the superblock's
		 * dirty list.  Add blockdev inodes as well.
		 */
		if (!S_ISBLK(inode->i_mode)) {
A
Al Viro 已提交
965
			if (inode_unhashed(inode))
966 967
				goto out;
		}
A
Al Viro 已提交
968
		if (inode->i_state & I_FREEING)
969 970 971 972 973 974 975
			goto out;

		/*
		 * If the inode was already on b_dirty/b_io/b_more_io, don't
		 * reposition it (that would break b_dirty time-ordering).
		 */
		if (!was_dirty) {
976 977 978 979 980 981 982 983 984 985 986 987 988 989
			bdi = inode_to_bdi(inode);

			if (bdi_cap_writeback_dirty(bdi)) {
				WARN(!test_bit(BDI_registered, &bdi->state),
				     "bdi-%s not registered\n", bdi->name);

				/*
				 * If this is the first dirty inode for this
				 * bdi, we have to wake-up the corresponding
				 * bdi thread to make sure background
				 * write-back happens later.
				 */
				if (!wb_has_dirty_io(&bdi->wb))
					wakeup_bdi = true;
990
			}
991 992

			inode->dirtied_when = jiffies;
993
			list_move(&inode->i_list, &bdi->wb.b_dirty);
L
Linus Torvalds 已提交
994 995
		}
	}
996 997
out:
	spin_unlock(&inode_lock);
998 999

	if (wakeup_bdi)
1000
		bdi_wakeup_thread_delayed(bdi);
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
}
EXPORT_SYMBOL(__mark_inode_dirty);

/*
 * Write out a superblock's list of dirty inodes.  A wait will be performed
 * upon no inodes, all inodes or the final one, depending upon sync_mode.
 *
 * If older_than_this is non-NULL, then only write out inodes which
 * had their first dirtying at a time earlier than *older_than_this.
 *
 * If `bdi' is non-zero then we're being asked to writeback a specific queue.
 * This function assumes that the blockdev superblock's inodes are backed by
 * a variety of queues, so all inodes are searched.  For other superblocks,
 * assume that all inodes are backed by the same queue.
 *
 * The inodes to be written are parked on bdi->b_io.  They are moved back onto
 * bdi->b_dirty as they are selected for writing.  This way, none can be missed
 * on the writer throttling path, and we get decent balancing between many
 * throttled threads: we don't want them all piling up on inode_sync_wait.
 */
1021
static void wait_sb_inodes(struct super_block *sb)
1022 1023 1024 1025 1026 1027 1028
{
	struct inode *inode, *old_inode = NULL;

	/*
	 * We need to be protected against the filesystem going from
	 * r/o to r/w or vice versa.
	 */
1029
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039

	spin_lock(&inode_lock);

	/*
	 * Data integrity sync. Must wait for all pages under writeback,
	 * because there may have been pages dirtied before our sync
	 * call, but which had writeout started before we write it out.
	 * In which case, the inode may not be on the dirty list, but
	 * we still have to wait for that writeout.
	 */
1040
	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
1041 1042
		struct address_space *mapping;

A
Al Viro 已提交
1043
		if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW))
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
			continue;
		mapping = inode->i_mapping;
		if (mapping->nrpages == 0)
			continue;
		__iget(inode);
		spin_unlock(&inode_lock);
		/*
		 * We hold a reference to 'inode' so it couldn't have
		 * been removed from s_inodes list while we dropped the
		 * inode_lock.  We cannot iput the inode now as we can
		 * be holding the last reference and we cannot iput it
		 * under inode_lock. So we keep the reference and iput
		 * it later.
		 */
		iput(old_inode);
		old_inode = inode;

		filemap_fdatawait(mapping);

		cond_resched();

		spin_lock(&inode_lock);
	}
	spin_unlock(&inode_lock);
	iput(old_inode);
L
Linus Torvalds 已提交
1069 1070
}

1071 1072 1073
/**
 * writeback_inodes_sb	-	writeback dirty inodes from given super_block
 * @sb: the superblock
L
Linus Torvalds 已提交
1074
 *
1075 1076 1077 1078
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
 * for IO completion of submitted IO. The number of pages submitted is
 * returned.
L
Linus Torvalds 已提交
1079
 */
1080
void writeback_inodes_sb(struct super_block *sb)
L
Linus Torvalds 已提交
1081
{
1082 1083
	unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY);
	unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS);
1084 1085
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1086 1087
		.sb		= sb,
		.sync_mode	= WB_SYNC_NONE,
1088
		.done		= &done,
1089
	};
1090

1091 1092
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

1093
	work.nr_pages = nr_dirty + nr_unstable + get_nr_dirty_inodes();
1094

1095 1096
	bdi_queue_work(sb->s_bdi, &work);
	wait_for_completion(&done);
1097
}
1098
EXPORT_SYMBOL(writeback_inodes_sb);
1099

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
/**
 * writeback_inodes_sb_if_idle	-	start writeback if none underway
 * @sb: the superblock
 *
 * Invoke writeback_inodes_sb if no writeback is currently underway.
 * Returns 1 if writeback was started, 0 if not.
 */
int writeback_inodes_sb_if_idle(struct super_block *sb)
{
	if (!writeback_in_progress(sb->s_bdi)) {
1110
		down_read(&sb->s_umount);
1111
		writeback_inodes_sb(sb);
1112
		up_read(&sb->s_umount);
1113 1114 1115 1116 1117 1118
		return 1;
	} else
		return 0;
}
EXPORT_SYMBOL(writeback_inodes_sb_if_idle);

1119 1120 1121 1122 1123 1124 1125
/**
 * sync_inodes_sb	-	sync sb inode pages
 * @sb: the superblock
 *
 * This function writes and waits on any dirty inode belonging to this
 * super_block. The number of pages synced is returned.
 */
1126
void sync_inodes_sb(struct super_block *sb)
1127
{
1128 1129
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1130 1131 1132 1133
		.sb		= sb,
		.sync_mode	= WB_SYNC_ALL,
		.nr_pages	= LONG_MAX,
		.range_cyclic	= 0,
1134
		.done		= &done,
1135 1136
	};

1137 1138
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

1139 1140 1141
	bdi_queue_work(sb->s_bdi, &work);
	wait_for_completion(&done);

1142
	wait_sb_inodes(sb);
L
Linus Torvalds 已提交
1143
}
1144
EXPORT_SYMBOL(sync_inodes_sb);
L
Linus Torvalds 已提交
1145 1146

/**
1147 1148 1149 1150 1151 1152
 * write_inode_now	-	write an inode to disk
 * @inode: inode to write to disk
 * @sync: whether the write should be synchronous or not
 *
 * This function commits an inode to disk immediately if it is dirty. This is
 * primarily needed by knfsd.
L
Linus Torvalds 已提交
1153
 *
1154
 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
L
Linus Torvalds 已提交
1155 1156 1157 1158 1159 1160
 */
int write_inode_now(struct inode *inode, int sync)
{
	int ret;
	struct writeback_control wbc = {
		.nr_to_write = LONG_MAX,
1161
		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
1162 1163
		.range_start = 0,
		.range_end = LLONG_MAX,
L
Linus Torvalds 已提交
1164 1165 1166
	};

	if (!mapping_cap_writeback_dirty(inode->i_mapping))
1167
		wbc.nr_to_write = 0;
L
Linus Torvalds 已提交
1168 1169 1170

	might_sleep();
	spin_lock(&inode_lock);
1171
	ret = writeback_single_inode(inode, &wbc);
L
Linus Torvalds 已提交
1172 1173
	spin_unlock(&inode_lock);
	if (sync)
J
Joern Engel 已提交
1174
		inode_sync_wait(inode);
L
Linus Torvalds 已提交
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
	return ret;
}
EXPORT_SYMBOL(write_inode_now);

/**
 * sync_inode - write an inode and its pages to disk.
 * @inode: the inode to sync
 * @wbc: controls the writeback mode
 *
 * sync_inode() will write an inode and its pages to disk.  It will also
 * correctly update the inode on its superblock's dirty inode lists and will
 * update inode->i_state.
 *
 * The caller must have a ref on the inode.
 */
int sync_inode(struct inode *inode, struct writeback_control *wbc)
{
	int ret;

	spin_lock(&inode_lock);
1195
	ret = writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
1196 1197 1198 1199
	spin_unlock(&inode_lock);
	return ret;
}
EXPORT_SYMBOL(sync_inode);
C
Christoph Hellwig 已提交
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219

/**
 * sync_inode - write an inode to disk
 * @inode: the inode to sync
 * @wait: wait for I/O to complete.
 *
 * Write an inode to disk and adjust it's dirty state after completion.
 *
 * Note: only writes the actual inode, no associated data or other metadata.
 */
int sync_inode_metadata(struct inode *inode, int wait)
{
	struct writeback_control wbc = {
		.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
		.nr_to_write = 0, /* metadata-only */
	};

	return sync_inode(inode, &wbc);
}
EXPORT_SYMBOL(sync_inode_metadata);