fs-writeback.c 33.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 * fs/fs-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * Contains all the functions related to writing back and waiting
 * upon dirty inodes against superblocks, and writing back dirty
 * pages against inodes.  ie: data writeback.  Writeout of the
 * inode itself is not handled here.
 *
11
 * 10Apr2002	Andrew Morton
L
Linus Torvalds 已提交
12 13 14 15 16
 *		Split out of fs/inode.c
 *		Additions for address_space-based writeback
 */

#include <linux/kernel.h>
J
Jens Axboe 已提交
17
#include <linux/module.h>
L
Linus Torvalds 已提交
18
#include <linux/spinlock.h>
19
#include <linux/slab.h>
L
Linus Torvalds 已提交
20 21 22
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
23 24
#include <linux/kthread.h>
#include <linux/freezer.h>
L
Linus Torvalds 已提交
25 26 27 28
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/buffer_head.h>
29
#include <linux/tracepoint.h>
30
#include "internal.h"
L
Linus Torvalds 已提交
31

32 33 34
/*
 * Passed into wb_writeback(), essentially a subset of writeback_control
 */
35
struct wb_writeback_work {
36 37 38
	long nr_pages;
	struct super_block *sb;
	enum writeback_sync_modes sync_mode;
39 40 41
	unsigned int for_kupdate:1;
	unsigned int range_cyclic:1;
	unsigned int for_background:1;
42

43
	struct list_head list;		/* pending work list */
44
	struct completion *done;	/* set if the caller waits */
45 46
};

47 48 49 50 51 52 53 54 55 56 57 58 59
/*
 * Include the creation of the trace points after defining the
 * wb_writeback_work structure so that the definition remains local to this
 * file.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/writeback.h>

/*
 * We don't actually have pdflush, but this one is exported though /proc...
 */
int nr_pdflush_threads;

60 61 62 63
/**
 * writeback_in_progress - determine whether there is writeback in progress
 * @bdi: the device's backing_dev_info structure.
 *
64 65
 * Determine whether there is writeback waiting to be handled against a
 * backing device.
66 67 68
 */
int writeback_in_progress(struct backing_dev_info *bdi)
{
J
Jan Kara 已提交
69
	return test_bit(BDI_writeback_running, &bdi->state);
70 71
}

72 73 74 75
static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
{
	struct super_block *sb = inode->i_sb;

76 77 78 79
	if (strcmp(sb->s_type->name, "bdev") == 0)
		return inode->i_mapping->backing_dev_info;

	return sb->s_bdi;
80 81
}

N
Nick Piggin 已提交
82 83 84 85 86
static inline struct inode *wb_inode(struct list_head *head)
{
	return list_entry(head, struct inode, i_wb_list);
}

87 88
/* Wakeup flusher thread or forker thread to fork it. Requires bdi->wb_lock. */
static void bdi_wakeup_flusher(struct backing_dev_info *bdi)
89
{
90 91 92 93 94 95 96
	if (bdi->wb.task) {
		wake_up_process(bdi->wb.task);
	} else {
		/*
		 * The bdi thread isn't there, wake up the forker thread which
		 * will create and run it.
		 */
97
		wake_up_process(default_backing_dev_info.wb.task);
L
Linus Torvalds 已提交
98
	}
99 100 101 102 103 104 105 106 107 108 109 110
}

static void bdi_queue_work(struct backing_dev_info *bdi,
			   struct wb_writeback_work *work)
{
	trace_writeback_queue(bdi, work);

	spin_lock_bh(&bdi->wb_lock);
	list_add_tail(&work->list, &bdi->work_list);
	if (!bdi->wb.task)
		trace_writeback_nothread(bdi, work);
	bdi_wakeup_flusher(bdi);
111
	spin_unlock_bh(&bdi->wb_lock);
L
Linus Torvalds 已提交
112 113
}

114 115
static void
__bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
116
		      bool range_cyclic)
L
Linus Torvalds 已提交
117
{
118
	struct wb_writeback_work *work;
119

120 121 122 123
	/*
	 * This is WB_SYNC_NONE writeback, so if allocation fails just
	 * wakeup the thread for old dirty data writeback
	 */
124 125
	work = kzalloc(sizeof(*work), GFP_ATOMIC);
	if (!work) {
126 127
		if (bdi->wb.task) {
			trace_writeback_nowork(bdi);
128
			wake_up_process(bdi->wb.task);
129
		}
130
		return;
131
	}
132

133 134 135
	work->sync_mode	= WB_SYNC_NONE;
	work->nr_pages	= nr_pages;
	work->range_cyclic = range_cyclic;
136

137
	bdi_queue_work(bdi, work);
138 139 140 141 142 143 144 145 146 147
}

/**
 * bdi_start_writeback - start writeback
 * @bdi: the backing device to write from
 * @nr_pages: the number of pages to write
 *
 * Description:
 *   This does WB_SYNC_NONE opportunistic writeback. The IO is only
 *   started when this function returns, we make no guarentees on
148
 *   completion. Caller need not hold sb s_umount semaphore.
149 150
 *
 */
151
void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages)
152
{
153
	__bdi_start_writeback(bdi, nr_pages, true);
154
}
155

156 157 158 159 160
/**
 * bdi_start_background_writeback - start background writeback
 * @bdi: the backing device to write from
 *
 * Description:
161 162 163 164
 *   This makes sure WB_SYNC_NONE background writeback happens. When
 *   this function returns, it is only guaranteed that for given BDI
 *   some IO is happening if we are over background dirty threshold.
 *   Caller need not hold sb s_umount semaphore.
165 166 167
 */
void bdi_start_background_writeback(struct backing_dev_info *bdi)
{
168 169 170 171
	/*
	 * We just wake up the flusher thread. It will perform background
	 * writeback as soon as there is no other work to do.
	 */
172
	trace_writeback_wake_background(bdi);
173 174 175
	spin_lock_bh(&bdi->wb_lock);
	bdi_wakeup_flusher(bdi);
	spin_unlock_bh(&bdi->wb_lock);
L
Linus Torvalds 已提交
176 177
}

178 179 180 181 182
/*
 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
 * furthest end of its superblock's dirty-inode list.
 *
 * Before stamping the inode's ->dirtied_when, we check to see whether it is
183
 * already the most-recently-dirtied inode on the b_dirty list.  If that is
184 185 186 187 188
 * the case then the inode must have been redirtied while it was being written
 * out and we don't reset its dirtied_when.
 */
static void redirty_tail(struct inode *inode)
{
189
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
190

191
	if (!list_empty(&wb->b_dirty)) {
192
		struct inode *tail;
193

N
Nick Piggin 已提交
194
		tail = wb_inode(wb->b_dirty.next);
195
		if (time_before(inode->dirtied_when, tail->dirtied_when))
196 197
			inode->dirtied_when = jiffies;
	}
N
Nick Piggin 已提交
198
	list_move(&inode->i_wb_list, &wb->b_dirty);
199 200
}

201
/*
202
 * requeue inode for re-scanning after bdi->b_io list is exhausted.
203
 */
204
static void requeue_io(struct inode *inode)
205
{
206 207
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;

N
Nick Piggin 已提交
208
	list_move(&inode->i_wb_list, &wb->b_more_io);
209 210
}

J
Joern Engel 已提交
211 212 213 214 215 216 217 218 219
static void inode_sync_complete(struct inode *inode)
{
	/*
	 * Prevent speculative execution through spin_unlock(&inode_lock);
	 */
	smp_mb();
	wake_up_bit(&inode->i_state, __I_SYNC);
}

220 221 222 223 224 225 226 227
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
	bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
	/*
	 * For inodes being constantly redirtied, dirtied_when can get stuck.
	 * It _appears_ to be in the future, but is actually in distant past.
	 * This test is necessary to prevent such wrapped-around relative times
228
	 * from permanently stopping the whole bdi writeback.
229 230 231 232 233 234
	 */
	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
	return ret;
}

235 236 237 238 239 240 241
/*
 * Move expired dirty inodes from @delaying_queue to @dispatch_queue.
 */
static void move_expired_inodes(struct list_head *delaying_queue,
			       struct list_head *dispatch_queue,
				unsigned long *older_than_this)
{
242 243
	LIST_HEAD(tmp);
	struct list_head *pos, *node;
244
	struct super_block *sb = NULL;
245
	struct inode *inode;
246
	int do_sb_sort = 0;
247

248
	while (!list_empty(delaying_queue)) {
N
Nick Piggin 已提交
249
		inode = wb_inode(delaying_queue->prev);
250
		if (older_than_this &&
251
		    inode_dirtied_after(inode, *older_than_this))
252
			break;
253 254 255
		if (sb && sb != inode->i_sb)
			do_sb_sort = 1;
		sb = inode->i_sb;
N
Nick Piggin 已提交
256
		list_move(&inode->i_wb_list, &tmp);
257 258
	}

259 260 261 262 263 264
	/* just one sb in list, splice to dispatch_queue and we're done */
	if (!do_sb_sort) {
		list_splice(&tmp, dispatch_queue);
		return;
	}

265 266
	/* Move inodes from one superblock together */
	while (!list_empty(&tmp)) {
N
Nick Piggin 已提交
267
		sb = wb_inode(tmp.prev)->i_sb;
268
		list_for_each_prev_safe(pos, node, &tmp) {
N
Nick Piggin 已提交
269
			inode = wb_inode(pos);
270
			if (inode->i_sb == sb)
N
Nick Piggin 已提交
271
				list_move(&inode->i_wb_list, dispatch_queue);
272
		}
273 274 275 276 277
	}
}

/*
 * Queue all expired dirty inodes for io, eldest first.
278 279 280 281 282 283 284 285
 * Before
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    gf         edc     BA
 * After
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    g          fBAedc
 *                                           |
 *                                           +--> dequeue for IO
286
 */
287
static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this)
288
{
289
	list_splice_init(&wb->b_more_io, &wb->b_io);
290
	move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this);
291 292
}

293
static int write_inode(struct inode *inode, struct writeback_control *wbc)
294
{
295
	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
296
		return inode->i_sb->s_op->write_inode(inode, wbc);
297
	return 0;
298 299
}

L
Linus Torvalds 已提交
300
/*
301 302 303 304 305 306 307 308
 * Wait for writeback on an inode to complete.
 */
static void inode_wait_for_writeback(struct inode *inode)
{
	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
	wait_queue_head_t *wqh;

	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
309
	 while (inode->i_state & I_SYNC) {
310 311 312
		spin_unlock(&inode_lock);
		__wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
		spin_lock(&inode_lock);
313
	}
314 315 316 317 318 319 320
}

/*
 * Write out an inode's dirty pages.  Called under inode_lock.  Either the
 * caller has ref on the inode (either via __iget or via syscall against an fd)
 * or the inode has I_WILL_FREE set (via generic_forget_inode)
 *
L
Linus Torvalds 已提交
321 322 323 324 325 326 327 328 329
 * If `wait' is set, wait on the writeout.
 *
 * The whole writeout design is quite complex and fragile.  We want to avoid
 * starvation of particular inodes when others are being redirtied, prevent
 * livelocks, etc.
 *
 * Called under inode_lock.
 */
static int
330
writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
L
Linus Torvalds 已提交
331 332
{
	struct address_space *mapping = inode->i_mapping;
333
	unsigned dirty;
L
Linus Torvalds 已提交
334 335
	int ret;

336 337 338 339 340 341 342 343
	if (!atomic_read(&inode->i_count))
		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
	else
		WARN_ON(inode->i_state & I_WILL_FREE);

	if (inode->i_state & I_SYNC) {
		/*
		 * If this inode is locked for writeback and we are not doing
344
		 * writeback-for-data-integrity, move it to b_more_io so that
345 346 347
		 * writeback can proceed with the other inodes on s_io.
		 *
		 * We'll have another go at writing back this inode when we
348
		 * completed a full scan of b_io.
349
		 */
350
		if (wbc->sync_mode != WB_SYNC_ALL) {
351 352 353 354 355 356 357 358 359 360
			requeue_io(inode);
			return 0;
		}

		/*
		 * It's a data-integrity sync.  We must wait.
		 */
		inode_wait_for_writeback(inode);
	}

J
Joern Engel 已提交
361
	BUG_ON(inode->i_state & I_SYNC);
L
Linus Torvalds 已提交
362

363
	/* Set I_SYNC, reset I_DIRTY_PAGES */
J
Joern Engel 已提交
364
	inode->i_state |= I_SYNC;
365
	inode->i_state &= ~I_DIRTY_PAGES;
L
Linus Torvalds 已提交
366 367 368 369
	spin_unlock(&inode_lock);

	ret = do_writepages(mapping, wbc);

370 371 372 373 374
	/*
	 * Make sure to wait on the data before writing out the metadata.
	 * This is important for filesystems that modify metadata on data
	 * I/O completion.
	 */
375
	if (wbc->sync_mode == WB_SYNC_ALL) {
376
		int err = filemap_fdatawait(mapping);
L
Linus Torvalds 已提交
377 378 379 380
		if (ret == 0)
			ret = err;
	}

381 382 383 384 385 386 387 388 389
	/*
	 * Some filesystems may redirty the inode during the writeback
	 * due to delalloc, clear dirty metadata flags right before
	 * write_inode()
	 */
	spin_lock(&inode_lock);
	dirty = inode->i_state & I_DIRTY;
	inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC);
	spin_unlock(&inode_lock);
390 391
	/* Don't write the inode if only I_DIRTY_PAGES was set */
	if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
392
		int err = write_inode(inode, wbc);
L
Linus Torvalds 已提交
393 394 395 396 397
		if (ret == 0)
			ret = err;
	}

	spin_lock(&inode_lock);
J
Joern Engel 已提交
398
	inode->i_state &= ~I_SYNC;
A
Al Viro 已提交
399
	if (!(inode->i_state & I_FREEING)) {
400
		if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
L
Linus Torvalds 已提交
401 402
			/*
			 * We didn't write back all the pages.  nfs_writepages()
403
			 * sometimes bales out without doing anything.
404
			 */
405 406
			inode->i_state |= I_DIRTY_PAGES;
			if (wbc->nr_to_write <= 0) {
L
Linus Torvalds 已提交
407
				/*
408
				 * slice used up: queue for next turn
L
Linus Torvalds 已提交
409
				 */
410
				requeue_io(inode);
L
Linus Torvalds 已提交
411 412
			} else {
				/*
413 414 415 416 417
				 * Writeback blocked by something other than
				 * congestion. Delay the inode for some time to
				 * avoid spinning on the CPU (100% iowait)
				 * retrying writeback of the dirty page/inode
				 * that cannot be performed immediately.
L
Linus Torvalds 已提交
418
				 */
419
				redirty_tail(inode);
L
Linus Torvalds 已提交
420
			}
421 422 423 424 425 426 427 428
		} else if (inode->i_state & I_DIRTY) {
			/*
			 * Filesystems can dirty the inode during writeback
			 * operations, such as delayed allocation during
			 * submission or metadata updates after data IO
			 * completion.
			 */
			redirty_tail(inode);
L
Linus Torvalds 已提交
429 430
		} else {
			/*
431 432 433
			 * The inode is clean.  At this point we either have
			 * a reference to the inode or it's on it's way out.
			 * No need to add it back to the LRU.
L
Linus Torvalds 已提交
434
			 */
N
Nick Piggin 已提交
435
			list_del_init(&inode->i_wb_list);
L
Linus Torvalds 已提交
436 437
		}
	}
J
Joern Engel 已提交
438
	inode_sync_complete(inode);
L
Linus Torvalds 已提交
439 440 441
	return ret;
}

442
/*
443
 * For background writeback the caller does not have the sb pinned
444 445 446
 * before calling writeback. So make sure that we do pin it, so it doesn't
 * go away while we are writing inodes from it.
 */
447
static bool pin_sb_for_writeback(struct super_block *sb)
448 449
{
	spin_lock(&sb_lock);
450 451 452 453 454
	if (list_empty(&sb->s_instances)) {
		spin_unlock(&sb_lock);
		return false;
	}

455
	sb->s_count++;
456 457
	spin_unlock(&sb_lock);

458
	if (down_read_trylock(&sb->s_umount)) {
459
		if (sb->s_root)
460
			return true;
461 462
		up_read(&sb->s_umount);
	}
463 464

	put_super(sb);
465
	return false;
466 467
}

468 469
/*
 * Write a portion of b_io inodes which belong to @sb.
470 471
 *
 * If @only_this_sb is true, then find and write all such
472 473
 * inodes. Otherwise write only ones which go sequentially
 * in reverse order.
474
 *
475 476 477
 * Return 1, if the caller writeback routine should be
 * interrupted. Otherwise return 0.
 */
478 479
static int writeback_sb_inodes(struct super_block *sb, struct bdi_writeback *wb,
		struct writeback_control *wbc, bool only_this_sb)
L
Linus Torvalds 已提交
480
{
481
	while (!list_empty(&wb->b_io)) {
L
Linus Torvalds 已提交
482
		long pages_skipped;
N
Nick Piggin 已提交
483
		struct inode *inode = wb_inode(wb->b_io.prev);
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500

		if (inode->i_sb != sb) {
			if (only_this_sb) {
				/*
				 * We only want to write back data for this
				 * superblock, move all inodes not belonging
				 * to it back onto the dirty list.
				 */
				redirty_tail(inode);
				continue;
			}

			/*
			 * The inode belongs to a different superblock.
			 * Bounce back to the caller to unpin this and
			 * pin the next superblock.
			 */
501
			return 0;
502 503
		}

504 505 506 507 508 509
		/*
		 * Don't bother with new inodes or inodes beeing freed, first
		 * kind does not need peridic writeout yet, and for the latter
		 * kind writeout is handled by the freer.
		 */
		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
510 511 512
			requeue_io(inode);
			continue;
		}
513

514 515 516 517
		/*
		 * Was this inode dirtied after sync_sb_inodes was called?
		 * This keeps sync from extra jobs and livelock.
		 */
518 519
		if (inode_dirtied_after(inode, wbc->wb_start))
			return 1;
L
Linus Torvalds 已提交
520 521 522

		__iget(inode);
		pages_skipped = wbc->pages_skipped;
523
		writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
524 525 526 527 528
		if (wbc->pages_skipped != pages_skipped) {
			/*
			 * writeback is not making progress due to locked
			 * buffers.  Skip this inode for now.
			 */
529
			redirty_tail(inode);
L
Linus Torvalds 已提交
530 531 532
		}
		spin_unlock(&inode_lock);
		iput(inode);
533
		cond_resched();
L
Linus Torvalds 已提交
534
		spin_lock(&inode_lock);
535 536
		if (wbc->nr_to_write <= 0) {
			wbc->more_io = 1;
537
			return 1;
538
		}
539
		if (!list_empty(&wb->b_more_io))
540
			wbc->more_io = 1;
L
Linus Torvalds 已提交
541
	}
542 543 544 545
	/* b_io is empty */
	return 1;
}

546 547
void writeback_inodes_wb(struct bdi_writeback *wb,
		struct writeback_control *wbc)
548 549 550
{
	int ret = 0;

551 552
	if (!wbc->wb_start)
		wbc->wb_start = jiffies; /* livelock avoidance */
553 554 555
	spin_lock(&inode_lock);
	if (!wbc->for_kupdate || list_empty(&wb->b_io))
		queue_io(wb, wbc->older_than_this);
N
Nick Piggin 已提交
556

557
	while (!list_empty(&wb->b_io)) {
N
Nick Piggin 已提交
558
		struct inode *inode = wb_inode(wb->b_io.prev);
559
		struct super_block *sb = inode->i_sb;
560

561 562 563
		if (!pin_sb_for_writeback(sb)) {
			requeue_io(inode);
			continue;
564
		}
565 566
		ret = writeback_sb_inodes(sb, wb, wbc, false);
		drop_super(sb);
567 568 569 570

		if (ret)
			break;
	}
571 572 573 574
	spin_unlock(&inode_lock);
	/* Leave any unwritten inodes on b_io */
}

575 576 577 578 579 580 581 582 583 584 585 586
static void __writeback_inodes_sb(struct super_block *sb,
		struct bdi_writeback *wb, struct writeback_control *wbc)
{
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

	spin_lock(&inode_lock);
	if (!wbc->for_kupdate || list_empty(&wb->b_io))
		queue_io(wb, wbc->older_than_this);
	writeback_sb_inodes(sb, wb, wbc, true);
	spin_unlock(&inode_lock);
}

587
/*
588 589 590 591 592 593 594 595 596 597 598 599
 * The maximum number of pages to writeout in a single bdi flush/kupdate
 * operation.  We do this so we don't hold I_SYNC against an inode for
 * enormous amounts of time, which would block a userspace task which has
 * been forced to throttle against that inode.  Also, the code reevaluates
 * the dirty each time it has written this many pages.
 */
#define MAX_WRITEBACK_PAGES     1024

static inline bool over_bground_thresh(void)
{
	unsigned long background_thresh, dirty_thresh;

600
	global_dirty_limits(&background_thresh, &dirty_thresh);
601 602

	return (global_page_state(NR_FILE_DIRTY) +
603
		global_page_state(NR_UNSTABLE_NFS) > background_thresh);
604 605 606 607
}

/*
 * Explicit flushing or periodic writeback of "old" data.
608
 *
609 610 611 612
 * Define "old": the first time one of an inode's pages is dirtied, we mark the
 * dirtying-time in the inode's address_space.  So this periodic writeback code
 * just walks the superblock inode list, writing back any inodes which are
 * older than a specific point in time.
613
 *
614 615 616
 * Try to run once per dirty_writeback_interval.  But if a writeback event
 * takes longer than a dirty_writeback_interval interval, then leave a
 * one-second gap.
617
 *
618 619
 * older_than_this takes precedence over nr_to_write.  So we'll only write back
 * all dirty pages if they are all attached to "old" mappings.
620
 */
621
static long wb_writeback(struct bdi_writeback *wb,
622
			 struct wb_writeback_work *work)
623
{
624
	struct writeback_control wbc = {
625
		.sync_mode		= work->sync_mode,
626
		.older_than_this	= NULL,
627 628 629
		.for_kupdate		= work->for_kupdate,
		.for_background		= work->for_background,
		.range_cyclic		= work->range_cyclic,
630 631 632
	};
	unsigned long oldest_jif;
	long wrote = 0;
J
Jan Kara 已提交
633
	struct inode *inode;
634

635 636 637 638 639
	if (wbc.for_kupdate) {
		wbc.older_than_this = &oldest_jif;
		oldest_jif = jiffies -
				msecs_to_jiffies(dirty_expire_interval * 10);
	}
640 641 642 643
	if (!wbc.range_cyclic) {
		wbc.range_start = 0;
		wbc.range_end = LLONG_MAX;
	}
N
Nick Piggin 已提交
644

645
	wbc.wb_start = jiffies; /* livelock avoidance */
646 647
	for (;;) {
		/*
648
		 * Stop writeback when nr_pages has been consumed
649
		 */
650
		if (work->nr_pages <= 0)
651
			break;
652

N
Nick Piggin 已提交
653
		/*
654 655
		 * For background writeout, stop when we are below the
		 * background dirty threshold
N
Nick Piggin 已提交
656
		 */
657
		if (work->for_background && !over_bground_thresh())
658
			break;
N
Nick Piggin 已提交
659

660 661 662
		wbc.more_io = 0;
		wbc.nr_to_write = MAX_WRITEBACK_PAGES;
		wbc.pages_skipped = 0;
663 664

		trace_wbc_writeback_start(&wbc, wb->bdi);
665 666
		if (work->sb)
			__writeback_inodes_sb(work->sb, wb, &wbc);
667 668
		else
			writeback_inodes_wb(wb, &wbc);
669 670
		trace_wbc_writeback_written(&wbc, wb->bdi);

671
		work->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
672 673 674
		wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write;

		/*
675
		 * If we consumed everything, see if we have more
676
		 */
677 678 679 680 681 682
		if (wbc.nr_to_write <= 0)
			continue;
		/*
		 * Didn't write everything and we don't have more IO, bail
		 */
		if (!wbc.more_io)
683
			break;
684 685 686 687 688 689 690 691 692 693 694 695
		/*
		 * Did we write something? Try for more
		 */
		if (wbc.nr_to_write < MAX_WRITEBACK_PAGES)
			continue;
		/*
		 * Nothing written. Wait for some inode to
		 * become available for writeback. Otherwise
		 * we'll just busyloop.
		 */
		spin_lock(&inode_lock);
		if (!list_empty(&wb->b_more_io))  {
N
Nick Piggin 已提交
696
			inode = wb_inode(wb->b_more_io.prev);
697
			trace_wbc_writeback_wait(&wbc, wb->bdi);
698
			inode_wait_for_writeback(inode);
699
		}
700
		spin_unlock(&inode_lock);
701 702 703 704 705 706
	}

	return wrote;
}

/*
707
 * Return the next wb_writeback_work struct that hasn't been processed yet.
708
 */
709
static struct wb_writeback_work *
710
get_next_work_item(struct backing_dev_info *bdi)
711
{
712
	struct wb_writeback_work *work = NULL;
713

714
	spin_lock_bh(&bdi->wb_lock);
715 716 717 718
	if (!list_empty(&bdi->work_list)) {
		work = list_entry(bdi->work_list.next,
				  struct wb_writeback_work, list);
		list_del_init(&work->list);
719
	}
720
	spin_unlock_bh(&bdi->wb_lock);
721
	return work;
722 723
}

724 725 726 727 728 729 730 731 732 733 734
/*
 * Add in the number of potentially dirty inodes, because each inode
 * write can dirty pagecache in the underlying blockdev.
 */
static unsigned long get_nr_dirty_pages(void)
{
	return global_page_state(NR_FILE_DIRTY) +
		global_page_state(NR_UNSTABLE_NFS) +
		get_nr_dirty_inodes();
}

735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
static long wb_check_background_flush(struct bdi_writeback *wb)
{
	if (over_bground_thresh()) {

		struct wb_writeback_work work = {
			.nr_pages	= LONG_MAX,
			.sync_mode	= WB_SYNC_NONE,
			.for_background	= 1,
			.range_cyclic	= 1,
		};

		return wb_writeback(wb, &work);
	}

	return 0;
}

752 753 754 755 756
static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
	unsigned long expired;
	long nr_pages;

757 758 759 760 761 762
	/*
	 * When set to zero, disable periodic writeback
	 */
	if (!dirty_writeback_interval)
		return 0;

763 764 765 766 767 768
	expired = wb->last_old_flush +
			msecs_to_jiffies(dirty_writeback_interval * 10);
	if (time_before(jiffies, expired))
		return 0;

	wb->last_old_flush = jiffies;
769
	nr_pages = get_nr_dirty_pages();
770

771
	if (nr_pages) {
772
		struct wb_writeback_work work = {
773 774 775 776 777 778
			.nr_pages	= nr_pages,
			.sync_mode	= WB_SYNC_NONE,
			.for_kupdate	= 1,
			.range_cyclic	= 1,
		};

779
		return wb_writeback(wb, &work);
780
	}
781 782 783 784 785 786 787 788 789 790

	return 0;
}

/*
 * Retrieve work items and do the writeback they describe
 */
long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
{
	struct backing_dev_info *bdi = wb->bdi;
791
	struct wb_writeback_work *work;
792
	long wrote = 0;
793

J
Jan Kara 已提交
794
	set_bit(BDI_writeback_running, &wb->bdi->state);
795
	while ((work = get_next_work_item(bdi)) != NULL) {
796 797
		/*
		 * Override sync mode, in case we must wait for completion
798
		 * because this thread is exiting now.
799 800
		 */
		if (force_wait)
801
			work->sync_mode = WB_SYNC_ALL;
802

803 804
		trace_writeback_exec(bdi, work);

805
		wrote += wb_writeback(wb, work);
806 807

		/*
808 809
		 * Notify the caller of completion if this is a synchronous
		 * work item, otherwise just free it.
810
		 */
811 812 813 814
		if (work->done)
			complete(work->done);
		else
			kfree(work);
815 816 817 818 819 820
	}

	/*
	 * Check for periodic writeback, kupdated() style
	 */
	wrote += wb_check_old_data_flush(wb);
821
	wrote += wb_check_background_flush(wb);
J
Jan Kara 已提交
822
	clear_bit(BDI_writeback_running, &wb->bdi->state);
823 824 825 826 827 828 829 830

	return wrote;
}

/*
 * Handle writeback of dirty data for the device backed by this bdi. Also
 * wakes up periodically and does kupdated style flushing.
 */
831
int bdi_writeback_thread(void *data)
832
{
833 834
	struct bdi_writeback *wb = data;
	struct backing_dev_info *bdi = wb->bdi;
835 836
	long pages_written;

P
Peter Zijlstra 已提交
837
	current->flags |= PF_SWAPWRITE;
838
	set_freezable();
839
	wb->last_active = jiffies;
840 841 842 843 844 845

	/*
	 * Our parent may run at a different priority, just set us to normal
	 */
	set_user_nice(current, 0);

846 847
	trace_writeback_thread_start(bdi);

848
	while (!kthread_should_stop()) {
849 850 851 852 853 854
		/*
		 * Remove own delayed wake-up timer, since we are already awake
		 * and we'll take care of the preriodic write-back.
		 */
		del_timer(&wb->wakeup_timer);

855 856
		pages_written = wb_do_writeback(wb, 0);

857 858
		trace_writeback_pages_written(pages_written);

859
		if (pages_written)
860
			wb->last_active = jiffies;
861

862
		set_current_state(TASK_INTERRUPTIBLE);
863
		if (!list_empty(&bdi->work_list) || kthread_should_stop()) {
864
			__set_current_state(TASK_RUNNING);
865
			continue;
866 867
		}

868
		if (wb_has_dirty_io(wb) && dirty_writeback_interval)
869
			schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10));
870 871 872 873 874 875
		else {
			/*
			 * We have nothing to do, so can go sleep without any
			 * timeout and save power. When a work is queued or
			 * something is made dirty - we will be woken up.
			 */
876
			schedule();
877
		}
878

879 880 881
		try_to_freeze();
	}

882
	/* Flush any work that raced with us exiting */
883 884
	if (!list_empty(&bdi->work_list))
		wb_do_writeback(wb, 1);
885 886

	trace_writeback_thread_stop(bdi);
887 888 889
	return 0;
}

890

891
/*
892 893
 * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
 * the whole world.
894
 */
895
void wakeup_flusher_threads(long nr_pages)
896
{
897
	struct backing_dev_info *bdi;
898

899 900
	if (!nr_pages) {
		nr_pages = global_page_state(NR_FILE_DIRTY) +
901 902
				global_page_state(NR_UNSTABLE_NFS);
	}
903

904
	rcu_read_lock();
905
	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
906 907
		if (!bdi_has_dirty_io(bdi))
			continue;
908
		__bdi_start_writeback(bdi, nr_pages, false);
909
	}
910
	rcu_read_unlock();
L
Linus Torvalds 已提交
911 912
}

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
static noinline void block_dump___mark_inode_dirty(struct inode *inode)
{
	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
		struct dentry *dentry;
		const char *name = "?";

		dentry = d_find_alias(inode);
		if (dentry) {
			spin_lock(&dentry->d_lock);
			name = (const char *) dentry->d_name.name;
		}
		printk(KERN_DEBUG
		       "%s(%d): dirtied inode %lu (%s) on %s\n",
		       current->comm, task_pid_nr(current), inode->i_ino,
		       name, inode->i_sb->s_id);
		if (dentry) {
			spin_unlock(&dentry->d_lock);
			dput(dentry);
		}
	}
}

/**
 *	__mark_inode_dirty -	internal function
 *	@inode: inode to mark
 *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
 *	Mark an inode as dirty. Callers should use mark_inode_dirty or
 *  	mark_inode_dirty_sync.
L
Linus Torvalds 已提交
941
 *
942 943 944 945 946 947 948 949 950
 * Put the inode on the super block's dirty list.
 *
 * CAREFUL! We mark it dirty unconditionally, but move it onto the
 * dirty list only if it is hashed or if it refers to a blockdev.
 * If it was not hashed, it will never be added to the dirty list
 * even if it is later hashed, as it will have been marked dirty already.
 *
 * In short, make sure you hash any inodes _before_ you start marking
 * them dirty.
L
Linus Torvalds 已提交
951
 *
952 953
 * This function *must* be atomic for the I_DIRTY_PAGES case -
 * set_page_dirty() is called under spinlock in several places.
L
Linus Torvalds 已提交
954
 *
955 956 957 958 959 960
 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
 * the kernel-internal blockdev inode represents the dirtying time of the
 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
 * page->mapping->host, so the page-dirtying time is recorded in the internal
 * blockdev inode.
L
Linus Torvalds 已提交
961
 */
962
void __mark_inode_dirty(struct inode *inode, int flags)
L
Linus Torvalds 已提交
963
{
964
	struct super_block *sb = inode->i_sb;
965 966
	struct backing_dev_info *bdi = NULL;
	bool wakeup_bdi = false;
L
Linus Torvalds 已提交
967

968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
	/*
	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
	 * dirty the inode itself
	 */
	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
		if (sb->s_op->dirty_inode)
			sb->s_op->dirty_inode(inode);
	}

	/*
	 * make sure that changes are seen by all cpus before we test i_state
	 * -- mikulas
	 */
	smp_mb();

	/* avoid the locking if we can */
	if ((inode->i_state & flags) == flags)
		return;

	if (unlikely(block_dump))
		block_dump___mark_inode_dirty(inode);

	spin_lock(&inode_lock);
	if ((inode->i_state & flags) != flags) {
		const int was_dirty = inode->i_state & I_DIRTY;

		inode->i_state |= flags;

		/*
		 * If the inode is being synced, just update its dirty state.
		 * The unlocker will place the inode on the appropriate
		 * superblock list, based upon its state.
		 */
		if (inode->i_state & I_SYNC)
			goto out;

		/*
		 * Only add valid (hashed) inodes to the superblock's
		 * dirty list.  Add blockdev inodes as well.
		 */
		if (!S_ISBLK(inode->i_mode)) {
A
Al Viro 已提交
1009
			if (inode_unhashed(inode))
1010 1011
				goto out;
		}
A
Al Viro 已提交
1012
		if (inode->i_state & I_FREEING)
1013 1014 1015 1016 1017 1018 1019
			goto out;

		/*
		 * If the inode was already on b_dirty/b_io/b_more_io, don't
		 * reposition it (that would break b_dirty time-ordering).
		 */
		if (!was_dirty) {
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
			bdi = inode_to_bdi(inode);

			if (bdi_cap_writeback_dirty(bdi)) {
				WARN(!test_bit(BDI_registered, &bdi->state),
				     "bdi-%s not registered\n", bdi->name);

				/*
				 * If this is the first dirty inode for this
				 * bdi, we have to wake-up the corresponding
				 * bdi thread to make sure background
				 * write-back happens later.
				 */
				if (!wb_has_dirty_io(&bdi->wb))
					wakeup_bdi = true;
1034
			}
1035 1036

			inode->dirtied_when = jiffies;
N
Nick Piggin 已提交
1037
			list_move(&inode->i_wb_list, &bdi->wb.b_dirty);
L
Linus Torvalds 已提交
1038 1039
		}
	}
1040 1041
out:
	spin_unlock(&inode_lock);
1042 1043

	if (wakeup_bdi)
1044
		bdi_wakeup_thread_delayed(bdi);
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
}
EXPORT_SYMBOL(__mark_inode_dirty);

/*
 * Write out a superblock's list of dirty inodes.  A wait will be performed
 * upon no inodes, all inodes or the final one, depending upon sync_mode.
 *
 * If older_than_this is non-NULL, then only write out inodes which
 * had their first dirtying at a time earlier than *older_than_this.
 *
 * If `bdi' is non-zero then we're being asked to writeback a specific queue.
 * This function assumes that the blockdev superblock's inodes are backed by
 * a variety of queues, so all inodes are searched.  For other superblocks,
 * assume that all inodes are backed by the same queue.
 *
 * The inodes to be written are parked on bdi->b_io.  They are moved back onto
 * bdi->b_dirty as they are selected for writing.  This way, none can be missed
 * on the writer throttling path, and we get decent balancing between many
 * throttled threads: we don't want them all piling up on inode_sync_wait.
 */
1065
static void wait_sb_inodes(struct super_block *sb)
1066 1067 1068 1069 1070 1071 1072
{
	struct inode *inode, *old_inode = NULL;

	/*
	 * We need to be protected against the filesystem going from
	 * r/o to r/w or vice versa.
	 */
1073
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083

	spin_lock(&inode_lock);

	/*
	 * Data integrity sync. Must wait for all pages under writeback,
	 * because there may have been pages dirtied before our sync
	 * call, but which had writeout started before we write it out.
	 * In which case, the inode may not be on the dirty list, but
	 * we still have to wait for that writeout.
	 */
1084
	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
1085 1086
		struct address_space *mapping;

A
Al Viro 已提交
1087
		if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW))
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
			continue;
		mapping = inode->i_mapping;
		if (mapping->nrpages == 0)
			continue;
		__iget(inode);
		spin_unlock(&inode_lock);
		/*
		 * We hold a reference to 'inode' so it couldn't have
		 * been removed from s_inodes list while we dropped the
		 * inode_lock.  We cannot iput the inode now as we can
		 * be holding the last reference and we cannot iput it
		 * under inode_lock. So we keep the reference and iput
		 * it later.
		 */
		iput(old_inode);
		old_inode = inode;

		filemap_fdatawait(mapping);

		cond_resched();

		spin_lock(&inode_lock);
	}
	spin_unlock(&inode_lock);
	iput(old_inode);
L
Linus Torvalds 已提交
1113 1114
}

1115
/**
1116
 * writeback_inodes_sb_nr -	writeback dirty inodes from given super_block
1117
 * @sb: the superblock
1118
 * @nr: the number of pages to write
L
Linus Torvalds 已提交
1119
 *
1120 1121
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
1122
 * for IO completion of submitted IO.
L
Linus Torvalds 已提交
1123
 */
1124
void writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr)
L
Linus Torvalds 已提交
1125
{
1126 1127
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1128 1129
		.sb		= sb,
		.sync_mode	= WB_SYNC_NONE,
1130
		.done		= &done,
1131
		.nr_pages	= nr,
1132
	};
1133

1134
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1135 1136
	bdi_queue_work(sb->s_bdi, &work);
	wait_for_completion(&done);
1137
}
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
EXPORT_SYMBOL(writeback_inodes_sb_nr);

/**
 * writeback_inodes_sb	-	writeback dirty inodes from given super_block
 * @sb: the superblock
 *
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
 * for IO completion of submitted IO.
 */
void writeback_inodes_sb(struct super_block *sb)
{
1150
	return writeback_inodes_sb_nr(sb, get_nr_dirty_pages());
1151
}
1152
EXPORT_SYMBOL(writeback_inodes_sb);
1153

1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
/**
 * writeback_inodes_sb_if_idle	-	start writeback if none underway
 * @sb: the superblock
 *
 * Invoke writeback_inodes_sb if no writeback is currently underway.
 * Returns 1 if writeback was started, 0 if not.
 */
int writeback_inodes_sb_if_idle(struct super_block *sb)
{
	if (!writeback_in_progress(sb->s_bdi)) {
1164
		down_read(&sb->s_umount);
1165
		writeback_inodes_sb(sb);
1166
		up_read(&sb->s_umount);
1167 1168 1169 1170 1171 1172
		return 1;
	} else
		return 0;
}
EXPORT_SYMBOL(writeback_inodes_sb_if_idle);

1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193
/**
 * writeback_inodes_sb_if_idle	-	start writeback if none underway
 * @sb: the superblock
 * @nr: the number of pages to write
 *
 * Invoke writeback_inodes_sb if no writeback is currently underway.
 * Returns 1 if writeback was started, 0 if not.
 */
int writeback_inodes_sb_nr_if_idle(struct super_block *sb,
				   unsigned long nr)
{
	if (!writeback_in_progress(sb->s_bdi)) {
		down_read(&sb->s_umount);
		writeback_inodes_sb_nr(sb, nr);
		up_read(&sb->s_umount);
		return 1;
	} else
		return 0;
}
EXPORT_SYMBOL(writeback_inodes_sb_nr_if_idle);

1194 1195 1196 1197 1198 1199 1200
/**
 * sync_inodes_sb	-	sync sb inode pages
 * @sb: the superblock
 *
 * This function writes and waits on any dirty inode belonging to this
 * super_block. The number of pages synced is returned.
 */
1201
void sync_inodes_sb(struct super_block *sb)
1202
{
1203 1204
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1205 1206 1207 1208
		.sb		= sb,
		.sync_mode	= WB_SYNC_ALL,
		.nr_pages	= LONG_MAX,
		.range_cyclic	= 0,
1209
		.done		= &done,
1210 1211
	};

1212 1213
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

1214 1215 1216
	bdi_queue_work(sb->s_bdi, &work);
	wait_for_completion(&done);

1217
	wait_sb_inodes(sb);
L
Linus Torvalds 已提交
1218
}
1219
EXPORT_SYMBOL(sync_inodes_sb);
L
Linus Torvalds 已提交
1220 1221

/**
1222 1223 1224 1225 1226 1227
 * write_inode_now	-	write an inode to disk
 * @inode: inode to write to disk
 * @sync: whether the write should be synchronous or not
 *
 * This function commits an inode to disk immediately if it is dirty. This is
 * primarily needed by knfsd.
L
Linus Torvalds 已提交
1228
 *
1229
 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
L
Linus Torvalds 已提交
1230 1231 1232 1233 1234 1235
 */
int write_inode_now(struct inode *inode, int sync)
{
	int ret;
	struct writeback_control wbc = {
		.nr_to_write = LONG_MAX,
1236
		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
1237 1238
		.range_start = 0,
		.range_end = LLONG_MAX,
L
Linus Torvalds 已提交
1239 1240 1241
	};

	if (!mapping_cap_writeback_dirty(inode->i_mapping))
1242
		wbc.nr_to_write = 0;
L
Linus Torvalds 已提交
1243 1244 1245

	might_sleep();
	spin_lock(&inode_lock);
1246
	ret = writeback_single_inode(inode, &wbc);
L
Linus Torvalds 已提交
1247 1248
	spin_unlock(&inode_lock);
	if (sync)
J
Joern Engel 已提交
1249
		inode_sync_wait(inode);
L
Linus Torvalds 已提交
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
	return ret;
}
EXPORT_SYMBOL(write_inode_now);

/**
 * sync_inode - write an inode and its pages to disk.
 * @inode: the inode to sync
 * @wbc: controls the writeback mode
 *
 * sync_inode() will write an inode and its pages to disk.  It will also
 * correctly update the inode on its superblock's dirty inode lists and will
 * update inode->i_state.
 *
 * The caller must have a ref on the inode.
 */
int sync_inode(struct inode *inode, struct writeback_control *wbc)
{
	int ret;

	spin_lock(&inode_lock);
1270
	ret = writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
1271 1272 1273 1274
	spin_unlock(&inode_lock);
	return ret;
}
EXPORT_SYMBOL(sync_inode);
C
Christoph Hellwig 已提交
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294

/**
 * sync_inode - write an inode to disk
 * @inode: the inode to sync
 * @wait: wait for I/O to complete.
 *
 * Write an inode to disk and adjust it's dirty state after completion.
 *
 * Note: only writes the actual inode, no associated data or other metadata.
 */
int sync_inode_metadata(struct inode *inode, int wait)
{
	struct writeback_control wbc = {
		.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
		.nr_to_write = 0, /* metadata-only */
	};

	return sync_inode(inode, &wbc);
}
EXPORT_SYMBOL(sync_inode_metadata);