fs-writeback.c 45.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 * fs/fs-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * Contains all the functions related to writing back and waiting
 * upon dirty inodes against superblocks, and writing back dirty
 * pages against inodes.  ie: data writeback.  Writeout of the
 * inode itself is not handled here.
 *
11
 * 10Apr2002	Andrew Morton
L
Linus Torvalds 已提交
12 13 14 15 16
 *		Split out of fs/inode.c
 *		Additions for address_space-based writeback
 */

#include <linux/kernel.h>
17
#include <linux/export.h>
L
Linus Torvalds 已提交
18
#include <linux/spinlock.h>
19
#include <linux/slab.h>
L
Linus Torvalds 已提交
20 21 22
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
23
#include <linux/pagemap.h>
24
#include <linux/kthread.h>
L
Linus Torvalds 已提交
25 26 27
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
28
#include <linux/tracepoint.h>
29
#include <linux/device.h>
30
#include "internal.h"
L
Linus Torvalds 已提交
31

32 33 34 35 36
/*
 * 4MB minimal write chunk size
 */
#define MIN_WRITEBACK_PAGES	(4096UL >> (PAGE_CACHE_SHIFT - 10))

37 38 39
/*
 * Passed into wb_writeback(), essentially a subset of writeback_control
 */
40
struct wb_writeback_work {
41 42
	long nr_pages;
	struct super_block *sb;
43
	unsigned long *older_than_this;
44
	enum writeback_sync_modes sync_mode;
45
	unsigned int tagged_writepages:1;
46 47 48
	unsigned int for_kupdate:1;
	unsigned int range_cyclic:1;
	unsigned int for_background:1;
49
	unsigned int for_sync:1;	/* sync(2) WB_SYNC_ALL writeback */
50
	enum wb_reason reason;		/* why was writeback initiated? */
51

52
	struct list_head list;		/* pending work list */
53
	struct completion *done;	/* set if the caller waits */
54 55
};

56 57 58 59 60 61 62 63 64 65 66 67
/*
 * If an inode is constantly having its pages dirtied, but then the
 * updates stop dirtytime_expire_interval seconds in the past, it's
 * possible for the worst case time between when an inode has its
 * timestamps updated and when they finally get written out to be two
 * dirtytime_expire_intervals.  We set the default to 12 hours (in
 * seconds), which means most of the time inodes will have their
 * timestamps written to disk after 12 hours, but in the worst case a
 * few inodes might not their timestamps updated for 24 hours.
 */
unsigned int dirtytime_expire_interval = 12 * 60 * 60;

N
Nick Piggin 已提交
68 69 70 71 72
static inline struct inode *wb_inode(struct list_head *head)
{
	return list_entry(head, struct inode, i_wb_list);
}

73 74 75 76 77 78 79 80
/*
 * Include the creation of the trace points after defining the
 * wb_writeback_work structure and inline functions so that the definition
 * remains local to this file.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/writeback.h>

81 82
EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);

83 84 85 86 87 88
static bool wb_io_lists_populated(struct bdi_writeback *wb)
{
	if (wb_has_dirty_io(wb)) {
		return false;
	} else {
		set_bit(WB_has_dirty_io, &wb->state);
89
		WARN_ON_ONCE(!wb->avg_write_bandwidth);
90 91
		atomic_long_add(wb->avg_write_bandwidth,
				&wb->bdi->tot_write_bandwidth);
92 93 94 95 96 97 98
		return true;
	}
}

static void wb_io_lists_depopulated(struct bdi_writeback *wb)
{
	if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&
99
	    list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {
100
		clear_bit(WB_has_dirty_io, &wb->state);
101 102
		WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,
					&wb->bdi->tot_write_bandwidth) < 0);
103
	}
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
}

/**
 * inode_wb_list_move_locked - move an inode onto a bdi_writeback IO list
 * @inode: inode to be moved
 * @wb: target bdi_writeback
 * @head: one of @wb->b_{dirty|io|more_io}
 *
 * Move @inode->i_wb_list to @list of @wb and set %WB_has_dirty_io.
 * Returns %true if @inode is the first occupant of the !dirty_time IO
 * lists; otherwise, %false.
 */
static bool inode_wb_list_move_locked(struct inode *inode,
				      struct bdi_writeback *wb,
				      struct list_head *head)
{
	assert_spin_locked(&wb->list_lock);

	list_move(&inode->i_wb_list, head);

	/* dirty_time doesn't count as dirty_io until expiration */
	if (head != &wb->b_dirty_time)
		return wb_io_lists_populated(wb);

	wb_io_lists_depopulated(wb);
	return false;
}

/**
 * inode_wb_list_del_locked - remove an inode from its bdi_writeback IO list
 * @inode: inode to be removed
 * @wb: bdi_writeback @inode is being removed from
 *
 * Remove @inode which may be on one of @wb->b_{dirty|io|more_io} lists and
 * clear %WB_has_dirty_io if all are empty afterwards.
 */
static void inode_wb_list_del_locked(struct inode *inode,
				     struct bdi_writeback *wb)
{
	assert_spin_locked(&wb->list_lock);

	list_del_init(&inode->i_wb_list);
	wb_io_lists_depopulated(wb);
}

149
static void wb_wakeup(struct bdi_writeback *wb)
J
Jan Kara 已提交
150
{
151 152 153 154
	spin_lock_bh(&wb->work_lock);
	if (test_bit(WB_registered, &wb->state))
		mod_delayed_work(bdi_wq, &wb->dwork, 0);
	spin_unlock_bh(&wb->work_lock);
J
Jan Kara 已提交
155 156
}

157 158
static void wb_queue_work(struct bdi_writeback *wb,
			  struct wb_writeback_work *work)
159
{
160
	trace_writeback_queue(wb->bdi, work);
161

162 163
	spin_lock_bh(&wb->work_lock);
	if (!test_bit(WB_registered, &wb->state)) {
J
Jan Kara 已提交
164 165 166 167
		if (work->done)
			complete(work->done);
		goto out_unlock;
	}
168 169
	list_add_tail(&work->list, &wb->work_list);
	mod_delayed_work(bdi_wq, &wb->dwork, 0);
J
Jan Kara 已提交
170
out_unlock:
171
	spin_unlock_bh(&wb->work_lock);
L
Linus Torvalds 已提交
172 173
}

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
#ifdef CONFIG_CGROUP_WRITEBACK

/**
 * inode_congested - test whether an inode is congested
 * @inode: inode to test for congestion
 * @cong_bits: mask of WB_[a]sync_congested bits to test
 *
 * Tests whether @inode is congested.  @cong_bits is the mask of congestion
 * bits to test and the return value is the mask of set bits.
 *
 * If cgroup writeback is enabled for @inode, the congestion state is
 * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg
 * associated with @inode is congested; otherwise, the root wb's congestion
 * state is used.
 */
int inode_congested(struct inode *inode, int cong_bits)
{
	if (inode) {
		struct bdi_writeback *wb = inode_to_wb(inode);
		if (wb)
			return wb_congested(wb, cong_bits);
	}

	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
}
EXPORT_SYMBOL_GPL(inode_congested);

#endif	/* CONFIG_CGROUP_WRITEBACK */

203 204
void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
			bool range_cyclic, enum wb_reason reason)
205
{
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
	struct wb_writeback_work *work;

	if (!wb_has_dirty_io(wb))
		return;

	/*
	 * This is WB_SYNC_NONE writeback, so if allocation fails just
	 * wakeup the thread for old dirty data writeback
	 */
	work = kzalloc(sizeof(*work), GFP_ATOMIC);
	if (!work) {
		trace_writeback_nowork(wb->bdi);
		wb_wakeup(wb);
		return;
	}

	work->sync_mode	= WB_SYNC_NONE;
	work->nr_pages	= nr_pages;
	work->range_cyclic = range_cyclic;
	work->reason	= reason;

	wb_queue_work(wb, work);
228
}
229

230 231 232 233 234
/**
 * bdi_start_background_writeback - start background writeback
 * @bdi: the backing device to write from
 *
 * Description:
235 236 237 238
 *   This makes sure WB_SYNC_NONE background writeback happens. When
 *   this function returns, it is only guaranteed that for given BDI
 *   some IO is happening if we are over background dirty threshold.
 *   Caller need not hold sb s_umount semaphore.
239 240 241
 */
void bdi_start_background_writeback(struct backing_dev_info *bdi)
{
242 243 244 245
	/*
	 * We just wake up the flusher thread. It will perform background
	 * writeback as soon as there is no other work to do.
	 */
246
	trace_writeback_wake_background(bdi);
247
	wb_wakeup(&bdi->wb);
L
Linus Torvalds 已提交
248 249
}

250 251 252 253 254
/*
 * Remove the inode from the writeback list it is on.
 */
void inode_wb_list_del(struct inode *inode)
{
255
	struct bdi_writeback *wb = inode_to_wb(inode);
256

257
	spin_lock(&wb->list_lock);
258
	inode_wb_list_del_locked(inode, wb);
259
	spin_unlock(&wb->list_lock);
260 261
}

262 263 264 265 266
/*
 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
 * furthest end of its superblock's dirty-inode list.
 *
 * Before stamping the inode's ->dirtied_when, we check to see whether it is
267
 * already the most-recently-dirtied inode on the b_dirty list.  If that is
268 269 270
 * the case then the inode must have been redirtied while it was being written
 * out and we don't reset its dirtied_when.
 */
271
static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
272
{
273
	if (!list_empty(&wb->b_dirty)) {
274
		struct inode *tail;
275

N
Nick Piggin 已提交
276
		tail = wb_inode(wb->b_dirty.next);
277
		if (time_before(inode->dirtied_when, tail->dirtied_when))
278 279
			inode->dirtied_when = jiffies;
	}
280
	inode_wb_list_move_locked(inode, wb, &wb->b_dirty);
281 282
}

283
/*
284
 * requeue inode for re-scanning after bdi->b_io list is exhausted.
285
 */
286
static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
287
{
288
	inode_wb_list_move_locked(inode, wb, &wb->b_more_io);
289 290
}

J
Joern Engel 已提交
291 292
static void inode_sync_complete(struct inode *inode)
{
293
	inode->i_state &= ~I_SYNC;
294 295
	/* If inode is clean an unused, put it into LRU now... */
	inode_add_lru(inode);
296
	/* Waiters must see I_SYNC cleared before being woken up */
J
Joern Engel 已提交
297 298 299 300
	smp_mb();
	wake_up_bit(&inode->i_state, __I_SYNC);
}

301 302 303 304 305 306 307 308
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
	bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
	/*
	 * For inodes being constantly redirtied, dirtied_when can get stuck.
	 * It _appears_ to be in the future, but is actually in distant past.
	 * This test is necessary to prevent such wrapped-around relative times
309
	 * from permanently stopping the whole bdi writeback.
310 311 312 313 314 315
	 */
	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
	return ret;
}

316 317
#define EXPIRE_DIRTY_ATIME 0x0001

318
/*
319
 * Move expired (dirtied before work->older_than_this) dirty inodes from
J
Jan Kara 已提交
320
 * @delaying_queue to @dispatch_queue.
321
 */
322
static int move_expired_inodes(struct list_head *delaying_queue,
323
			       struct list_head *dispatch_queue,
324
			       int flags,
325
			       struct wb_writeback_work *work)
326
{
327 328
	unsigned long *older_than_this = NULL;
	unsigned long expire_time;
329 330
	LIST_HEAD(tmp);
	struct list_head *pos, *node;
331
	struct super_block *sb = NULL;
332
	struct inode *inode;
333
	int do_sb_sort = 0;
334
	int moved = 0;
335

336 337
	if ((flags & EXPIRE_DIRTY_ATIME) == 0)
		older_than_this = work->older_than_this;
338 339
	else if (!work->for_sync) {
		expire_time = jiffies - (dirtytime_expire_interval * HZ);
340 341
		older_than_this = &expire_time;
	}
342
	while (!list_empty(delaying_queue)) {
N
Nick Piggin 已提交
343
		inode = wb_inode(delaying_queue->prev);
344 345
		if (older_than_this &&
		    inode_dirtied_after(inode, *older_than_this))
346
			break;
347 348
		list_move(&inode->i_wb_list, &tmp);
		moved++;
349 350
		if (flags & EXPIRE_DIRTY_ATIME)
			set_bit(__I_DIRTY_TIME_EXPIRED, &inode->i_state);
351 352
		if (sb_is_blkdev_sb(inode->i_sb))
			continue;
353 354 355
		if (sb && sb != inode->i_sb)
			do_sb_sort = 1;
		sb = inode->i_sb;
356 357
	}

358 359 360
	/* just one sb in list, splice to dispatch_queue and we're done */
	if (!do_sb_sort) {
		list_splice(&tmp, dispatch_queue);
361
		goto out;
362 363
	}

364 365
	/* Move inodes from one superblock together */
	while (!list_empty(&tmp)) {
N
Nick Piggin 已提交
366
		sb = wb_inode(tmp.prev)->i_sb;
367
		list_for_each_prev_safe(pos, node, &tmp) {
N
Nick Piggin 已提交
368
			inode = wb_inode(pos);
369
			if (inode->i_sb == sb)
N
Nick Piggin 已提交
370
				list_move(&inode->i_wb_list, dispatch_queue);
371
		}
372
	}
373 374
out:
	return moved;
375 376 377 378
}

/*
 * Queue all expired dirty inodes for io, eldest first.
379 380 381 382 383 384 385 386
 * Before
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    gf         edc     BA
 * After
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    g          fBAedc
 *                                           |
 *                                           +--> dequeue for IO
387
 */
388
static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work)
389
{
390
	int moved;
391

392
	assert_spin_locked(&wb->list_lock);
393
	list_splice_init(&wb->b_more_io, &wb->b_io);
394 395 396
	moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, 0, work);
	moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,
				     EXPIRE_DIRTY_ATIME, work);
397 398
	if (moved)
		wb_io_lists_populated(wb);
399
	trace_writeback_queue_io(wb, work, moved);
400 401
}

402
static int write_inode(struct inode *inode, struct writeback_control *wbc)
403
{
T
Tejun Heo 已提交
404 405 406 407 408 409 410 411
	int ret;

	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
		trace_writeback_write_inode_start(inode, wbc);
		ret = inode->i_sb->s_op->write_inode(inode, wbc);
		trace_writeback_write_inode(inode, wbc);
		return ret;
	}
412
	return 0;
413 414
}

L
Linus Torvalds 已提交
415
/*
416 417
 * Wait for writeback on an inode to complete. Called with i_lock held.
 * Caller must make sure inode cannot go away when we drop i_lock.
418
 */
419 420 421
static void __inode_wait_for_writeback(struct inode *inode)
	__releases(inode->i_lock)
	__acquires(inode->i_lock)
422 423 424 425 426
{
	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
	wait_queue_head_t *wqh;

	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
427 428
	while (inode->i_state & I_SYNC) {
		spin_unlock(&inode->i_lock);
429 430
		__wait_on_bit(wqh, &wq, bit_wait,
			      TASK_UNINTERRUPTIBLE);
431
		spin_lock(&inode->i_lock);
432
	}
433 434
}

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
/*
 * Wait for writeback on an inode to complete. Caller must have inode pinned.
 */
void inode_wait_for_writeback(struct inode *inode)
{
	spin_lock(&inode->i_lock);
	__inode_wait_for_writeback(inode);
	spin_unlock(&inode->i_lock);
}

/*
 * Sleep until I_SYNC is cleared. This function must be called with i_lock
 * held and drops it. It is aimed for callers not holding any inode reference
 * so once i_lock is dropped, inode can go away.
 */
static void inode_sleep_on_writeback(struct inode *inode)
	__releases(inode->i_lock)
{
	DEFINE_WAIT(wait);
	wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
	int sleep;

	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	sleep = inode->i_state & I_SYNC;
	spin_unlock(&inode->i_lock);
	if (sleep)
		schedule();
	finish_wait(wqh, &wait);
}

465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
/*
 * Find proper writeback list for the inode depending on its current state and
 * possibly also change of its state while we were doing writeback.  Here we
 * handle things such as livelock prevention or fairness of writeback among
 * inodes. This function can be called only by flusher thread - noone else
 * processes all inodes in writeback lists and requeueing inodes behind flusher
 * thread's back can have unexpected consequences.
 */
static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
			  struct writeback_control *wbc)
{
	if (inode->i_state & I_FREEING)
		return;

	/*
	 * Sync livelock prevention. Each inode is tagged and synced in one
	 * shot. If still dirty, it will be redirty_tail()'ed below.  Update
	 * the dirty time to prevent enqueue and sync it again.
	 */
	if ((inode->i_state & I_DIRTY) &&
	    (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
		inode->dirtied_when = jiffies;

488 489 490 491 492 493 494 495 496
	if (wbc->pages_skipped) {
		/*
		 * writeback is not making progress due to locked
		 * buffers. Skip this inode for now.
		 */
		redirty_tail(inode, wb);
		return;
	}

497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
	if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
		/*
		 * We didn't write back all the pages.  nfs_writepages()
		 * sometimes bales out without doing anything.
		 */
		if (wbc->nr_to_write <= 0) {
			/* Slice used up. Queue for next turn. */
			requeue_io(inode, wb);
		} else {
			/*
			 * Writeback blocked by something other than
			 * congestion. Delay the inode for some time to
			 * avoid spinning on the CPU (100% iowait)
			 * retrying writeback of the dirty page/inode
			 * that cannot be performed immediately.
			 */
			redirty_tail(inode, wb);
		}
	} else if (inode->i_state & I_DIRTY) {
		/*
		 * Filesystems can dirty the inode during writeback operations,
		 * such as delayed allocation during submission or metadata
		 * updates after data IO completion.
		 */
		redirty_tail(inode, wb);
522
	} else if (inode->i_state & I_DIRTY_TIME) {
523
		inode->dirtied_when = jiffies;
524
		inode_wb_list_move_locked(inode, wb, &wb->b_dirty_time);
525 526
	} else {
		/* The inode is clean. Remove from writeback lists. */
527
		inode_wb_list_del_locked(inode, wb);
528 529 530
	}
}

531
/*
532 533 534
 * Write out an inode and its dirty pages. Do not update the writeback list
 * linkage. That is left to the caller. The caller is also responsible for
 * setting I_SYNC flag and calling inode_sync_complete() to clear it.
L
Linus Torvalds 已提交
535 536
 */
static int
537
__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
L
Linus Torvalds 已提交
538 539
{
	struct address_space *mapping = inode->i_mapping;
540
	long nr_to_write = wbc->nr_to_write;
541
	unsigned dirty;
L
Linus Torvalds 已提交
542 543
	int ret;

544
	WARN_ON(!(inode->i_state & I_SYNC));
L
Linus Torvalds 已提交
545

T
Tejun Heo 已提交
546 547
	trace_writeback_single_inode_start(inode, wbc, nr_to_write);

L
Linus Torvalds 已提交
548 549
	ret = do_writepages(mapping, wbc);

550 551 552
	/*
	 * Make sure to wait on the data before writing out the metadata.
	 * This is important for filesystems that modify metadata on data
553 554 555
	 * I/O completion. We don't do it for sync(2) writeback because it has a
	 * separate, external IO completion path and ->sync_fs for guaranteeing
	 * inode metadata is written back correctly.
556
	 */
557
	if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
558
		int err = filemap_fdatawait(mapping);
L
Linus Torvalds 已提交
559 560 561 562
		if (ret == 0)
			ret = err;
	}

563 564 565 566 567
	/*
	 * Some filesystems may redirty the inode during the writeback
	 * due to delalloc, clear dirty metadata flags right before
	 * write_inode()
	 */
568
	spin_lock(&inode->i_lock);
569

570
	dirty = inode->i_state & I_DIRTY;
571 572 573 574 575 576 577 578 579 580 581
	if (inode->i_state & I_DIRTY_TIME) {
		if ((dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) ||
		    unlikely(inode->i_state & I_DIRTY_TIME_EXPIRED) ||
		    unlikely(time_after(jiffies,
					(inode->dirtied_time_when +
					 dirtytime_expire_interval * HZ)))) {
			dirty |= I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED;
			trace_writeback_lazytime(inode);
		}
	} else
		inode->i_state &= ~I_DIRTY_TIME_EXPIRED;
582
	inode->i_state &= ~dirty;
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599

	/*
	 * Paired with smp_mb() in __mark_inode_dirty().  This allows
	 * __mark_inode_dirty() to test i_state without grabbing i_lock -
	 * either they see the I_DIRTY bits cleared or we see the dirtied
	 * inode.
	 *
	 * I_DIRTY_PAGES is always cleared together above even if @mapping
	 * still has dirty pages.  The flag is reinstated after smp_mb() if
	 * necessary.  This guarantees that either __mark_inode_dirty()
	 * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
	 */
	smp_mb();

	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
		inode->i_state |= I_DIRTY_PAGES;

600
	spin_unlock(&inode->i_lock);
601

602 603
	if (dirty & I_DIRTY_TIME)
		mark_inode_dirty_sync(inode);
604
	/* Don't write the inode if only I_DIRTY_PAGES was set */
605
	if (dirty & ~I_DIRTY_PAGES) {
606
		int err = write_inode(inode, wbc);
L
Linus Torvalds 已提交
607 608 609
		if (ret == 0)
			ret = err;
	}
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
	trace_writeback_single_inode(inode, wbc, nr_to_write);
	return ret;
}

/*
 * Write out an inode's dirty pages. Either the caller has an active reference
 * on the inode or the inode has I_WILL_FREE set.
 *
 * This function is designed to be called for writing back one inode which
 * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
 * and does more profound writeback list handling in writeback_sb_inodes().
 */
static int
writeback_single_inode(struct inode *inode, struct bdi_writeback *wb,
		       struct writeback_control *wbc)
{
	int ret = 0;

	spin_lock(&inode->i_lock);
	if (!atomic_read(&inode->i_count))
		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
	else
		WARN_ON(inode->i_state & I_WILL_FREE);

	if (inode->i_state & I_SYNC) {
		if (wbc->sync_mode != WB_SYNC_ALL)
			goto out;
		/*
638 639 640
		 * It's a data-integrity sync. We must wait. Since callers hold
		 * inode reference or inode has I_WILL_FREE set, it cannot go
		 * away under us.
641
		 */
642
		__inode_wait_for_writeback(inode);
643 644 645
	}
	WARN_ON(inode->i_state & I_SYNC);
	/*
J
Jan Kara 已提交
646 647 648 649 650 651
	 * Skip inode if it is clean and we have no outstanding writeback in
	 * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this
	 * function since flusher thread may be doing for example sync in
	 * parallel and if we move the inode, it could get skipped. So here we
	 * make sure inode is on some writeback list and leave it there unless
	 * we have completely cleaned the inode.
652
	 */
653
	if (!(inode->i_state & I_DIRTY_ALL) &&
J
Jan Kara 已提交
654 655
	    (wbc->sync_mode != WB_SYNC_ALL ||
	     !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
656 657 658 659
		goto out;
	inode->i_state |= I_SYNC;
	spin_unlock(&inode->i_lock);

660
	ret = __writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
661

662
	spin_lock(&wb->list_lock);
663
	spin_lock(&inode->i_lock);
664 665 666 667
	/*
	 * If inode is clean, remove it from writeback lists. Otherwise don't
	 * touch it. See comment above for explanation.
	 */
668
	if (!(inode->i_state & I_DIRTY_ALL))
669
		inode_wb_list_del_locked(inode, wb);
670
	spin_unlock(&wb->list_lock);
J
Joern Engel 已提交
671
	inode_sync_complete(inode);
672 673
out:
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
674 675 676
	return ret;
}

677
static long writeback_chunk_size(struct bdi_writeback *wb,
678
				 struct wb_writeback_work *work)
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
{
	long pages;

	/*
	 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
	 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
	 * here avoids calling into writeback_inodes_wb() more than once.
	 *
	 * The intended call sequence for WB_SYNC_ALL writeback is:
	 *
	 *      wb_writeback()
	 *          writeback_sb_inodes()       <== called only once
	 *              write_cache_pages()     <== called once for each inode
	 *                   (quickly) tag currently dirty pages
	 *                   (maybe slowly) sync all tagged pages
	 */
	if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
		pages = LONG_MAX;
697
	else {
698
		pages = min(wb->avg_write_bandwidth / 2,
699 700 701 702 703
			    global_dirty_limit / DIRTY_SCOPE);
		pages = min(pages, work->nr_pages);
		pages = round_down(pages + MIN_WRITEBACK_PAGES,
				   MIN_WRITEBACK_PAGES);
	}
704 705 706 707

	return pages;
}

708 709
/*
 * Write a portion of b_io inodes which belong to @sb.
710
 *
711
 * Return the number of pages and/or inodes written.
712
 */
713 714 715
static long writeback_sb_inodes(struct super_block *sb,
				struct bdi_writeback *wb,
				struct wb_writeback_work *work)
L
Linus Torvalds 已提交
716
{
717 718 719 720 721
	struct writeback_control wbc = {
		.sync_mode		= work->sync_mode,
		.tagged_writepages	= work->tagged_writepages,
		.for_kupdate		= work->for_kupdate,
		.for_background		= work->for_background,
722
		.for_sync		= work->for_sync,
723 724 725 726 727 728 729 730
		.range_cyclic		= work->range_cyclic,
		.range_start		= 0,
		.range_end		= LLONG_MAX,
	};
	unsigned long start_time = jiffies;
	long write_chunk;
	long wrote = 0;  /* count both pages and inodes */

731
	while (!list_empty(&wb->b_io)) {
N
Nick Piggin 已提交
732
		struct inode *inode = wb_inode(wb->b_io.prev);
733 734

		if (inode->i_sb != sb) {
735
			if (work->sb) {
736 737 738 739 740
				/*
				 * We only want to write back data for this
				 * superblock, move all inodes not belonging
				 * to it back onto the dirty list.
				 */
741
				redirty_tail(inode, wb);
742 743 744 745 746 747 748 749
				continue;
			}

			/*
			 * The inode belongs to a different superblock.
			 * Bounce back to the caller to unpin this and
			 * pin the next superblock.
			 */
750
			break;
751 752
		}

753
		/*
W
Wanpeng Li 已提交
754 755
		 * Don't bother with new inodes or inodes being freed, first
		 * kind does not need periodic writeout yet, and for the latter
756 757
		 * kind writeout is handled by the freer.
		 */
758
		spin_lock(&inode->i_lock);
759
		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
760
			spin_unlock(&inode->i_lock);
761
			redirty_tail(inode, wb);
762 763
			continue;
		}
764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
		if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
			/*
			 * If this inode is locked for writeback and we are not
			 * doing writeback-for-data-integrity, move it to
			 * b_more_io so that writeback can proceed with the
			 * other inodes on s_io.
			 *
			 * We'll have another go at writing back this inode
			 * when we completed a full scan of b_io.
			 */
			spin_unlock(&inode->i_lock);
			requeue_io(inode, wb);
			trace_writeback_sb_inodes_requeue(inode);
			continue;
		}
779 780
		spin_unlock(&wb->list_lock);

781 782 783 784 785
		/*
		 * We already requeued the inode if it had I_SYNC set and we
		 * are doing WB_SYNC_NONE writeback. So this catches only the
		 * WB_SYNC_ALL case.
		 */
786 787 788 789
		if (inode->i_state & I_SYNC) {
			/* Wait for I_SYNC. This function drops i_lock... */
			inode_sleep_on_writeback(inode);
			/* Inode may be gone, start again */
790
			spin_lock(&wb->list_lock);
791 792
			continue;
		}
793 794
		inode->i_state |= I_SYNC;
		spin_unlock(&inode->i_lock);
795

796
		write_chunk = writeback_chunk_size(wb, work);
797 798
		wbc.nr_to_write = write_chunk;
		wbc.pages_skipped = 0;
799

800 801 802 803
		/*
		 * We use I_SYNC to pin the inode in memory. While it is set
		 * evict_inode() will wait so the inode cannot be freed.
		 */
804
		__writeback_single_inode(inode, &wbc);
805

806 807
		work->nr_pages -= write_chunk - wbc.nr_to_write;
		wrote += write_chunk - wbc.nr_to_write;
808 809
		spin_lock(&wb->list_lock);
		spin_lock(&inode->i_lock);
810
		if (!(inode->i_state & I_DIRTY_ALL))
811
			wrote++;
812 813
		requeue_inode(inode, wb, &wbc);
		inode_sync_complete(inode);
814
		spin_unlock(&inode->i_lock);
815
		cond_resched_lock(&wb->list_lock);
816 817 818 819 820 821 822 823 824
		/*
		 * bail out to wb_writeback() often enough to check
		 * background threshold and other termination conditions.
		 */
		if (wrote) {
			if (time_is_before_jiffies(start_time + HZ / 10UL))
				break;
			if (work->nr_pages <= 0)
				break;
825
		}
L
Linus Torvalds 已提交
826
	}
827
	return wrote;
828 829
}

830 831
static long __writeback_inodes_wb(struct bdi_writeback *wb,
				  struct wb_writeback_work *work)
832
{
833 834
	unsigned long start_time = jiffies;
	long wrote = 0;
N
Nick Piggin 已提交
835

836
	while (!list_empty(&wb->b_io)) {
N
Nick Piggin 已提交
837
		struct inode *inode = wb_inode(wb->b_io.prev);
838
		struct super_block *sb = inode->i_sb;
839

840
		if (!trylock_super(sb)) {
841
			/*
842
			 * trylock_super() may fail consistently due to
843 844 845 846
			 * s_umount being grabbed by someone else. Don't use
			 * requeue_io() to avoid busy retrying the inode/sb.
			 */
			redirty_tail(inode, wb);
847
			continue;
848
		}
849
		wrote += writeback_sb_inodes(sb, wb, work);
850
		up_read(&sb->s_umount);
851

852 853 854 855 856 857 858
		/* refer to the same tests at the end of writeback_sb_inodes */
		if (wrote) {
			if (time_is_before_jiffies(start_time + HZ / 10UL))
				break;
			if (work->nr_pages <= 0)
				break;
		}
859
	}
860
	/* Leave any unwritten inodes on b_io */
861
	return wrote;
862 863
}

864
static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
865
				enum wb_reason reason)
866
{
867 868 869 870
	struct wb_writeback_work work = {
		.nr_pages	= nr_pages,
		.sync_mode	= WB_SYNC_NONE,
		.range_cyclic	= 1,
871
		.reason		= reason,
872
	};
873

874
	spin_lock(&wb->list_lock);
W
Wu Fengguang 已提交
875
	if (list_empty(&wb->b_io))
876
		queue_io(wb, &work);
877
	__writeback_inodes_wb(wb, &work);
878
	spin_unlock(&wb->list_lock);
879

880 881
	return nr_pages - work.nr_pages;
}
882

883
static bool over_bground_thresh(struct bdi_writeback *wb)
884 885 886
{
	unsigned long background_thresh, dirty_thresh;

887
	global_dirty_limits(&background_thresh, &dirty_thresh);
888

889 890 891 892
	if (global_page_state(NR_FILE_DIRTY) +
	    global_page_state(NR_UNSTABLE_NFS) > background_thresh)
		return true;

893
	if (wb_stat(wb, WB_RECLAIMABLE) > wb_dirty_limit(wb, background_thresh))
894 895 896
		return true;

	return false;
897 898
}

899 900 901 902 903 904 905
/*
 * Called under wb->list_lock. If there are multiple wb per bdi,
 * only the flusher working on the first wb should do it.
 */
static void wb_update_bandwidth(struct bdi_writeback *wb,
				unsigned long start_time)
{
906
	__wb_update_bandwidth(wb, 0, 0, 0, 0, 0, start_time);
907 908
}

909 910
/*
 * Explicit flushing or periodic writeback of "old" data.
911
 *
912 913 914 915
 * Define "old": the first time one of an inode's pages is dirtied, we mark the
 * dirtying-time in the inode's address_space.  So this periodic writeback code
 * just walks the superblock inode list, writing back any inodes which are
 * older than a specific point in time.
916
 *
917 918 919
 * Try to run once per dirty_writeback_interval.  But if a writeback event
 * takes longer than a dirty_writeback_interval interval, then leave a
 * one-second gap.
920
 *
921 922
 * older_than_this takes precedence over nr_to_write.  So we'll only write back
 * all dirty pages if they are all attached to "old" mappings.
923
 */
924
static long wb_writeback(struct bdi_writeback *wb,
925
			 struct wb_writeback_work *work)
926
{
927
	unsigned long wb_start = jiffies;
928
	long nr_pages = work->nr_pages;
929
	unsigned long oldest_jif;
J
Jan Kara 已提交
930
	struct inode *inode;
931
	long progress;
932

933 934
	oldest_jif = jiffies;
	work->older_than_this = &oldest_jif;
N
Nick Piggin 已提交
935

936
	spin_lock(&wb->list_lock);
937 938
	for (;;) {
		/*
939
		 * Stop writeback when nr_pages has been consumed
940
		 */
941
		if (work->nr_pages <= 0)
942
			break;
943

944 945 946 947 948 949 950
		/*
		 * Background writeout and kupdate-style writeback may
		 * run forever. Stop them if there is other work to do
		 * so that e.g. sync can proceed. They'll be restarted
		 * after the other works are all done.
		 */
		if ((work->for_background || work->for_kupdate) &&
951
		    !list_empty(&wb->work_list))
952 953
			break;

N
Nick Piggin 已提交
954
		/*
955 956
		 * For background writeout, stop when we are below the
		 * background dirty threshold
N
Nick Piggin 已提交
957
		 */
958
		if (work->for_background && !over_bground_thresh(wb))
959
			break;
N
Nick Piggin 已提交
960

961 962 963 964 965 966
		/*
		 * Kupdate and background works are special and we want to
		 * include all inodes that need writing. Livelock avoidance is
		 * handled by these works yielding to any other work so we are
		 * safe.
		 */
967
		if (work->for_kupdate) {
968
			oldest_jif = jiffies -
969
				msecs_to_jiffies(dirty_expire_interval * 10);
970
		} else if (work->for_background)
971
			oldest_jif = jiffies;
972

973
		trace_writeback_start(wb->bdi, work);
974
		if (list_empty(&wb->b_io))
975
			queue_io(wb, work);
976
		if (work->sb)
977
			progress = writeback_sb_inodes(work->sb, wb, work);
978
		else
979 980
			progress = __writeback_inodes_wb(wb, work);
		trace_writeback_written(wb->bdi, work);
981

982
		wb_update_bandwidth(wb, wb_start);
983 984

		/*
985 986 987 988 989 990
		 * Did we write something? Try for more
		 *
		 * Dirty inodes are moved to b_io for writeback in batches.
		 * The completion of the current batch does not necessarily
		 * mean the overall work is done. So we keep looping as long
		 * as made some progress on cleaning pages or inodes.
991
		 */
992
		if (progress)
993 994
			continue;
		/*
995
		 * No more inodes for IO, bail
996
		 */
997
		if (list_empty(&wb->b_more_io))
998
			break;
999 1000 1001 1002 1003 1004
		/*
		 * Nothing written. Wait for some inode to
		 * become available for writeback. Otherwise
		 * we'll just busyloop.
		 */
		if (!list_empty(&wb->b_more_io))  {
1005
			trace_writeback_wait(wb->bdi, work);
N
Nick Piggin 已提交
1006
			inode = wb_inode(wb->b_more_io.prev);
1007
			spin_lock(&inode->i_lock);
1008
			spin_unlock(&wb->list_lock);
1009 1010
			/* This function drops i_lock... */
			inode_sleep_on_writeback(inode);
1011
			spin_lock(&wb->list_lock);
1012 1013
		}
	}
1014
	spin_unlock(&wb->list_lock);
1015

1016
	return nr_pages - work->nr_pages;
1017 1018 1019
}

/*
1020
 * Return the next wb_writeback_work struct that hasn't been processed yet.
1021
 */
1022
static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)
1023
{
1024
	struct wb_writeback_work *work = NULL;
1025

1026 1027 1028
	spin_lock_bh(&wb->work_lock);
	if (!list_empty(&wb->work_list)) {
		work = list_entry(wb->work_list.next,
1029 1030
				  struct wb_writeback_work, list);
		list_del_init(&work->list);
1031
	}
1032
	spin_unlock_bh(&wb->work_lock);
1033
	return work;
1034 1035
}

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
/*
 * Add in the number of potentially dirty inodes, because each inode
 * write can dirty pagecache in the underlying blockdev.
 */
static unsigned long get_nr_dirty_pages(void)
{
	return global_page_state(NR_FILE_DIRTY) +
		global_page_state(NR_UNSTABLE_NFS) +
		get_nr_dirty_inodes();
}

1047 1048
static long wb_check_background_flush(struct bdi_writeback *wb)
{
1049
	if (over_bground_thresh(wb)) {
1050 1051 1052 1053 1054 1055

		struct wb_writeback_work work = {
			.nr_pages	= LONG_MAX,
			.sync_mode	= WB_SYNC_NONE,
			.for_background	= 1,
			.range_cyclic	= 1,
1056
			.reason		= WB_REASON_BACKGROUND,
1057 1058 1059 1060 1061 1062 1063 1064
		};

		return wb_writeback(wb, &work);
	}

	return 0;
}

1065 1066 1067 1068 1069
static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
	unsigned long expired;
	long nr_pages;

1070 1071 1072 1073 1074 1075
	/*
	 * When set to zero, disable periodic writeback
	 */
	if (!dirty_writeback_interval)
		return 0;

1076 1077 1078 1079 1080 1081
	expired = wb->last_old_flush +
			msecs_to_jiffies(dirty_writeback_interval * 10);
	if (time_before(jiffies, expired))
		return 0;

	wb->last_old_flush = jiffies;
1082
	nr_pages = get_nr_dirty_pages();
1083

1084
	if (nr_pages) {
1085
		struct wb_writeback_work work = {
1086 1087 1088 1089
			.nr_pages	= nr_pages,
			.sync_mode	= WB_SYNC_NONE,
			.for_kupdate	= 1,
			.range_cyclic	= 1,
1090
			.reason		= WB_REASON_PERIODIC,
1091 1092
		};

1093
		return wb_writeback(wb, &work);
1094
	}
1095 1096 1097 1098 1099 1100 1101

	return 0;
}

/*
 * Retrieve work items and do the writeback they describe
 */
1102
static long wb_do_writeback(struct bdi_writeback *wb)
1103
{
1104
	struct wb_writeback_work *work;
1105
	long wrote = 0;
1106

1107
	set_bit(WB_writeback_running, &wb->state);
1108
	while ((work = get_next_work_item(wb)) != NULL) {
1109

1110
		trace_writeback_exec(wb->bdi, work);
1111

1112
		wrote += wb_writeback(wb, work);
1113 1114

		/*
1115 1116
		 * Notify the caller of completion if this is a synchronous
		 * work item, otherwise just free it.
1117
		 */
1118 1119 1120 1121
		if (work->done)
			complete(work->done);
		else
			kfree(work);
1122 1123 1124 1125 1126 1127
	}

	/*
	 * Check for periodic writeback, kupdated() style
	 */
	wrote += wb_check_old_data_flush(wb);
1128
	wrote += wb_check_background_flush(wb);
1129
	clear_bit(WB_writeback_running, &wb->state);
1130 1131 1132 1133 1134 1135

	return wrote;
}

/*
 * Handle writeback of dirty data for the device backed by this bdi. Also
1136
 * reschedules periodically and does kupdated style flushing.
1137
 */
1138
void wb_workfn(struct work_struct *work)
1139
{
1140 1141
	struct bdi_writeback *wb = container_of(to_delayed_work(work),
						struct bdi_writeback, dwork);
1142 1143
	long pages_written;

1144
	set_worker_desc("flush-%s", dev_name(wb->bdi->dev));
P
Peter Zijlstra 已提交
1145
	current->flags |= PF_SWAPWRITE;
1146

1147
	if (likely(!current_is_workqueue_rescuer() ||
1148
		   !test_bit(WB_registered, &wb->state))) {
1149
		/*
1150
		 * The normal path.  Keep writing back @wb until its
1151
		 * work_list is empty.  Note that this path is also taken
1152
		 * if @wb is shutting down even when we're running off the
1153
		 * rescuer as work_list needs to be drained.
1154
		 */
1155
		do {
1156
			pages_written = wb_do_writeback(wb);
1157
			trace_writeback_pages_written(pages_written);
1158
		} while (!list_empty(&wb->work_list));
1159 1160 1161 1162 1163 1164
	} else {
		/*
		 * bdi_wq can't get enough workers and we're running off
		 * the emergency worker.  Don't hog it.  Hopefully, 1024 is
		 * enough for efficient IO.
		 */
1165
		pages_written = writeback_inodes_wb(wb, 1024,
1166
						    WB_REASON_FORKER_THREAD);
1167
		trace_writeback_pages_written(pages_written);
1168 1169
	}

1170
	if (!list_empty(&wb->work_list))
1171 1172
		mod_delayed_work(bdi_wq, &wb->dwork, 0);
	else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
1173
		wb_wakeup_delayed(wb);
1174

1175
	current->flags &= ~PF_SWAPWRITE;
1176 1177 1178
}

/*
1179 1180
 * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
 * the whole world.
1181
 */
1182
void wakeup_flusher_threads(long nr_pages, enum wb_reason reason)
1183
{
1184
	struct backing_dev_info *bdi;
1185

1186 1187
	if (!nr_pages)
		nr_pages = get_nr_dirty_pages();
1188

1189
	rcu_read_lock();
1190
	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
1191
		wb_start_writeback(&bdi->wb, nr_pages, false, reason);
1192
	rcu_read_unlock();
L
Linus Torvalds 已提交
1193 1194
}

1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
/*
 * Wake up bdi's periodically to make sure dirtytime inodes gets
 * written back periodically.  We deliberately do *not* check the
 * b_dirtytime list in wb_has_dirty_io(), since this would cause the
 * kernel to be constantly waking up once there are any dirtytime
 * inodes on the system.  So instead we define a separate delayed work
 * function which gets called much more rarely.  (By default, only
 * once every 12 hours.)
 *
 * If there is any other write activity going on in the file system,
 * this function won't be necessary.  But if the only thing that has
 * happened on the file system is a dirtytime inode caused by an atime
 * update, we need this infrastructure below to make sure that inode
 * eventually gets pushed out to disk.
 */
static void wakeup_dirtytime_writeback(struct work_struct *w);
static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);

static void wakeup_dirtytime_writeback(struct work_struct *w)
{
	struct backing_dev_info *bdi;

	rcu_read_lock();
	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
		if (list_empty(&bdi->wb.b_dirty_time))
			continue;
1221
		wb_wakeup(&bdi->wb);
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
	}
	rcu_read_unlock();
	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
}

static int __init start_dirtytime_writeback(void)
{
	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
	return 0;
}
__initcall(start_dirtytime_writeback);

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
int dirtytime_interval_handler(struct ctl_table *table, int write,
			       void __user *buffer, size_t *lenp, loff_t *ppos)
{
	int ret;

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
	if (ret == 0 && write)
		mod_delayed_work(system_wq, &dirtytime_work, 0);
	return ret;
}

1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
static noinline void block_dump___mark_inode_dirty(struct inode *inode)
{
	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
		struct dentry *dentry;
		const char *name = "?";

		dentry = d_find_alias(inode);
		if (dentry) {
			spin_lock(&dentry->d_lock);
			name = (const char *) dentry->d_name.name;
		}
		printk(KERN_DEBUG
		       "%s(%d): dirtied inode %lu (%s) on %s\n",
		       current->comm, task_pid_nr(current), inode->i_ino,
		       name, inode->i_sb->s_id);
		if (dentry) {
			spin_unlock(&dentry->d_lock);
			dput(dentry);
		}
	}
}

/**
 *	__mark_inode_dirty -	internal function
 *	@inode: inode to mark
 *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
 *	Mark an inode as dirty. Callers should use mark_inode_dirty or
 *  	mark_inode_dirty_sync.
L
Linus Torvalds 已提交
1273
 *
1274 1275 1276 1277 1278 1279 1280 1281 1282
 * Put the inode on the super block's dirty list.
 *
 * CAREFUL! We mark it dirty unconditionally, but move it onto the
 * dirty list only if it is hashed or if it refers to a blockdev.
 * If it was not hashed, it will never be added to the dirty list
 * even if it is later hashed, as it will have been marked dirty already.
 *
 * In short, make sure you hash any inodes _before_ you start marking
 * them dirty.
L
Linus Torvalds 已提交
1283
 *
1284 1285 1286 1287 1288 1289
 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
 * the kernel-internal blockdev inode represents the dirtying time of the
 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
 * page->mapping->host, so the page-dirtying time is recorded in the internal
 * blockdev inode.
L
Linus Torvalds 已提交
1290
 */
1291
#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
1292
void __mark_inode_dirty(struct inode *inode, int flags)
L
Linus Torvalds 已提交
1293
{
1294
	struct super_block *sb = inode->i_sb;
1295
	struct backing_dev_info *bdi = NULL;
1296 1297 1298
	int dirtytime;

	trace_writeback_mark_inode_dirty(inode, flags);
L
Linus Torvalds 已提交
1299

1300 1301 1302 1303
	/*
	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
	 * dirty the inode itself
	 */
1304
	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC | I_DIRTY_TIME)) {
T
Tejun Heo 已提交
1305 1306
		trace_writeback_dirty_inode_start(inode, flags);

1307
		if (sb->s_op->dirty_inode)
1308
			sb->s_op->dirty_inode(inode, flags);
T
Tejun Heo 已提交
1309 1310

		trace_writeback_dirty_inode(inode, flags);
1311
	}
1312 1313 1314
	if (flags & I_DIRTY_INODE)
		flags &= ~I_DIRTY_TIME;
	dirtytime = flags & I_DIRTY_TIME;
1315 1316

	/*
1317 1318
	 * Paired with smp_mb() in __writeback_single_inode() for the
	 * following lockless i_state test.  See there for details.
1319 1320 1321
	 */
	smp_mb();

1322 1323
	if (((inode->i_state & flags) == flags) ||
	    (dirtytime && (inode->i_state & I_DIRTY_INODE)))
1324 1325 1326 1327 1328
		return;

	if (unlikely(block_dump))
		block_dump___mark_inode_dirty(inode);

1329
	spin_lock(&inode->i_lock);
1330 1331
	if (dirtytime && (inode->i_state & I_DIRTY_INODE))
		goto out_unlock_inode;
1332 1333 1334
	if ((inode->i_state & flags) != flags) {
		const int was_dirty = inode->i_state & I_DIRTY;

1335 1336
		inode_attach_wb(inode, NULL);

1337 1338
		if (flags & I_DIRTY_INODE)
			inode->i_state &= ~I_DIRTY_TIME;
1339 1340 1341 1342 1343 1344 1345 1346
		inode->i_state |= flags;

		/*
		 * If the inode is being synced, just update its dirty state.
		 * The unlocker will place the inode on the appropriate
		 * superblock list, based upon its state.
		 */
		if (inode->i_state & I_SYNC)
1347
			goto out_unlock_inode;
1348 1349 1350 1351 1352 1353

		/*
		 * Only add valid (hashed) inodes to the superblock's
		 * dirty list.  Add blockdev inodes as well.
		 */
		if (!S_ISBLK(inode->i_mode)) {
A
Al Viro 已提交
1354
			if (inode_unhashed(inode))
1355
				goto out_unlock_inode;
1356
		}
A
Al Viro 已提交
1357
		if (inode->i_state & I_FREEING)
1358
			goto out_unlock_inode;
1359 1360 1361 1362 1363 1364

		/*
		 * If the inode was already on b_dirty/b_io/b_more_io, don't
		 * reposition it (that would break b_dirty time-ordering).
		 */
		if (!was_dirty) {
1365
			struct list_head *dirty_list;
1366
			bool wakeup_bdi = false;
1367 1368
			bdi = inode_to_bdi(inode);

1369 1370
			spin_unlock(&inode->i_lock);
			spin_lock(&bdi->wb.list_lock);
1371

1372 1373 1374
			WARN(bdi_cap_writeback_dirty(bdi) &&
			     !test_bit(WB_registered, &bdi->wb.state),
			     "bdi-%s not registered\n", bdi->name);
1375 1376

			inode->dirtied_when = jiffies;
1377 1378
			if (dirtytime)
				inode->dirtied_time_when = jiffies;
1379

1380
			if (inode->i_state & (I_DIRTY_INODE | I_DIRTY_PAGES))
1381
				dirty_list = &bdi->wb.b_dirty;
1382
			else
1383 1384 1385 1386 1387
				dirty_list = &bdi->wb.b_dirty_time;

			wakeup_bdi = inode_wb_list_move_locked(inode, &bdi->wb,
							       dirty_list);

1388
			spin_unlock(&bdi->wb.list_lock);
1389
			trace_writeback_dirty_inode_enqueue(inode);
1390

1391 1392 1393 1394 1395 1396 1397
			/*
			 * If this is the first dirty inode for this bdi,
			 * we have to wake-up the corresponding bdi thread
			 * to make sure background write-back happens
			 * later.
			 */
			if (bdi_cap_writeback_dirty(bdi) && wakeup_bdi)
1398
				wb_wakeup_delayed(&bdi->wb);
1399
			return;
L
Linus Torvalds 已提交
1400 1401
		}
	}
1402 1403
out_unlock_inode:
	spin_unlock(&inode->i_lock);
1404

1405 1406 1407
}
EXPORT_SYMBOL(__mark_inode_dirty);

1408
static void wait_sb_inodes(struct super_block *sb)
1409 1410 1411 1412 1413 1414 1415
{
	struct inode *inode, *old_inode = NULL;

	/*
	 * We need to be protected against the filesystem going from
	 * r/o to r/w or vice versa.
	 */
1416
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1417

1418
	spin_lock(&inode_sb_list_lock);
1419 1420 1421 1422 1423 1424 1425 1426

	/*
	 * Data integrity sync. Must wait for all pages under writeback,
	 * because there may have been pages dirtied before our sync
	 * call, but which had writeout started before we write it out.
	 * In which case, the inode may not be on the dirty list, but
	 * we still have to wait for that writeout.
	 */
1427
	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
1428
		struct address_space *mapping = inode->i_mapping;
1429

1430 1431 1432 1433
		spin_lock(&inode->i_lock);
		if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
		    (mapping->nrpages == 0)) {
			spin_unlock(&inode->i_lock);
1434
			continue;
1435
		}
1436
		__iget(inode);
1437
		spin_unlock(&inode->i_lock);
1438 1439
		spin_unlock(&inode_sb_list_lock);

1440
		/*
1441 1442 1443 1444 1445 1446
		 * We hold a reference to 'inode' so it couldn't have been
		 * removed from s_inodes list while we dropped the
		 * inode_sb_list_lock.  We cannot iput the inode now as we can
		 * be holding the last reference and we cannot iput it under
		 * inode_sb_list_lock. So we keep the reference and iput it
		 * later.
1447 1448 1449 1450 1451 1452 1453 1454
		 */
		iput(old_inode);
		old_inode = inode;

		filemap_fdatawait(mapping);

		cond_resched();

1455
		spin_lock(&inode_sb_list_lock);
1456
	}
1457
	spin_unlock(&inode_sb_list_lock);
1458
	iput(old_inode);
L
Linus Torvalds 已提交
1459 1460
}

1461
/**
1462
 * writeback_inodes_sb_nr -	writeback dirty inodes from given super_block
1463
 * @sb: the superblock
1464
 * @nr: the number of pages to write
1465
 * @reason: reason why some writeback work initiated
L
Linus Torvalds 已提交
1466
 *
1467 1468
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
1469
 * for IO completion of submitted IO.
L
Linus Torvalds 已提交
1470
 */
1471 1472 1473
void writeback_inodes_sb_nr(struct super_block *sb,
			    unsigned long nr,
			    enum wb_reason reason)
L
Linus Torvalds 已提交
1474
{
1475 1476
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1477 1478 1479 1480 1481
		.sb			= sb,
		.sync_mode		= WB_SYNC_NONE,
		.tagged_writepages	= 1,
		.done			= &done,
		.nr_pages		= nr,
1482
		.reason			= reason,
1483
	};
1484
	struct backing_dev_info *bdi = sb->s_bdi;
1485

1486
	if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
1487
		return;
1488
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1489
	wb_queue_work(&bdi->wb, &work);
1490
	wait_for_completion(&done);
1491
}
1492 1493 1494 1495 1496
EXPORT_SYMBOL(writeback_inodes_sb_nr);

/**
 * writeback_inodes_sb	-	writeback dirty inodes from given super_block
 * @sb: the superblock
1497
 * @reason: reason why some writeback work was initiated
1498 1499 1500 1501 1502
 *
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
 * for IO completion of submitted IO.
 */
1503
void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
1504
{
1505
	return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
1506
}
1507
EXPORT_SYMBOL(writeback_inodes_sb);
1508

1509
/**
1510
 * try_to_writeback_inodes_sb_nr - try to start writeback if none underway
1511
 * @sb: the superblock
1512 1513
 * @nr: the number of pages to write
 * @reason: the reason of writeback
1514
 *
1515
 * Invoke writeback_inodes_sb_nr if no writeback is currently underway.
1516 1517
 * Returns 1 if writeback was started, 0 if not.
 */
1518 1519 1520
int try_to_writeback_inodes_sb_nr(struct super_block *sb,
				  unsigned long nr,
				  enum wb_reason reason)
1521
{
1522
	if (writeback_in_progress(&sb->s_bdi->wb))
1523
		return 1;
1524 1525

	if (!down_read_trylock(&sb->s_umount))
1526
		return 0;
1527 1528 1529 1530

	writeback_inodes_sb_nr(sb, nr, reason);
	up_read(&sb->s_umount);
	return 1;
1531
}
1532
EXPORT_SYMBOL(try_to_writeback_inodes_sb_nr);
1533

1534
/**
1535
 * try_to_writeback_inodes_sb - try to start writeback if none underway
1536
 * @sb: the superblock
1537
 * @reason: reason why some writeback work was initiated
1538
 *
1539
 * Implement by try_to_writeback_inodes_sb_nr()
1540 1541
 * Returns 1 if writeback was started, 0 if not.
 */
1542
int try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
1543
{
1544
	return try_to_writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
1545
}
1546
EXPORT_SYMBOL(try_to_writeback_inodes_sb);
1547

1548 1549
/**
 * sync_inodes_sb	-	sync sb inode pages
1550
 * @sb: the superblock
1551 1552
 *
 * This function writes and waits on any dirty inode belonging to this
1553
 * super_block.
1554
 */
1555
void sync_inodes_sb(struct super_block *sb)
1556
{
1557 1558
	DECLARE_COMPLETION_ONSTACK(done);
	struct wb_writeback_work work = {
1559 1560 1561 1562
		.sb		= sb,
		.sync_mode	= WB_SYNC_ALL,
		.nr_pages	= LONG_MAX,
		.range_cyclic	= 0,
1563
		.done		= &done,
1564
		.reason		= WB_REASON_SYNC,
1565
		.for_sync	= 1,
1566
	};
1567
	struct backing_dev_info *bdi = sb->s_bdi;
1568

1569
	/* Nothing to do? */
1570
	if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
1571
		return;
1572 1573
	WARN_ON(!rwsem_is_locked(&sb->s_umount));

1574
	wb_queue_work(&bdi->wb, &work);
1575 1576
	wait_for_completion(&done);

1577
	wait_sb_inodes(sb);
L
Linus Torvalds 已提交
1578
}
1579
EXPORT_SYMBOL(sync_inodes_sb);
L
Linus Torvalds 已提交
1580 1581

/**
1582 1583 1584 1585 1586 1587
 * write_inode_now	-	write an inode to disk
 * @inode: inode to write to disk
 * @sync: whether the write should be synchronous or not
 *
 * This function commits an inode to disk immediately if it is dirty. This is
 * primarily needed by knfsd.
L
Linus Torvalds 已提交
1588
 *
1589
 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
L
Linus Torvalds 已提交
1590 1591 1592
 */
int write_inode_now(struct inode *inode, int sync)
{
1593
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
L
Linus Torvalds 已提交
1594 1595
	struct writeback_control wbc = {
		.nr_to_write = LONG_MAX,
1596
		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
1597 1598
		.range_start = 0,
		.range_end = LLONG_MAX,
L
Linus Torvalds 已提交
1599 1600 1601
	};

	if (!mapping_cap_writeback_dirty(inode->i_mapping))
1602
		wbc.nr_to_write = 0;
L
Linus Torvalds 已提交
1603 1604

	might_sleep();
1605
	return writeback_single_inode(inode, wb, &wbc);
L
Linus Torvalds 已提交
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
}
EXPORT_SYMBOL(write_inode_now);

/**
 * sync_inode - write an inode and its pages to disk.
 * @inode: the inode to sync
 * @wbc: controls the writeback mode
 *
 * sync_inode() will write an inode and its pages to disk.  It will also
 * correctly update the inode on its superblock's dirty inode lists and will
 * update inode->i_state.
 *
 * The caller must have a ref on the inode.
 */
int sync_inode(struct inode *inode, struct writeback_control *wbc)
{
1622
	return writeback_single_inode(inode, &inode_to_bdi(inode)->wb, wbc);
L
Linus Torvalds 已提交
1623 1624
}
EXPORT_SYMBOL(sync_inode);
C
Christoph Hellwig 已提交
1625 1626

/**
A
Andrew Morton 已提交
1627
 * sync_inode_metadata - write an inode to disk
C
Christoph Hellwig 已提交
1628 1629 1630
 * @inode: the inode to sync
 * @wait: wait for I/O to complete.
 *
A
Andrew Morton 已提交
1631
 * Write an inode to disk and adjust its dirty state after completion.
C
Christoph Hellwig 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
 *
 * Note: only writes the actual inode, no associated data or other metadata.
 */
int sync_inode_metadata(struct inode *inode, int wait)
{
	struct writeback_control wbc = {
		.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
		.nr_to_write = 0, /* metadata-only */
	};

	return sync_inode(inode, &wbc);
}
EXPORT_SYMBOL(sync_inode_metadata);