fs-writeback.c 30.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 * fs/fs-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * Contains all the functions related to writing back and waiting
 * upon dirty inodes against superblocks, and writing back dirty
 * pages against inodes.  ie: data writeback.  Writeout of the
 * inode itself is not handled here.
 *
11
 * 10Apr2002	Andrew Morton
L
Linus Torvalds 已提交
12 13 14 15 16
 *		Split out of fs/inode.c
 *		Additions for address_space-based writeback
 */

#include <linux/kernel.h>
J
Jens Axboe 已提交
17
#include <linux/module.h>
L
Linus Torvalds 已提交
18 19 20 21
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
22 23
#include <linux/kthread.h>
#include <linux/freezer.h>
L
Linus Torvalds 已提交
24 25 26 27
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/buffer_head.h>
28
#include "internal.h"
L
Linus Torvalds 已提交
29

30
#define inode_to_bdi(inode)	((inode)->i_mapping->backing_dev_info)
31

32 33 34 35 36
/*
 * We don't actually have pdflush, but this one is exported though /proc...
 */
int nr_pdflush_threads;

37 38 39 40 41 42 43 44 45 46 47
/*
 * Passed into wb_writeback(), essentially a subset of writeback_control
 */
struct wb_writeback_args {
	long nr_pages;
	struct super_block *sb;
	enum writeback_sync_modes sync_mode;
	int for_kupdate;
	int range_cyclic;
};

48 49
/*
 * Work items for the bdi_writeback threads
50
 */
51 52 53 54 55 56 57
struct bdi_work {
	struct list_head list;
	struct rcu_head rcu_head;

	unsigned long seen;
	atomic_t pending;

58
	struct wb_writeback_args args;
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

	unsigned long state;
};

enum {
	WS_USED_B = 0,
	WS_ONSTACK_B,
};

#define WS_USED (1 << WS_USED_B)
#define WS_ONSTACK (1 << WS_ONSTACK_B)

static inline bool bdi_work_on_stack(struct bdi_work *work)
{
	return test_bit(WS_ONSTACK_B, &work->state);
}

static inline void bdi_work_init(struct bdi_work *work,
				 struct writeback_control *wbc)
{
	INIT_RCU_HEAD(&work->rcu_head);
80 81 82 83 84
	work->args.sb = wbc->sb;
	work->args.nr_pages = wbc->nr_to_write;
	work->args.sync_mode = wbc->sync_mode;
	work->args.range_cyclic = wbc->range_cyclic;
	work->args.for_kupdate = 0;
85 86 87
	work->state = WS_USED;
}

88 89 90 91
/**
 * writeback_in_progress - determine whether there is writeback in progress
 * @bdi: the device's backing_dev_info structure.
 *
92 93
 * Determine whether there is writeback waiting to be handled against a
 * backing device.
94 95 96
 */
int writeback_in_progress(struct backing_dev_info *bdi)
{
97
	return !list_empty(&bdi->work_list);
98 99
}

100
static void bdi_work_clear(struct bdi_work *work)
101
{
102 103 104
	clear_bit(WS_USED_B, &work->state);
	smp_mb__after_clear_bit();
	wake_up_bit(&work->state, WS_USED_B);
105 106
}

107
static void bdi_work_free(struct rcu_head *head)
108
{
109
	struct bdi_work *work = container_of(head, struct bdi_work, rcu_head);
110

111 112 113 114
	if (!bdi_work_on_stack(work))
		kfree(work);
	else
		bdi_work_clear(work);
115 116
}

117
static void wb_work_complete(struct bdi_work *work)
L
Linus Torvalds 已提交
118
{
119
	const enum writeback_sync_modes sync_mode = work->args.sync_mode;
L
Linus Torvalds 已提交
120 121

	/*
122 123 124 125
	 * For allocated work, we can clear the done/seen bit right here.
	 * For on-stack work, we need to postpone both the clear and free
	 * to after the RCU grace period, since the stack could be invalidated
	 * as soon as bdi_work_clear() has done the wakeup.
L
Linus Torvalds 已提交
126
	 */
127 128 129 130 131
	if (!bdi_work_on_stack(work))
		bdi_work_clear(work);
	if (sync_mode == WB_SYNC_NONE || bdi_work_on_stack(work))
		call_rcu(&work->rcu_head, bdi_work_free);
}
L
Linus Torvalds 已提交
132

133 134
static void wb_clear_pending(struct bdi_writeback *wb, struct bdi_work *work)
{
L
Linus Torvalds 已提交
135
	/*
136 137
	 * The caller has retrieved the work arguments from this work,
	 * drop our reference. If this is the last ref, delete and free it
L
Linus Torvalds 已提交
138
	 */
139 140
	if (atomic_dec_and_test(&work->pending)) {
		struct backing_dev_info *bdi = wb->bdi;
L
Linus Torvalds 已提交
141

142 143 144
		spin_lock(&bdi->wb_lock);
		list_del_rcu(&work->list);
		spin_unlock(&bdi->wb_lock);
L
Linus Torvalds 已提交
145

146 147 148
		wb_work_complete(work);
	}
}
L
Linus Torvalds 已提交
149

150 151 152 153 154 155 156
static void bdi_queue_work(struct backing_dev_info *bdi, struct bdi_work *work)
{
	if (work) {
		work->seen = bdi->wb_mask;
		BUG_ON(!work->seen);
		atomic_set(&work->pending, bdi->wb_cnt);
		BUG_ON(!bdi->wb_cnt);
L
Linus Torvalds 已提交
157 158

		/*
159
		 * Make sure stores are seen before it appears on the list
L
Linus Torvalds 已提交
160
		 */
161
		smp_mb();
L
Linus Torvalds 已提交
162

163 164 165 166 167 168 169 170 171 172 173 174 175
		spin_lock(&bdi->wb_lock);
		list_add_tail_rcu(&work->list, &bdi->work_list);
		spin_unlock(&bdi->wb_lock);
	}

	/*
	 * If the default thread isn't there, make sure we add it. When
	 * it gets created and wakes up, we'll run this work.
	 */
	if (unlikely(list_empty_careful(&bdi->wb_list)))
		wake_up_process(default_backing_dev_info.wb.task);
	else {
		struct bdi_writeback *wb = &bdi->wb;
L
Linus Torvalds 已提交
176 177

		/*
178 179 180
		 * If we failed allocating the bdi work item, wake up the wb
		 * thread always. As a safety precaution, it'll flush out
		 * everything
L
Linus Torvalds 已提交
181
		 */
182 183 184 185 186
		if (!wb_has_dirty_io(wb)) {
			if (work)
				wb_clear_pending(wb, work);
		} else if (wb->task)
			wake_up_process(wb->task);
L
Linus Torvalds 已提交
187 188 189
	}
}

190 191 192 193 194 195 196 197 198
/*
 * Used for on-stack allocated work items. The caller needs to wait until
 * the wb threads have acked the work before it's safe to continue.
 */
static void bdi_wait_on_work_clear(struct bdi_work *work)
{
	wait_on_bit(&work->state, WS_USED_B, bdi_sched_wait,
		    TASK_UNINTERRUPTIBLE);
}
L
Linus Torvalds 已提交
199

200 201
static void bdi_alloc_queue_work(struct backing_dev_info *bdi,
				 struct writeback_control *wbc)
L
Linus Torvalds 已提交
202
{
203 204 205 206 207 208
	struct bdi_work *work;

	work = kmalloc(sizeof(*work), GFP_ATOMIC);
	if (work)
		bdi_work_init(work, wbc);

209
	bdi_queue_work(bdi, work);
210 211 212 213
}

void bdi_start_writeback(struct writeback_control *wbc)
{
214 215 216 217 218
	/*
	 * WB_SYNC_NONE is opportunistic writeback. If this allocation fails,
	 * bdi_queue_work() will wake up the thread and flush old data. This
	 * should ensure some amount of progress in freeing memory.
	 */
219 220 221
	if (wbc->sync_mode != WB_SYNC_ALL)
		bdi_alloc_queue_work(wbc->bdi, wbc);
	else {
222
		struct bdi_work work;
223

224 225
		bdi_work_init(&work, wbc);
		work.state |= WS_ONSTACK;
226

227 228
		bdi_queue_work(wbc->bdi, &work);
		bdi_wait_on_work_clear(&work);
229
	}
L
Linus Torvalds 已提交
230 231
}

232 233 234 235 236
/*
 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
 * furthest end of its superblock's dirty-inode list.
 *
 * Before stamping the inode's ->dirtied_when, we check to see whether it is
237
 * already the most-recently-dirtied inode on the b_dirty list.  If that is
238 239 240 241 242
 * the case then the inode must have been redirtied while it was being written
 * out and we don't reset its dirtied_when.
 */
static void redirty_tail(struct inode *inode)
{
243
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
244

245
	if (!list_empty(&wb->b_dirty)) {
246
		struct inode *tail;
247

248
		tail = list_entry(wb->b_dirty.next, struct inode, i_list);
249
		if (time_before(inode->dirtied_when, tail->dirtied_when))
250 251
			inode->dirtied_when = jiffies;
	}
252
	list_move(&inode->i_list, &wb->b_dirty);
253 254
}

255
/*
256
 * requeue inode for re-scanning after bdi->b_io list is exhausted.
257
 */
258
static void requeue_io(struct inode *inode)
259
{
260 261 262
	struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;

	list_move(&inode->i_list, &wb->b_more_io);
263 264
}

J
Joern Engel 已提交
265 266 267 268 269 270 271 272 273
static void inode_sync_complete(struct inode *inode)
{
	/*
	 * Prevent speculative execution through spin_unlock(&inode_lock);
	 */
	smp_mb();
	wake_up_bit(&inode->i_state, __I_SYNC);
}

274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
	bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
	/*
	 * For inodes being constantly redirtied, dirtied_when can get stuck.
	 * It _appears_ to be in the future, but is actually in distant past.
	 * This test is necessary to prevent such wrapped-around relative times
	 * from permanently stopping the whole pdflush writeback.
	 */
	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
	return ret;
}

289 290 291 292 293 294 295 296 297 298 299
/*
 * Move expired dirty inodes from @delaying_queue to @dispatch_queue.
 */
static void move_expired_inodes(struct list_head *delaying_queue,
			       struct list_head *dispatch_queue,
				unsigned long *older_than_this)
{
	while (!list_empty(delaying_queue)) {
		struct inode *inode = list_entry(delaying_queue->prev,
						struct inode, i_list);
		if (older_than_this &&
300
		    inode_dirtied_after(inode, *older_than_this))
301 302 303 304 305 306 307 308
			break;
		list_move(&inode->i_list, dispatch_queue);
	}
}

/*
 * Queue all expired dirty inodes for io, eldest first.
 */
309
static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this)
310
{
311 312
	list_splice_init(&wb->b_more_io, wb->b_io.prev);
	move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this);
313 314
}

315
static int write_inode(struct inode *inode, int sync)
316
{
317 318 319
	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
		return inode->i_sb->s_op->write_inode(inode, sync);
	return 0;
320 321
}

L
Linus Torvalds 已提交
322
/*
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
 * Wait for writeback on an inode to complete.
 */
static void inode_wait_for_writeback(struct inode *inode)
{
	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
	wait_queue_head_t *wqh;

	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
	do {
		spin_unlock(&inode_lock);
		__wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
		spin_lock(&inode_lock);
	} while (inode->i_state & I_SYNC);
}

/*
 * Write out an inode's dirty pages.  Called under inode_lock.  Either the
 * caller has ref on the inode (either via __iget or via syscall against an fd)
 * or the inode has I_WILL_FREE set (via generic_forget_inode)
 *
L
Linus Torvalds 已提交
343 344 345 346 347 348 349 350 351
 * If `wait' is set, wait on the writeout.
 *
 * The whole writeout design is quite complex and fragile.  We want to avoid
 * starvation of particular inodes when others are being redirtied, prevent
 * livelocks, etc.
 *
 * Called under inode_lock.
 */
static int
352
writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
L
Linus Torvalds 已提交
353 354 355
{
	struct address_space *mapping = inode->i_mapping;
	int wait = wbc->sync_mode == WB_SYNC_ALL;
356
	unsigned dirty;
L
Linus Torvalds 已提交
357 358
	int ret;

359 360 361 362 363 364 365 366
	if (!atomic_read(&inode->i_count))
		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
	else
		WARN_ON(inode->i_state & I_WILL_FREE);

	if (inode->i_state & I_SYNC) {
		/*
		 * If this inode is locked for writeback and we are not doing
367
		 * writeback-for-data-integrity, move it to b_more_io so that
368 369 370
		 * writeback can proceed with the other inodes on s_io.
		 *
		 * We'll have another go at writing back this inode when we
371
		 * completed a full scan of b_io.
372 373 374 375 376 377 378 379 380 381 382 383
		 */
		if (!wait) {
			requeue_io(inode);
			return 0;
		}

		/*
		 * It's a data-integrity sync.  We must wait.
		 */
		inode_wait_for_writeback(inode);
	}

J
Joern Engel 已提交
384
	BUG_ON(inode->i_state & I_SYNC);
L
Linus Torvalds 已提交
385

J
Joern Engel 已提交
386
	/* Set I_SYNC, reset I_DIRTY */
L
Linus Torvalds 已提交
387
	dirty = inode->i_state & I_DIRTY;
J
Joern Engel 已提交
388
	inode->i_state |= I_SYNC;
L
Linus Torvalds 已提交
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
	inode->i_state &= ~I_DIRTY;

	spin_unlock(&inode_lock);

	ret = do_writepages(mapping, wbc);

	/* Don't write the inode if only I_DIRTY_PAGES was set */
	if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
		int err = write_inode(inode, wait);
		if (ret == 0)
			ret = err;
	}

	if (wait) {
		int err = filemap_fdatawait(mapping);
		if (ret == 0)
			ret = err;
	}

	spin_lock(&inode_lock);
J
Joern Engel 已提交
409
	inode->i_state &= ~I_SYNC;
410
	if (!(inode->i_state & (I_FREEING | I_CLEAR))) {
L
Linus Torvalds 已提交
411 412 413 414 415
		if (!(inode->i_state & I_DIRTY) &&
		    mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
			/*
			 * We didn't write back all the pages.  nfs_writepages()
			 * sometimes bales out without doing anything. Redirty
416
			 * the inode; Move it from b_io onto b_more_io/b_dirty.
417 418 419
			 */
			/*
			 * akpm: if the caller was the kupdate function we put
420
			 * this inode at the head of b_dirty so it gets first
421 422 423 424 425
			 * consideration.  Otherwise, move it to the tail, for
			 * the reasons described there.  I'm not really sure
			 * how much sense this makes.  Presumably I had a good
			 * reasons for doing it this way, and I'd rather not
			 * muck with it at present.
L
Linus Torvalds 已提交
426 427 428
			 */
			if (wbc->for_kupdate) {
				/*
429
				 * For the kupdate function we move the inode
430
				 * to b_more_io so it will get more writeout as
431
				 * soon as the queue becomes uncongested.
L
Linus Torvalds 已提交
432 433
				 */
				inode->i_state |= I_DIRTY_PAGES;
434 435 436 437 438 439 440 441 442 443 444
				if (wbc->nr_to_write <= 0) {
					/*
					 * slice used up: queue for next turn
					 */
					requeue_io(inode);
				} else {
					/*
					 * somehow blocked: retry later
					 */
					redirty_tail(inode);
				}
L
Linus Torvalds 已提交
445 446 447 448 449 450 451 452 453
			} else {
				/*
				 * Otherwise fully redirty the inode so that
				 * other inodes on this superblock will get some
				 * writeout.  Otherwise heavy writing to one
				 * file would indefinitely suspend writeout of
				 * all the other files.
				 */
				inode->i_state |= I_DIRTY_PAGES;
454
				redirty_tail(inode);
L
Linus Torvalds 已提交
455 456 457 458 459 460
			}
		} else if (inode->i_state & I_DIRTY) {
			/*
			 * Someone redirtied the inode while were writing back
			 * the pages.
			 */
461
			redirty_tail(inode);
L
Linus Torvalds 已提交
462 463 464 465 466 467 468 469 470 471 472 473
		} else if (atomic_read(&inode->i_count)) {
			/*
			 * The inode is clean, inuse
			 */
			list_move(&inode->i_list, &inode_in_use);
		} else {
			/*
			 * The inode is clean, unused
			 */
			list_move(&inode->i_list, &inode_unused);
		}
	}
J
Joern Engel 已提交
474
	inode_sync_complete(inode);
L
Linus Torvalds 已提交
475 476 477
	return ret;
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
/*
 * For WB_SYNC_NONE writeback, the caller does not have the sb pinned
 * before calling writeback. So make sure that we do pin it, so it doesn't
 * go away while we are writing inodes from it.
 *
 * Returns 0 if the super was successfully pinned (or pinning wasn't needed),
 * 1 if we failed.
 */
static int pin_sb_for_writeback(struct writeback_control *wbc,
				   struct inode *inode)
{
	struct super_block *sb = inode->i_sb;

	/*
	 * Caller must already hold the ref for this
	 */
	if (wbc->sync_mode == WB_SYNC_ALL) {
		WARN_ON(!rwsem_is_locked(&sb->s_umount));
		return 0;
	}

	spin_lock(&sb_lock);
	sb->s_count++;
	if (down_read_trylock(&sb->s_umount)) {
		if (sb->s_root) {
			spin_unlock(&sb_lock);
			return 0;
		}
		/*
		 * umounted, drop rwsem again and fall through to failure
		 */
		up_read(&sb->s_umount);
	}

	sb->s_count--;
	spin_unlock(&sb_lock);
	return 1;
}

static void unpin_sb_for_writeback(struct writeback_control *wbc,
				   struct inode *inode)
{
	struct super_block *sb = inode->i_sb;

	if (wbc->sync_mode == WB_SYNC_ALL)
		return;

	up_read(&sb->s_umount);
	put_super(sb);
}

static void writeback_inodes_wb(struct bdi_writeback *wb,
				struct writeback_control *wbc)
L
Linus Torvalds 已提交
531
{
532
	struct super_block *sb = wbc->sb;
533
	const int is_blkdev_sb = sb_is_blkdev_sb(sb);
L
Linus Torvalds 已提交
534 535
	const unsigned long start = jiffies;	/* livelock avoidance */

536
	spin_lock(&inode_lock);
L
Linus Torvalds 已提交
537

538 539
	if (!wbc->for_kupdate || list_empty(&wb->b_io))
		queue_io(wb, wbc->older_than_this);
540

541 542
	while (!list_empty(&wb->b_io)) {
		struct inode *inode = list_entry(wb->b_io.prev,
L
Linus Torvalds 已提交
543 544 545
						struct inode, i_list);
		long pages_skipped;

546 547 548 549 550 551 552 553
		/*
		 * super block given and doesn't match, skip this inode
		 */
		if (sb && sb != inode->i_sb) {
			redirty_tail(inode);
			continue;
		}

554
		if (!bdi_cap_writeback_dirty(wb->bdi)) {
555
			redirty_tail(inode);
556
			if (is_blkdev_sb) {
L
Linus Torvalds 已提交
557 558 559 560 561 562 563 564 565 566 567 568 569 570
				/*
				 * Dirty memory-backed blockdev: the ramdisk
				 * driver does this.  Skip just this inode
				 */
				continue;
			}
			/*
			 * Dirty memory-backed inode against a filesystem other
			 * than the kernel-internal bdev filesystem.  Skip the
			 * entire superblock.
			 */
			break;
		}

571
		if (inode->i_state & (I_NEW | I_WILL_FREE)) {
572 573 574 575
			requeue_io(inode);
			continue;
		}

576
		if (wbc->nonblocking && bdi_write_congested(wb->bdi)) {
L
Linus Torvalds 已提交
577
			wbc->encountered_congestion = 1;
578
			if (!is_blkdev_sb)
L
Linus Torvalds 已提交
579
				break;		/* Skip a congested fs */
580
			requeue_io(inode);
L
Linus Torvalds 已提交
581 582 583
			continue;		/* Skip a congested blockdev */
		}

584 585 586 587 588
		/*
		 * Was this inode dirtied after sync_sb_inodes was called?
		 * This keeps sync from extra jobs and livelock.
		 */
		if (inode_dirtied_after(inode, start))
L
Linus Torvalds 已提交
589 590
			break;

591 592 593 594
		if (pin_sb_for_writeback(wbc, inode)) {
			requeue_io(inode);
			continue;
		}
L
Linus Torvalds 已提交
595

596
		BUG_ON(inode->i_state & (I_FREEING | I_CLEAR));
L
Linus Torvalds 已提交
597 598
		__iget(inode);
		pages_skipped = wbc->pages_skipped;
599
		writeback_single_inode(inode, wbc);
600
		unpin_sb_for_writeback(wbc, inode);
L
Linus Torvalds 已提交
601 602 603 604 605
		if (wbc->pages_skipped != pages_skipped) {
			/*
			 * writeback is not making progress due to locked
			 * buffers.  Skip this inode for now.
			 */
606
			redirty_tail(inode);
L
Linus Torvalds 已提交
607 608 609
		}
		spin_unlock(&inode_lock);
		iput(inode);
610
		cond_resched();
L
Linus Torvalds 已提交
611
		spin_lock(&inode_lock);
612 613
		if (wbc->nr_to_write <= 0) {
			wbc->more_io = 1;
L
Linus Torvalds 已提交
614
			break;
615
		}
616
		if (!list_empty(&wb->b_more_io))
617
			wbc->more_io = 1;
L
Linus Torvalds 已提交
618
	}
N
Nick Piggin 已提交
619

620 621 622 623
	spin_unlock(&inode_lock);
	/* Leave any unwritten inodes on b_io */
}

624 625 626 627 628 629 630
void writeback_inodes_wbc(struct writeback_control *wbc)
{
	struct backing_dev_info *bdi = wbc->bdi;

	writeback_inodes_wb(&bdi->wb, wbc);
}

631
/*
632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
 * The maximum number of pages to writeout in a single bdi flush/kupdate
 * operation.  We do this so we don't hold I_SYNC against an inode for
 * enormous amounts of time, which would block a userspace task which has
 * been forced to throttle against that inode.  Also, the code reevaluates
 * the dirty each time it has written this many pages.
 */
#define MAX_WRITEBACK_PAGES     1024

static inline bool over_bground_thresh(void)
{
	unsigned long background_thresh, dirty_thresh;

	get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);

	return (global_page_state(NR_FILE_DIRTY) +
		global_page_state(NR_UNSTABLE_NFS) >= background_thresh);
}

/*
 * Explicit flushing or periodic writeback of "old" data.
652
 *
653 654 655 656
 * Define "old": the first time one of an inode's pages is dirtied, we mark the
 * dirtying-time in the inode's address_space.  So this periodic writeback code
 * just walks the superblock inode list, writing back any inodes which are
 * older than a specific point in time.
657
 *
658 659 660
 * Try to run once per dirty_writeback_interval.  But if a writeback event
 * takes longer than a dirty_writeback_interval interval, then leave a
 * one-second gap.
661
 *
662 663
 * older_than_this takes precedence over nr_to_write.  So we'll only write back
 * all dirty pages if they are all attached to "old" mappings.
664
 */
665 666
static long wb_writeback(struct bdi_writeback *wb,
			 struct wb_writeback_args *args)
667
{
668 669
	struct writeback_control wbc = {
		.bdi			= wb->bdi,
670 671
		.sb			= args->sb,
		.sync_mode		= args->sync_mode,
672
		.older_than_this	= NULL,
673 674
		.for_kupdate		= args->for_kupdate,
		.range_cyclic		= args->range_cyclic,
675 676 677
	};
	unsigned long oldest_jif;
	long wrote = 0;
678

679 680 681 682 683
	if (wbc.for_kupdate) {
		wbc.older_than_this = &oldest_jif;
		oldest_jif = jiffies -
				msecs_to_jiffies(dirty_expire_interval * 10);
	}
684 685 686 687
	if (!wbc.range_cyclic) {
		wbc.range_start = 0;
		wbc.range_end = LLONG_MAX;
	}
N
Nick Piggin 已提交
688

689 690 691 692 693
	for (;;) {
		/*
		 * Don't flush anything for non-integrity writeback where
		 * no nr_pages was given
		 */
694 695
		if (!args->for_kupdate && args->nr_pages <= 0 &&
		     args->sync_mode == WB_SYNC_NONE)
696
			break;
697

N
Nick Piggin 已提交
698
		/*
699 700 701
		 * If no specific pages were given and this is just a
		 * periodic background writeout and we are below the
		 * background dirty threshold, don't do anything
N
Nick Piggin 已提交
702
		 */
703 704
		if (args->for_kupdate && args->nr_pages <= 0 &&
		    !over_bground_thresh())
705
			break;
N
Nick Piggin 已提交
706

707 708 709 710 711
		wbc.more_io = 0;
		wbc.encountered_congestion = 0;
		wbc.nr_to_write = MAX_WRITEBACK_PAGES;
		wbc.pages_skipped = 0;
		writeback_inodes_wb(wb, &wbc);
712
		args->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
713 714 715 716 717 718 719
		wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write;

		/*
		 * If we ran out of stuff to write, bail unless more_io got set
		 */
		if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
			if (wbc.more_io && !wbc.for_kupdate)
N
Nick Piggin 已提交
720
				continue;
721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
			break;
		}
	}

	return wrote;
}

/*
 * Return the next bdi_work struct that hasn't been processed by this
 * wb thread yet
 */
static struct bdi_work *get_next_work_item(struct backing_dev_info *bdi,
					   struct bdi_writeback *wb)
{
	struct bdi_work *work, *ret = NULL;

	rcu_read_lock();

	list_for_each_entry_rcu(work, &bdi->work_list, list) {
		if (!test_and_clear_bit(wb->nr, &work->seen))
			continue;

		ret = work;
		break;
	}

	rcu_read_unlock();
	return ret;
}

static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
	unsigned long expired;
	long nr_pages;

	expired = wb->last_old_flush +
			msecs_to_jiffies(dirty_writeback_interval * 10);
	if (time_before(jiffies, expired))
		return 0;

	wb->last_old_flush = jiffies;
	nr_pages = global_page_state(NR_FILE_DIRTY) +
			global_page_state(NR_UNSTABLE_NFS) +
			(inodes_stat.nr_inodes - inodes_stat.nr_unused);

766 767 768 769 770 771 772 773 774 775
	if (nr_pages) {
		struct wb_writeback_args args = {
			.nr_pages	= nr_pages,
			.sync_mode	= WB_SYNC_NONE,
			.for_kupdate	= 1,
			.range_cyclic	= 1,
		};

		return wb_writeback(wb, &args);
	}
776 777 778 779 780 781 782 783 784 785 786

	return 0;
}

/*
 * Retrieve work items and do the writeback they describe
 */
long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
{
	struct backing_dev_info *bdi = wb->bdi;
	struct bdi_work *work;
787
	long wrote = 0;
788 789

	while ((work = get_next_work_item(bdi, wb)) != NULL) {
790
		struct wb_writeback_args args = work->args;
791 792 793 794 795

		/*
		 * Override sync mode, in case we must wait for completion
		 */
		if (force_wait)
796
			work->args.sync_mode = args.sync_mode = WB_SYNC_ALL;
797 798 799 800 801

		/*
		 * If this isn't a data integrity operation, just notify
		 * that we have seen this work and we are now starting it.
		 */
802
		if (args.sync_mode == WB_SYNC_NONE)
803 804
			wb_clear_pending(wb, work);

805
		wrote += wb_writeback(wb, &args);
806 807 808 809 810

		/*
		 * This is a data integrity writeback, so only do the
		 * notification when we have completed the work.
		 */
811
		if (args.sync_mode == WB_SYNC_ALL)
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
			wb_clear_pending(wb, work);
	}

	/*
	 * Check for periodic writeback, kupdated() style
	 */
	wrote += wb_check_old_data_flush(wb);

	return wrote;
}

/*
 * Handle writeback of dirty data for the device backed by this bdi. Also
 * wakes up periodically and does kupdated style flushing.
 */
int bdi_writeback_task(struct bdi_writeback *wb)
{
	unsigned long last_active = jiffies;
	unsigned long wait_jiffies = -1UL;
	long pages_written;

	while (!kthread_should_stop()) {
		pages_written = wb_do_writeback(wb, 0);

		if (pages_written)
			last_active = jiffies;
		else if (wait_jiffies != -1UL) {
			unsigned long max_idle;

N
Nick Piggin 已提交
841
			/*
842 843 844
			 * Longest period of inactivity that we tolerate. If we
			 * see dirty data again later, the task will get
			 * recreated automatically.
N
Nick Piggin 已提交
845
			 */
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
			max_idle = max(5UL * 60 * HZ, wait_jiffies);
			if (time_after(jiffies, max_idle + last_active))
				break;
		}

		wait_jiffies = msecs_to_jiffies(dirty_writeback_interval * 10);
		set_current_state(TASK_INTERRUPTIBLE);
		schedule_timeout(wait_jiffies);
		try_to_freeze();
	}

	return 0;
}

/*
861 862 863
 * Schedule writeback for all backing devices. Can only be used for
 * WB_SYNC_NONE writeback, WB_SYNC_ALL should use bdi_start_writeback()
 * and pass in the superblock.
864 865 866 867 868
 */
static void bdi_writeback_all(struct writeback_control *wbc)
{
	struct backing_dev_info *bdi;

869 870
	WARN_ON(wbc->sync_mode == WB_SYNC_ALL);

871 872 873 874 875
	spin_lock(&bdi_lock);

	list_for_each_entry(bdi, &bdi_list, bdi_list) {
		if (!bdi_has_dirty_io(bdi))
			continue;
N
Nick Piggin 已提交
876

877
		bdi_alloc_queue_work(bdi, wbc);
878 879 880
	}

	spin_unlock(&bdi_lock);
L
Linus Torvalds 已提交
881 882 883
}

/*
884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
 * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
 * the whole world.
 */
void wakeup_flusher_threads(long nr_pages)
{
	struct writeback_control wbc = {
		.sync_mode	= WB_SYNC_NONE,
		.older_than_this = NULL,
		.range_cyclic	= 1,
	};

	if (nr_pages == 0)
		nr_pages = global_page_state(NR_FILE_DIRTY) +
				global_page_state(NR_UNSTABLE_NFS);
	wbc.nr_to_write = nr_pages;
	bdi_writeback_all(&wbc);
}

static noinline void block_dump___mark_inode_dirty(struct inode *inode)
{
	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
		struct dentry *dentry;
		const char *name = "?";

		dentry = d_find_alias(inode);
		if (dentry) {
			spin_lock(&dentry->d_lock);
			name = (const char *) dentry->d_name.name;
		}
		printk(KERN_DEBUG
		       "%s(%d): dirtied inode %lu (%s) on %s\n",
		       current->comm, task_pid_nr(current), inode->i_ino,
		       name, inode->i_sb->s_id);
		if (dentry) {
			spin_unlock(&dentry->d_lock);
			dput(dentry);
		}
	}
}

/**
 *	__mark_inode_dirty -	internal function
 *	@inode: inode to mark
 *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
 *	Mark an inode as dirty. Callers should use mark_inode_dirty or
 *  	mark_inode_dirty_sync.
L
Linus Torvalds 已提交
930
 *
931 932 933 934 935 936 937 938 939
 * Put the inode on the super block's dirty list.
 *
 * CAREFUL! We mark it dirty unconditionally, but move it onto the
 * dirty list only if it is hashed or if it refers to a blockdev.
 * If it was not hashed, it will never be added to the dirty list
 * even if it is later hashed, as it will have been marked dirty already.
 *
 * In short, make sure you hash any inodes _before_ you start marking
 * them dirty.
L
Linus Torvalds 已提交
940
 *
941 942
 * This function *must* be atomic for the I_DIRTY_PAGES case -
 * set_page_dirty() is called under spinlock in several places.
L
Linus Torvalds 已提交
943
 *
944 945 946 947 948 949
 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
 * the kernel-internal blockdev inode represents the dirtying time of the
 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
 * page->mapping->host, so the page-dirtying time is recorded in the internal
 * blockdev inode.
L
Linus Torvalds 已提交
950
 */
951
void __mark_inode_dirty(struct inode *inode, int flags)
L
Linus Torvalds 已提交
952
{
953
	struct super_block *sb = inode->i_sb;
L
Linus Torvalds 已提交
954

955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
	/*
	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
	 * dirty the inode itself
	 */
	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
		if (sb->s_op->dirty_inode)
			sb->s_op->dirty_inode(inode);
	}

	/*
	 * make sure that changes are seen by all cpus before we test i_state
	 * -- mikulas
	 */
	smp_mb();

	/* avoid the locking if we can */
	if ((inode->i_state & flags) == flags)
		return;

	if (unlikely(block_dump))
		block_dump___mark_inode_dirty(inode);

	spin_lock(&inode_lock);
	if ((inode->i_state & flags) != flags) {
		const int was_dirty = inode->i_state & I_DIRTY;

		inode->i_state |= flags;

		/*
		 * If the inode is being synced, just update its dirty state.
		 * The unlocker will place the inode on the appropriate
		 * superblock list, based upon its state.
		 */
		if (inode->i_state & I_SYNC)
			goto out;

		/*
		 * Only add valid (hashed) inodes to the superblock's
		 * dirty list.  Add blockdev inodes as well.
		 */
		if (!S_ISBLK(inode->i_mode)) {
			if (hlist_unhashed(&inode->i_hash))
				goto out;
		}
		if (inode->i_state & (I_FREEING|I_CLEAR))
			goto out;

		/*
		 * If the inode was already on b_dirty/b_io/b_more_io, don't
		 * reposition it (that would break b_dirty time-ordering).
		 */
		if (!was_dirty) {
			struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
1008 1009 1010 1011 1012 1013 1014 1015
			struct backing_dev_info *bdi = wb->bdi;

			if (bdi_cap_writeback_dirty(bdi) &&
			    !test_bit(BDI_registered, &bdi->state)) {
				WARN_ON(1);
				printk(KERN_ERR "bdi-%s not registered\n",
								bdi->name);
			}
1016 1017 1018

			inode->dirtied_when = jiffies;
			list_move(&inode->i_list, &wb->b_dirty);
L
Linus Torvalds 已提交
1019 1020
		}
	}
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
out:
	spin_unlock(&inode_lock);
}
EXPORT_SYMBOL(__mark_inode_dirty);

/*
 * Write out a superblock's list of dirty inodes.  A wait will be performed
 * upon no inodes, all inodes or the final one, depending upon sync_mode.
 *
 * If older_than_this is non-NULL, then only write out inodes which
 * had their first dirtying at a time earlier than *older_than_this.
 *
 * If we're a pdlfush thread, then implement pdflush collision avoidance
 * against the entire list.
 *
 * If `bdi' is non-zero then we're being asked to writeback a specific queue.
 * This function assumes that the blockdev superblock's inodes are backed by
 * a variety of queues, so all inodes are searched.  For other superblocks,
 * assume that all inodes are backed by the same queue.
 *
 * The inodes to be written are parked on bdi->b_io.  They are moved back onto
 * bdi->b_dirty as they are selected for writing.  This way, none can be missed
 * on the writer throttling path, and we get decent balancing between many
 * throttled threads: we don't want them all piling up on inode_sync_wait.
 */
static void wait_sb_inodes(struct writeback_control *wbc)
{
	struct inode *inode, *old_inode = NULL;

	/*
	 * We need to be protected against the filesystem going from
	 * r/o to r/w or vice versa.
	 */
	WARN_ON(!rwsem_is_locked(&wbc->sb->s_umount));

	spin_lock(&inode_lock);

	/*
	 * Data integrity sync. Must wait for all pages under writeback,
	 * because there may have been pages dirtied before our sync
	 * call, but which had writeout started before we write it out.
	 * In which case, the inode may not be on the dirty list, but
	 * we still have to wait for that writeout.
	 */
	list_for_each_entry(inode, &wbc->sb->s_inodes, i_sb_list) {
		struct address_space *mapping;

		if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE|I_NEW))
			continue;
		mapping = inode->i_mapping;
		if (mapping->nrpages == 0)
			continue;
		__iget(inode);
		spin_unlock(&inode_lock);
		/*
		 * We hold a reference to 'inode' so it couldn't have
		 * been removed from s_inodes list while we dropped the
		 * inode_lock.  We cannot iput the inode now as we can
		 * be holding the last reference and we cannot iput it
		 * under inode_lock. So we keep the reference and iput
		 * it later.
		 */
		iput(old_inode);
		old_inode = inode;

		filemap_fdatawait(mapping);

		cond_resched();

		spin_lock(&inode_lock);
	}
	spin_unlock(&inode_lock);
	iput(old_inode);
L
Linus Torvalds 已提交
1094 1095
}

1096 1097 1098
/**
 * writeback_inodes_sb	-	writeback dirty inodes from given super_block
 * @sb: the superblock
L
Linus Torvalds 已提交
1099
 *
1100 1101 1102 1103
 * Start writeback on some inodes on this super_block. No guarantees are made
 * on how many (if any) will be written, and this function does not wait
 * for IO completion of submitted IO. The number of pages submitted is
 * returned.
L
Linus Torvalds 已提交
1104
 */
1105
long writeback_inodes_sb(struct super_block *sb)
L
Linus Torvalds 已提交
1106 1107
{
	struct writeback_control wbc = {
1108
		.sb		= sb,
1109
		.sync_mode	= WB_SYNC_NONE,
1110 1111
		.range_start	= 0,
		.range_end	= LLONG_MAX,
L
Linus Torvalds 已提交
1112
	};
1113 1114 1115
	unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY);
	unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS);
	long nr_to_write;
L
Linus Torvalds 已提交
1116

1117
	nr_to_write = nr_dirty + nr_unstable +
N
Nick Piggin 已提交
1118 1119
			(inodes_stat.nr_inodes - inodes_stat.nr_unused);

1120
	wbc.nr_to_write = nr_to_write;
1121
	bdi_writeback_all(&wbc);
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
	return nr_to_write - wbc.nr_to_write;
}
EXPORT_SYMBOL(writeback_inodes_sb);

/**
 * sync_inodes_sb	-	sync sb inode pages
 * @sb: the superblock
 *
 * This function writes and waits on any dirty inode belonging to this
 * super_block. The number of pages synced is returned.
 */
long sync_inodes_sb(struct super_block *sb)
{
	struct writeback_control wbc = {
1136
		.sb		= sb,
1137
		.bdi		= sb->s_bdi,
1138 1139 1140 1141 1142 1143 1144
		.sync_mode	= WB_SYNC_ALL,
		.range_start	= 0,
		.range_end	= LLONG_MAX,
	};
	long nr_to_write = LONG_MAX; /* doesn't actually matter */

	wbc.nr_to_write = nr_to_write;
1145
	bdi_start_writeback(&wbc);
1146
	wait_sb_inodes(&wbc);
1147
	return nr_to_write - wbc.nr_to_write;
L
Linus Torvalds 已提交
1148
}
1149
EXPORT_SYMBOL(sync_inodes_sb);
L
Linus Torvalds 已提交
1150 1151

/**
1152 1153 1154 1155 1156 1157
 * write_inode_now	-	write an inode to disk
 * @inode: inode to write to disk
 * @sync: whether the write should be synchronous or not
 *
 * This function commits an inode to disk immediately if it is dirty. This is
 * primarily needed by knfsd.
L
Linus Torvalds 已提交
1158
 *
1159
 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
L
Linus Torvalds 已提交
1160 1161 1162 1163 1164 1165
 */
int write_inode_now(struct inode *inode, int sync)
{
	int ret;
	struct writeback_control wbc = {
		.nr_to_write = LONG_MAX,
1166
		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
1167 1168
		.range_start = 0,
		.range_end = LLONG_MAX,
L
Linus Torvalds 已提交
1169 1170 1171
	};

	if (!mapping_cap_writeback_dirty(inode->i_mapping))
1172
		wbc.nr_to_write = 0;
L
Linus Torvalds 已提交
1173 1174 1175

	might_sleep();
	spin_lock(&inode_lock);
1176
	ret = writeback_single_inode(inode, &wbc);
L
Linus Torvalds 已提交
1177 1178
	spin_unlock(&inode_lock);
	if (sync)
J
Joern Engel 已提交
1179
		inode_sync_wait(inode);
L
Linus Torvalds 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
	return ret;
}
EXPORT_SYMBOL(write_inode_now);

/**
 * sync_inode - write an inode and its pages to disk.
 * @inode: the inode to sync
 * @wbc: controls the writeback mode
 *
 * sync_inode() will write an inode and its pages to disk.  It will also
 * correctly update the inode on its superblock's dirty inode lists and will
 * update inode->i_state.
 *
 * The caller must have a ref on the inode.
 */
int sync_inode(struct inode *inode, struct writeback_control *wbc)
{
	int ret;

	spin_lock(&inode_lock);
1200
	ret = writeback_single_inode(inode, wbc);
L
Linus Torvalds 已提交
1201 1202 1203 1204
	spin_unlock(&inode_lock);
	return ret;
}
EXPORT_SYMBOL(sync_inode);