vmscan.c 51.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
/*
 *  linux/mm/vmscan.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  Swap reorganised 29.12.95, Stephen Tweedie.
 *  kswapd added: 7.1.96  sct
 *  Removed kswapd_ctl limits, and swap out as many pages as needed
 *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
 *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
 *  Multiqueue VM started 5.8.00, Rik van Riel.
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/file.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>	/* for try_to_release_page(),
					buffer_heads_over_limit */
#include <linux/mm_inline.h>
#include <linux/pagevec.h>
#include <linux/backing-dev.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/notifier.h>
#include <linux/rwsem.h>

#include <asm/tlbflush.h>
#include <asm/div64.h>

#include <linux/swapops.h>

/* possible outcome of pageout() */
typedef enum {
	/* failed to write page out, page is locked */
	PAGE_KEEP,
	/* move page to the active list, page is locked */
	PAGE_ACTIVATE,
	/* page has been sent to the disk successfully, page is unlocked */
	PAGE_SUCCESS,
	/* page is clean and locked */
	PAGE_CLEAN,
} pageout_t;

struct scan_control {
	/* Incremented by the number of inactive pages that were scanned */
	unsigned long nr_scanned;

	unsigned long nr_mapped;	/* From page_state */

	/* This context's GFP mask */
A
Al Viro 已提交
61
	gfp_t gfp_mask;
L
Linus Torvalds 已提交
62 63 64

	int may_writepage;

65 66 67
	/* Can pages be swapped as part of reclaim? */
	int may_swap;

L
Linus Torvalds 已提交
68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
	 * In this context, it doesn't matter that we scan the
	 * whole list at once. */
	int swap_cluster_max;
};

/*
 * The list of shrinker callbacks used by to apply pressure to
 * ageable caches.
 */
struct shrinker {
	shrinker_t		shrinker;
	struct list_head	list;
	int			seeks;	/* seeks to recreate an obj */
	long			nr;	/* objs pending delete */
};

#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))

#ifdef ARCH_HAS_PREFETCH
#define prefetch_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetch(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

#ifdef ARCH_HAS_PREFETCHW
#define prefetchw_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetchw(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

/*
 * From 0 .. 100.  Higher means more swappy.
 */
int vm_swappiness = 60;
static long total_memory;

static LIST_HEAD(shrinker_list);
static DECLARE_RWSEM(shrinker_rwsem);

/*
 * Add a shrinker callback to be called from the vm
 */
struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker)
{
        struct shrinker *shrinker;

        shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL);
        if (shrinker) {
	        shrinker->shrinker = theshrinker;
	        shrinker->seeks = seeks;
	        shrinker->nr = 0;
	        down_write(&shrinker_rwsem);
	        list_add_tail(&shrinker->list, &shrinker_list);
	        up_write(&shrinker_rwsem);
	}
	return shrinker;
}
EXPORT_SYMBOL(set_shrinker);

/*
 * Remove one
 */
void remove_shrinker(struct shrinker *shrinker)
{
	down_write(&shrinker_rwsem);
	list_del(&shrinker->list);
	up_write(&shrinker_rwsem);
	kfree(shrinker);
}
EXPORT_SYMBOL(remove_shrinker);

#define SHRINK_BATCH 128
/*
 * Call the shrink functions to age shrinkable caches
 *
 * Here we assume it costs one seek to replace a lru page and that it also
 * takes a seek to recreate a cache object.  With this in mind we age equal
 * percentages of the lru and ageable caches.  This should balance the seeks
 * generated by these structures.
 *
 * If the vm encounted mapped pages on the LRU it increase the pressure on
 * slab to avoid swapping.
 *
 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
 *
 * `lru_pages' represents the number of on-LRU pages in all the zones which
 * are eligible for the caller's allocation attempt.  It is used for balancing
 * slab reclaim versus page reclaim.
174 175
 *
 * Returns the number of slab objects which we shrunk.
L
Linus Torvalds 已提交
176
 */
177 178
unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
			unsigned long lru_pages)
L
Linus Torvalds 已提交
179 180
{
	struct shrinker *shrinker;
181
	unsigned long ret = 0;
L
Linus Torvalds 已提交
182 183 184 185 186

	if (scanned == 0)
		scanned = SWAP_CLUSTER_MAX;

	if (!down_read_trylock(&shrinker_rwsem))
187
		return 1;	/* Assume we'll be able to shrink next time */
L
Linus Torvalds 已提交
188 189 190 191

	list_for_each_entry(shrinker, &shrinker_list, list) {
		unsigned long long delta;
		unsigned long total_scan;
192
		unsigned long max_pass = (*shrinker->shrinker)(0, gfp_mask);
L
Linus Torvalds 已提交
193 194

		delta = (4 * scanned) / shrinker->seeks;
195
		delta *= max_pass;
L
Linus Torvalds 已提交
196 197
		do_div(delta, lru_pages + 1);
		shrinker->nr += delta;
198 199 200 201 202 203 204 205 206 207 208 209 210
		if (shrinker->nr < 0) {
			printk(KERN_ERR "%s: nr=%ld\n",
					__FUNCTION__, shrinker->nr);
			shrinker->nr = max_pass;
		}

		/*
		 * Avoid risking looping forever due to too large nr value:
		 * never try to free more than twice the estimate number of
		 * freeable entries.
		 */
		if (shrinker->nr > max_pass * 2)
			shrinker->nr = max_pass * 2;
L
Linus Torvalds 已提交
211 212 213 214 215 216 217

		total_scan = shrinker->nr;
		shrinker->nr = 0;

		while (total_scan >= SHRINK_BATCH) {
			long this_scan = SHRINK_BATCH;
			int shrink_ret;
218
			int nr_before;
L
Linus Torvalds 已提交
219

220
			nr_before = (*shrinker->shrinker)(0, gfp_mask);
L
Linus Torvalds 已提交
221 222 223
			shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask);
			if (shrink_ret == -1)
				break;
224 225
			if (shrink_ret < nr_before)
				ret += nr_before - shrink_ret;
L
Linus Torvalds 已提交
226 227 228 229 230 231 232 233 234
			mod_page_state(slabs_scanned, this_scan);
			total_scan -= this_scan;

			cond_resched();
		}

		shrinker->nr += total_scan;
	}
	up_read(&shrinker_rwsem);
235
	return ret;
L
Linus Torvalds 已提交
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
}

/* Called without lock on whether page is mapped, so answer is unstable */
static inline int page_mapping_inuse(struct page *page)
{
	struct address_space *mapping;

	/* Page is in somebody's page tables. */
	if (page_mapped(page))
		return 1;

	/* Be more reluctant to reclaim swapcache than pagecache */
	if (PageSwapCache(page))
		return 1;

	mapping = page_mapping(page);
	if (!mapping)
		return 0;

	/* File is mmap'd by somebody? */
	return mapping_mapped(mapping);
}

static inline int is_page_cache_freeable(struct page *page)
{
	return page_count(page) - !!PagePrivate(page) == 2;
}

static int may_write_to_queue(struct backing_dev_info *bdi)
{
266
	if (current->flags & PF_SWAPWRITE)
L
Linus Torvalds 已提交
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
		return 1;
	if (!bdi_write_congested(bdi))
		return 1;
	if (bdi == current->backing_dev_info)
		return 1;
	return 0;
}

/*
 * We detected a synchronous write error writing a page out.  Probably
 * -ENOSPC.  We need to propagate that into the address_space for a subsequent
 * fsync(), msync() or close().
 *
 * The tricky part is that after writepage we cannot touch the mapping: nothing
 * prevents it from being freed up.  But we have a ref on the page and once
 * that page is locked, the mapping is pinned.
 *
 * We're allowed to run sleeping lock_page() here because we know the caller has
 * __GFP_FS.
 */
static void handle_write_error(struct address_space *mapping,
				struct page *page, int error)
{
	lock_page(page);
	if (page_mapping(page) == mapping) {
		if (error == -ENOSPC)
			set_bit(AS_ENOSPC, &mapping->flags);
		else
			set_bit(AS_EIO, &mapping->flags);
	}
	unlock_page(page);
}

/*
A
Andrew Morton 已提交
301 302
 * pageout is called by shrink_page_list() for each dirty page.
 * Calls ->writepage().
L
Linus Torvalds 已提交
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
 */
static pageout_t pageout(struct page *page, struct address_space *mapping)
{
	/*
	 * If the page is dirty, only perform writeback if that write
	 * will be non-blocking.  To prevent this allocation from being
	 * stalled by pagecache activity.  But note that there may be
	 * stalls if we need to run get_block().  We could test
	 * PagePrivate for that.
	 *
	 * If this process is currently in generic_file_write() against
	 * this page's queue, we can perform writeback even if that
	 * will block.
	 *
	 * If the page is swapcache, write it back even if that would
	 * block, for some throttling. This happens by accident, because
	 * swap_backing_dev_info is bust: it doesn't reflect the
	 * congestion state of the swapdevs.  Easy to fix, if needed.
	 * See swapfile.c:page_queue_congested().
	 */
	if (!is_page_cache_freeable(page))
		return PAGE_KEEP;
	if (!mapping) {
		/*
		 * Some data journaling orphaned pages can have
		 * page->mapping == NULL while being dirty with clean buffers.
		 */
330
		if (PagePrivate(page)) {
L
Linus Torvalds 已提交
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
			if (try_to_free_buffers(page)) {
				ClearPageDirty(page);
				printk("%s: orphaned page\n", __FUNCTION__);
				return PAGE_CLEAN;
			}
		}
		return PAGE_KEEP;
	}
	if (mapping->a_ops->writepage == NULL)
		return PAGE_ACTIVATE;
	if (!may_write_to_queue(mapping->backing_dev_info))
		return PAGE_KEEP;

	if (clear_page_dirty_for_io(page)) {
		int res;
		struct writeback_control wbc = {
			.sync_mode = WB_SYNC_NONE,
			.nr_to_write = SWAP_CLUSTER_MAX,
			.nonblocking = 1,
			.for_reclaim = 1,
		};

		SetPageReclaim(page);
		res = mapping->a_ops->writepage(page, &wbc);
		if (res < 0)
			handle_write_error(mapping, page, res);
357
		if (res == AOP_WRITEPAGE_ACTIVATE) {
L
Linus Torvalds 已提交
358 359 360 361 362 363 364 365 366 367 368 369 370 371
			ClearPageReclaim(page);
			return PAGE_ACTIVATE;
		}
		if (!PageWriteback(page)) {
			/* synchronous write or broken a_ops? */
			ClearPageReclaim(page);
		}

		return PAGE_SUCCESS;
	}

	return PAGE_CLEAN;
}

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
static int remove_mapping(struct address_space *mapping, struct page *page)
{
	if (!mapping)
		return 0;		/* truncate got there first */

	write_lock_irq(&mapping->tree_lock);

	/*
	 * The non-racy check for busy page.  It is critical to check
	 * PageDirty _after_ making sure that the page is freeable and
	 * not in use by anybody. 	(pagecache + us == 2)
	 */
	if (unlikely(page_count(page) != 2))
		goto cannot_free;
	smp_rmb();
	if (unlikely(PageDirty(page)))
		goto cannot_free;

	if (PageSwapCache(page)) {
		swp_entry_t swap = { .val = page_private(page) };
		__delete_from_swap_cache(page);
		write_unlock_irq(&mapping->tree_lock);
		swap_free(swap);
		__put_page(page);	/* The pagecache ref */
		return 1;
	}

	__remove_from_page_cache(page);
	write_unlock_irq(&mapping->tree_lock);
	__put_page(page);
	return 1;

cannot_free:
	write_unlock_irq(&mapping->tree_lock);
	return 0;
}

L
Linus Torvalds 已提交
409
/*
A
Andrew Morton 已提交
410
 * shrink_page_list() returns the number of reclaimed pages
L
Linus Torvalds 已提交
411
 */
A
Andrew Morton 已提交
412 413
static unsigned long shrink_page_list(struct list_head *page_list,
					struct scan_control *sc)
L
Linus Torvalds 已提交
414 415 416 417
{
	LIST_HEAD(ret_pages);
	struct pagevec freed_pvec;
	int pgactivate = 0;
418
	unsigned long nr_reclaimed = 0;
L
Linus Torvalds 已提交
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439

	cond_resched();

	pagevec_init(&freed_pvec, 1);
	while (!list_empty(page_list)) {
		struct address_space *mapping;
		struct page *page;
		int may_enter_fs;
		int referenced;

		cond_resched();

		page = lru_to_page(page_list);
		list_del(&page->lru);

		if (TestSetPageLocked(page))
			goto keep;

		BUG_ON(PageActive(page));

		sc->nr_scanned++;
440 441 442 443

		if (!sc->may_swap && page_mapped(page))
			goto keep_locked;

L
Linus Torvalds 已提交
444 445 446 447 448 449 450
		/* Double the slab pressure for mapped and swapcache pages */
		if (page_mapped(page) || PageSwapCache(page))
			sc->nr_scanned++;

		if (PageWriteback(page))
			goto keep_locked;

451
		referenced = page_referenced(page, 1);
L
Linus Torvalds 已提交
452 453 454 455 456 457 458 459 460
		/* In active use or really unfreeable?  Activate it. */
		if (referenced && page_mapping_inuse(page))
			goto activate_locked;

#ifdef CONFIG_SWAP
		/*
		 * Anonymous process memory has backing store?
		 * Try to allocate it some swap space here.
		 */
461
		if (PageAnon(page) && !PageSwapCache(page)) {
462 463
			if (!sc->may_swap)
				goto keep_locked;
464
			if (!add_to_swap(page, GFP_ATOMIC))
L
Linus Torvalds 已提交
465 466 467 468 469 470 471 472 473 474 475 476 477
				goto activate_locked;
		}
#endif /* CONFIG_SWAP */

		mapping = page_mapping(page);
		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));

		/*
		 * The page is mapped into the page tables of one or more
		 * processes. Try to unmap it here.
		 */
		if (page_mapped(page) && mapping) {
478 479 480 481 482 483
			/*
			 * No unmapping if we do not swap
			 */
			if (!sc->may_swap)
				goto keep_locked;

484
			switch (try_to_unmap(page, 0)) {
L
Linus Torvalds 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498
			case SWAP_FAIL:
				goto activate_locked;
			case SWAP_AGAIN:
				goto keep_locked;
			case SWAP_SUCCESS:
				; /* try to free the page below */
			}
		}

		if (PageDirty(page)) {
			if (referenced)
				goto keep_locked;
			if (!may_enter_fs)
				goto keep_locked;
499
			if (!sc->may_writepage)
L
Linus Torvalds 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
				goto keep_locked;

			/* Page is dirty, try to write it out here */
			switch(pageout(page, mapping)) {
			case PAGE_KEEP:
				goto keep_locked;
			case PAGE_ACTIVATE:
				goto activate_locked;
			case PAGE_SUCCESS:
				if (PageWriteback(page) || PageDirty(page))
					goto keep;
				/*
				 * A synchronous write - probably a ramdisk.  Go
				 * ahead and try to reclaim the page.
				 */
				if (TestSetPageLocked(page))
					goto keep;
				if (PageDirty(page) || PageWriteback(page))
					goto keep_locked;
				mapping = page_mapping(page);
			case PAGE_CLEAN:
				; /* try to free the page below */
			}
		}

		/*
		 * If the page has buffers, try to free the buffer mappings
		 * associated with this page. If we succeed we try to free
		 * the page as well.
		 *
		 * We do this even if the page is PageDirty().
		 * try_to_release_page() does not perform I/O, but it is
		 * possible for a page to have PageDirty set, but it is actually
		 * clean (all its buffers are clean).  This happens if the
		 * buffers were written out directly, with submit_bh(). ext3
		 * will do this, as well as the blockdev mapping. 
		 * try_to_release_page() will discover that cleanness and will
		 * drop the buffers and mark the page clean - it can be freed.
		 *
		 * Rarely, pages can have buffers and no ->mapping.  These are
		 * the pages which were not successfully invalidated in
		 * truncate_complete_page().  We try to drop those buffers here
		 * and if that worked, and the page is no longer mapped into
		 * process address space (page_count == 1) it can be freed.
		 * Otherwise, leave the page on the LRU so it is swappable.
		 */
		if (PagePrivate(page)) {
			if (!try_to_release_page(page, sc->gfp_mask))
				goto activate_locked;
			if (!mapping && page_count(page) == 1)
				goto free_it;
		}

553 554
		if (!remove_mapping(mapping, page))
			goto keep_locked;
L
Linus Torvalds 已提交
555 556 557

free_it:
		unlock_page(page);
558
		nr_reclaimed++;
L
Linus Torvalds 已提交
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
		if (!pagevec_add(&freed_pvec, page))
			__pagevec_release_nonlru(&freed_pvec);
		continue;

activate_locked:
		SetPageActive(page);
		pgactivate++;
keep_locked:
		unlock_page(page);
keep:
		list_add(&page->lru, &ret_pages);
		BUG_ON(PageLRU(page));
	}
	list_splice(&ret_pages, page_list);
	if (pagevec_count(&freed_pvec))
		__pagevec_release_nonlru(&freed_pvec);
	mod_page_state(pgactivate, pgactivate);
576
	return nr_reclaimed;
L
Linus Torvalds 已提交
577 578
}

579
#ifdef CONFIG_MIGRATION
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
static inline void move_to_lru(struct page *page)
{
	list_del(&page->lru);
	if (PageActive(page)) {
		/*
		 * lru_cache_add_active checks that
		 * the PG_active bit is off.
		 */
		ClearPageActive(page);
		lru_cache_add_active(page);
	} else {
		lru_cache_add(page);
	}
	put_page(page);
}

/*
597
 * Add isolated pages on the list back to the LRU.
598 599 600
 *
 * returns the number of pages put back.
 */
601
unsigned long putback_lru_pages(struct list_head *l)
602 603 604
{
	struct page *page;
	struct page *page2;
605
	unsigned long count = 0;
606 607 608 609 610 611 612 613

	list_for_each_entry_safe(page, page2, l, lru) {
		move_to_lru(page);
		count++;
	}
	return count;
}

614 615 616 617 618 619 620 621 622
/*
 * Non migratable page
 */
int fail_migrate_page(struct page *newpage, struct page *page)
{
	return -EIO;
}
EXPORT_SYMBOL(fail_migrate_page);

623 624 625 626 627 628 629 630 631
/*
 * swapout a single page
 * page is locked upon entry, unlocked on exit
 */
static int swap_page(struct page *page)
{
	struct address_space *mapping = page_mapping(page);

	if (page_mapped(page) && mapping)
632
		if (try_to_unmap(page, 1) != SWAP_SUCCESS)
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
			goto unlock_retry;

	if (PageDirty(page)) {
		/* Page is dirty, try to write it out here */
		switch(pageout(page, mapping)) {
		case PAGE_KEEP:
		case PAGE_ACTIVATE:
			goto unlock_retry;

		case PAGE_SUCCESS:
			goto retry;

		case PAGE_CLEAN:
			; /* try to free the page below */
		}
	}

	if (PagePrivate(page)) {
		if (!try_to_release_page(page, GFP_KERNEL) ||
		    (!mapping && page_count(page) == 1))
			goto unlock_retry;
	}

	if (remove_mapping(mapping, page)) {
		/* Success */
		unlock_page(page);
		return 0;
	}

unlock_retry:
	unlock_page(page);

retry:
666
	return -EAGAIN;
667
}
668
EXPORT_SYMBOL(swap_page);
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683

/*
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
 * Christoph Lameter <clameter@sgi.com>
 */

/*
 * Remove references for a page and establish the new page with the correct
 * basic settings to be able to stop accesses to the page.
 */
684
int migrate_page_remove_references(struct page *newpage,
685 686 687 688 689 690 691 692 693 694 695
				struct page *page, int nr_refs)
{
	struct address_space *mapping = page_mapping(page);
	struct page **radix_pointer;

	/*
	 * Avoid doing any of the following work if the page count
	 * indicates that the page is in use or truncate has removed
	 * the page.
	 */
	if (!mapping || page_mapcount(page) + nr_refs != page_count(page))
696
		return -EAGAIN;
697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716

	/*
	 * Establish swap ptes for anonymous pages or destroy pte
	 * maps for files.
	 *
	 * In order to reestablish file backed mappings the fault handlers
	 * will take the radix tree_lock which may then be used to stop
  	 * processses from accessing this page until the new page is ready.
	 *
	 * A process accessing via a swap pte (an anonymous page) will take a
	 * page_lock on the old page which will block the process until the
	 * migration attempt is complete. At that time the PageSwapCache bit
	 * will be examined. If the page was migrated then the PageSwapCache
	 * bit will be clear and the operation to retrieve the page will be
	 * retried which will find the new page in the radix tree. Then a new
	 * direct mapping may be generated based on the radix tree contents.
	 *
	 * If the page was not migrated then the PageSwapCache bit
	 * is still set and the operation may continue.
	 */
717 718 719
	if (try_to_unmap(page, 1) == SWAP_FAIL)
		/* A vma has VM_LOCKED set -> Permanent failure */
		return -EPERM;
720 721 722 723 724

	/*
	 * Give up if we were unable to remove all mappings.
	 */
	if (page_mapcount(page))
725
		return -EAGAIN;
726 727 728 729 730 731 732 733 734 735

	write_lock_irq(&mapping->tree_lock);

	radix_pointer = (struct page **)radix_tree_lookup_slot(
						&mapping->page_tree,
						page_index(page));

	if (!page_mapping(page) || page_count(page) != nr_refs ||
			*radix_pointer != page) {
		write_unlock_irq(&mapping->tree_lock);
736
		return -EAGAIN;
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
	}

	/*
	 * Now we know that no one else is looking at the page.
	 *
	 * Certain minimal information about a page must be available
	 * in order for other subsystems to properly handle the page if they
	 * find it through the radix tree update before we are finished
	 * copying the page.
	 */
	get_page(newpage);
	newpage->index = page->index;
	newpage->mapping = page->mapping;
	if (PageSwapCache(page)) {
		SetPageSwapCache(newpage);
		set_page_private(newpage, page_private(page));
	}

	*radix_pointer = newpage;
	__put_page(page);
	write_unlock_irq(&mapping->tree_lock);

	return 0;
}
761
EXPORT_SYMBOL(migrate_page_remove_references);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800

/*
 * Copy the page to its new location
 */
void migrate_page_copy(struct page *newpage, struct page *page)
{
	copy_highpage(newpage, page);

	if (PageError(page))
		SetPageError(newpage);
	if (PageReferenced(page))
		SetPageReferenced(newpage);
	if (PageUptodate(page))
		SetPageUptodate(newpage);
	if (PageActive(page))
		SetPageActive(newpage);
	if (PageChecked(page))
		SetPageChecked(newpage);
	if (PageMappedToDisk(page))
		SetPageMappedToDisk(newpage);

	if (PageDirty(page)) {
		clear_page_dirty_for_io(page);
		set_page_dirty(newpage);
 	}

	ClearPageSwapCache(page);
	ClearPageActive(page);
	ClearPagePrivate(page);
	set_page_private(page, 0);
	page->mapping = NULL;

	/*
	 * If any waiters have accumulated on the new page then
	 * wake them up.
	 */
	if (PageWriteback(newpage))
		end_page_writeback(newpage);
}
801
EXPORT_SYMBOL(migrate_page_copy);
802 803 804 805 806 807 808 809 810

/*
 * Common logic to directly migrate a single page suitable for
 * pages that do not use PagePrivate.
 *
 * Pages are locked upon entry and exit.
 */
int migrate_page(struct page *newpage, struct page *page)
{
811 812
	int rc;

813 814
	BUG_ON(PageWriteback(page));	/* Writeback must be complete */

815 816 817 818
	rc = migrate_page_remove_references(newpage, page, 2);

	if (rc)
		return rc;
819 820 821

	migrate_page_copy(newpage, page);

822 823 824 825 826 827 828 829 830
	/*
	 * Remove auxiliary swap entries and replace
	 * them with real ptes.
	 *
	 * Note that a real pte entry will allow processes that are not
	 * waiting on the page lock to use the new page via the page tables
	 * before the new page is unlocked.
	 */
	remove_from_swap(newpage);
831 832
	return 0;
}
833
EXPORT_SYMBOL(migrate_page);
834

835 836 837 838 839 840 841 842 843 844
/*
 * migrate_pages
 *
 * Two lists are passed to this function. The first list
 * contains the pages isolated from the LRU to be migrated.
 * The second list contains new pages that the pages isolated
 * can be moved to. If the second list is NULL then all
 * pages are swapped out.
 *
 * The function returns after 10 attempts or if no pages
845
 * are movable anymore because to has become empty
846 847
 * or no retryable pages exist anymore.
 *
848
 * Return: Number of pages not migrated when "to" ran empty.
849
 */
850
unsigned long migrate_pages(struct list_head *from, struct list_head *to,
851
		  struct list_head *moved, struct list_head *failed)
852
{
853 854
	unsigned long retry;
	unsigned long nr_failed = 0;
855 856 857 858
	int pass = 0;
	struct page *page;
	struct page *page2;
	int swapwrite = current->flags & PF_SWAPWRITE;
859
	int rc;
860 861 862 863 864 865 866

	if (!swapwrite)
		current->flags |= PF_SWAPWRITE;

redo:
	retry = 0;

867
	list_for_each_entry_safe(page, page2, from, lru) {
868 869 870
		struct page *newpage = NULL;
		struct address_space *mapping;

871 872
		cond_resched();

873 874
		rc = 0;
		if (page_count(page) == 1)
875
			/* page was freed from under us. So we are done. */
876 877
			goto next;

878 879 880
		if (to && list_empty(to))
			break;

881 882
		/*
		 * Skip locked pages during the first two passes to give the
883 884 885
		 * functions holding the lock time to release the page. Later we
		 * use lock_page() to have a higher chance of acquiring the
		 * lock.
886
		 */
887
		rc = -EAGAIN;
888 889 890 891
		if (pass > 2)
			lock_page(page);
		else
			if (TestSetPageLocked(page))
892
				goto next;
893 894 895 896 897

		/*
		 * Only wait on writeback if we have already done a pass where
		 * we we may have triggered writeouts for lots of pages.
		 */
898
		if (pass > 0) {
899
			wait_on_page_writeback(page);
900
		} else {
901 902
			if (PageWriteback(page))
				goto unlock_page;
903
		}
904

905 906 907 908 909
		/*
		 * Anonymous pages must have swap cache references otherwise
		 * the information contained in the page maps cannot be
		 * preserved.
		 */
910
		if (PageAnon(page) && !PageSwapCache(page)) {
911
			if (!add_to_swap(page, GFP_KERNEL)) {
912 913
				rc = -ENOMEM;
				goto unlock_page;
914 915 916
			}
		}

917 918 919 920 921 922 923 924
		if (!to) {
			rc = swap_page(page);
			goto next;
		}

		newpage = lru_to_page(to);
		lock_page(newpage);

925
		/*
926
		 * Pages are properly locked and writeback is complete.
927 928
		 * Try to migrate the page.
		 */
929 930 931 932
		mapping = page_mapping(page);
		if (!mapping)
			goto unlock_both;

933
		if (mapping->a_ops->migratepage) {
934 935 936 937 938 939 940
			/*
			 * Most pages have a mapping and most filesystems
			 * should provide a migration function. Anonymous
			 * pages are part of swap space which also has its
			 * own migration function. This is the most common
			 * path for page migration.
			 */
941 942 943 944
			rc = mapping->a_ops->migratepage(newpage, page);
			goto unlock_both;
                }

945
		/*
946 947 948
		 * Default handling if a filesystem does not provide
		 * a migration function. We can only migrate clean
		 * pages so try to write out any dirty pages first.
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
		 */
		if (PageDirty(page)) {
			switch (pageout(page, mapping)) {
			case PAGE_KEEP:
			case PAGE_ACTIVATE:
				goto unlock_both;

			case PAGE_SUCCESS:
				unlock_page(newpage);
				goto next;

			case PAGE_CLEAN:
				; /* try to migrate the page below */
			}
                }
964

965
		/*
966 967
		 * Buffers are managed in a filesystem specific way.
		 * We must have no buffers or drop them.
968 969 970 971 972 973 974 975 976 977 978 979 980 981
		 */
		if (!page_has_buffers(page) ||
		    try_to_release_page(page, GFP_KERNEL)) {
			rc = migrate_page(newpage, page);
			goto unlock_both;
		}

		/*
		 * On early passes with mapped pages simply
		 * retry. There may be a lock held for some
		 * buffers that may go away. Later
		 * swap them out.
		 */
		if (pass > 4) {
982 983 984 985 986
			/*
			 * Persistently unable to drop buffers..... As a
			 * measure of last resort we fall back to
			 * swap_page().
			 */
987 988 989 990 991 992 993 994
			unlock_page(newpage);
			newpage = NULL;
			rc = swap_page(page);
			goto next;
		}

unlock_both:
		unlock_page(newpage);
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006

unlock_page:
		unlock_page(page);

next:
		if (rc == -EAGAIN) {
			retry++;
		} else if (rc) {
			/* Permanent failure */
			list_move(&page->lru, failed);
			nr_failed++;
		} else {
1007 1008 1009 1010
			if (newpage) {
				/* Successful migration. Return page to LRU */
				move_to_lru(newpage);
			}
1011 1012
			list_move(&page->lru, moved);
		}
1013 1014 1015 1016 1017 1018 1019 1020 1021
	}
	if (retry && pass++ < 10)
		goto redo;

	if (!swapwrite)
		current->flags &= ~PF_SWAPWRITE;

	return nr_failed + retry;
}
1022 1023 1024

/*
 * Isolate one page from the LRU lists and put it on the
1025
 * indicated list with elevated refcount.
1026 1027 1028 1029 1030 1031 1032
 *
 * Result:
 *  0 = page not on LRU list
 *  1 = page removed from LRU list and added to the specified list.
 */
int isolate_lru_page(struct page *page)
{
1033
	int ret = 0;
1034

1035 1036 1037
	if (PageLRU(page)) {
		struct zone *zone = page_zone(page);
		spin_lock_irq(&zone->lru_lock);
N
Nick Piggin 已提交
1038
		if (PageLRU(page)) {
1039 1040
			ret = 1;
			get_page(page);
N
Nick Piggin 已提交
1041
			ClearPageLRU(page);
1042 1043 1044 1045 1046 1047
			if (PageActive(page))
				del_page_from_active_list(zone, page);
			else
				del_page_from_inactive_list(zone, page);
		}
		spin_unlock_irq(&zone->lru_lock);
1048
	}
1049 1050

	return ret;
1051
}
1052
#endif
1053

L
Linus Torvalds 已提交
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
/*
 * zone->lru_lock is heavily contended.  Some of the functions that
 * shrink the lists perform better by taking out a batch of pages
 * and working on them outside the LRU lock.
 *
 * For pagecache intensive workloads, this function is the hottest
 * spot in the kernel (apart from copy_*_user functions).
 *
 * Appropriate locks must be held before calling this function.
 *
 * @nr_to_scan:	The number of pages to look through on the list.
 * @src:	The LRU list to pull pages off.
 * @dst:	The temp list to put pages on to.
 * @scanned:	The number of pages that were scanned.
 *
 * returns how many pages were moved onto *@dst.
 */
1071 1072 1073
static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
		struct list_head *src, struct list_head *dst,
		unsigned long *scanned)
L
Linus Torvalds 已提交
1074
{
1075
	unsigned long nr_taken = 0;
L
Linus Torvalds 已提交
1076
	struct page *page;
1077
	unsigned long scan = 0;
L
Linus Torvalds 已提交
1078 1079

	while (scan++ < nr_to_scan && !list_empty(src)) {
1080
		struct list_head *target;
L
Linus Torvalds 已提交
1081 1082 1083
		page = lru_to_page(src);
		prefetchw_prev_lru_page(page, src, flags);

N
Nick Piggin 已提交
1084 1085
		BUG_ON(!PageLRU(page));

1086
		list_del(&page->lru);
1087 1088
		target = src;
		if (likely(get_page_unless_zero(page))) {
1089
			/*
1090 1091 1092
			 * Be careful not to clear PageLRU until after we're
			 * sure the page is not being freed elsewhere -- the
			 * page release code relies on it.
1093
			 */
1094 1095 1096 1097
			ClearPageLRU(page);
			target = dst;
			nr_taken++;
		} /* else it is being freed elsewhere */
1098

1099
		list_add(&page->lru, target);
L
Linus Torvalds 已提交
1100 1101 1102 1103 1104 1105 1106
	}

	*scanned = scan;
	return nr_taken;
}

/*
A
Andrew Morton 已提交
1107 1108
 * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
 * of reclaimed pages
L
Linus Torvalds 已提交
1109
 */
A
Andrew Morton 已提交
1110 1111
static unsigned long shrink_inactive_list(unsigned long max_scan,
				struct zone *zone, struct scan_control *sc)
L
Linus Torvalds 已提交
1112 1113 1114
{
	LIST_HEAD(page_list);
	struct pagevec pvec;
1115
	unsigned long nr_scanned = 0;
1116
	unsigned long nr_reclaimed = 0;
L
Linus Torvalds 已提交
1117 1118 1119 1120 1121

	pagevec_init(&pvec, 1);

	lru_add_drain();
	spin_lock_irq(&zone->lru_lock);
1122
	do {
L
Linus Torvalds 已提交
1123
		struct page *page;
1124 1125 1126
		unsigned long nr_taken;
		unsigned long nr_scan;
		unsigned long nr_freed;
L
Linus Torvalds 已提交
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137

		nr_taken = isolate_lru_pages(sc->swap_cluster_max,
					     &zone->inactive_list,
					     &page_list, &nr_scan);
		zone->nr_inactive -= nr_taken;
		zone->pages_scanned += nr_scan;
		spin_unlock_irq(&zone->lru_lock);

		if (nr_taken == 0)
			goto done;

1138
		nr_scanned += nr_scan;
A
Andrew Morton 已提交
1139
		nr_freed = shrink_page_list(&page_list, sc);
1140
		nr_reclaimed += nr_freed;
N
Nick Piggin 已提交
1141 1142 1143 1144 1145 1146 1147 1148 1149
		local_irq_disable();
		if (current_is_kswapd()) {
			__mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
			__mod_page_state(kswapd_steal, nr_freed);
		} else
			__mod_page_state_zone(zone, pgscan_direct, nr_scan);
		__mod_page_state_zone(zone, pgsteal, nr_freed);

		spin_lock(&zone->lru_lock);
L
Linus Torvalds 已提交
1150 1151 1152 1153 1154
		/*
		 * Put back any unfreeable pages.
		 */
		while (!list_empty(&page_list)) {
			page = lru_to_page(&page_list);
N
Nick Piggin 已提交
1155 1156
			BUG_ON(PageLRU(page));
			SetPageLRU(page);
L
Linus Torvalds 已提交
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
			list_del(&page->lru);
			if (PageActive(page))
				add_page_to_active_list(zone, page);
			else
				add_page_to_inactive_list(zone, page);
			if (!pagevec_add(&pvec, page)) {
				spin_unlock_irq(&zone->lru_lock);
				__pagevec_release(&pvec);
				spin_lock_irq(&zone->lru_lock);
			}
		}
1168
  	} while (nr_scanned < max_scan);
L
Linus Torvalds 已提交
1169 1170 1171
	spin_unlock_irq(&zone->lru_lock);
done:
	pagevec_release(&pvec);
1172
	return nr_reclaimed;
L
Linus Torvalds 已提交
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
}

/*
 * This moves pages from the active list to the inactive list.
 *
 * We move them the other way if the page is referenced by one or more
 * processes, from rmap.
 *
 * If the pages are mostly unmapped, the processing is fast and it is
 * appropriate to hold zone->lru_lock across the whole operation.  But if
 * the pages are mapped, the processing is slow (page_referenced()) so we
 * should drop zone->lru_lock around each page.  It's impossible to balance
 * this, so instead we remove the pages from the LRU while processing them.
 * It is safe to rely on PG_active against the non-LRU pages in here because
 * nobody will play with that bit on a non-LRU page.
 *
 * The downside is that we have to touch page->_count against each page.
 * But we had to alter page->flags anyway.
 */
A
Andrew Morton 已提交
1192 1193
static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
				struct scan_control *sc)
L
Linus Torvalds 已提交
1194
{
1195
	unsigned long pgmoved;
L
Linus Torvalds 已提交
1196
	int pgdeactivate = 0;
1197
	unsigned long pgscanned;
L
Linus Torvalds 已提交
1198 1199 1200 1201 1202 1203
	LIST_HEAD(l_hold);	/* The pages which were snipped off */
	LIST_HEAD(l_inactive);	/* Pages to go onto the inactive_list */
	LIST_HEAD(l_active);	/* Pages to go onto the active_list */
	struct page *page;
	struct pagevec pvec;
	int reclaim_mapped = 0;
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244

	if (unlikely(sc->may_swap)) {
		long mapped_ratio;
		long distress;
		long swap_tendency;

		/*
		 * `distress' is a measure of how much trouble we're having
		 * reclaiming pages.  0 -> no problems.  100 -> great trouble.
		 */
		distress = 100 >> zone->prev_priority;

		/*
		 * The point of this algorithm is to decide when to start
		 * reclaiming mapped memory instead of just pagecache.  Work out
		 * how much memory
		 * is mapped.
		 */
		mapped_ratio = (sc->nr_mapped * 100) / total_memory;

		/*
		 * Now decide how much we really want to unmap some pages.  The
		 * mapped ratio is downgraded - just because there's a lot of
		 * mapped memory doesn't necessarily mean that page reclaim
		 * isn't succeeding.
		 *
		 * The distress ratio is important - we don't want to start
		 * going oom.
		 *
		 * A 100% value of vm_swappiness overrides this algorithm
		 * altogether.
		 */
		swap_tendency = mapped_ratio / 2 + distress + vm_swappiness;

		/*
		 * Now use this metric to decide whether to start moving mapped
		 * memory onto the inactive list.
		 */
		if (swap_tendency >= 100)
			reclaim_mapped = 1;
	}
L
Linus Torvalds 已提交
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260

	lru_add_drain();
	spin_lock_irq(&zone->lru_lock);
	pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
				    &l_hold, &pgscanned);
	zone->pages_scanned += pgscanned;
	zone->nr_active -= pgmoved;
	spin_unlock_irq(&zone->lru_lock);

	while (!list_empty(&l_hold)) {
		cond_resched();
		page = lru_to_page(&l_hold);
		list_del(&page->lru);
		if (page_mapped(page)) {
			if (!reclaim_mapped ||
			    (total_swap_pages == 0 && PageAnon(page)) ||
1261
			    page_referenced(page, 0)) {
L
Linus Torvalds 已提交
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
				list_add(&page->lru, &l_active);
				continue;
			}
		}
		list_add(&page->lru, &l_inactive);
	}

	pagevec_init(&pvec, 1);
	pgmoved = 0;
	spin_lock_irq(&zone->lru_lock);
	while (!list_empty(&l_inactive)) {
		page = lru_to_page(&l_inactive);
		prefetchw_prev_lru_page(page, &l_inactive, flags);
N
Nick Piggin 已提交
1275 1276
		BUG_ON(PageLRU(page));
		SetPageLRU(page);
N
Nick Piggin 已提交
1277 1278 1279
		BUG_ON(!PageActive(page));
		ClearPageActive(page);

L
Linus Torvalds 已提交
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
		list_move(&page->lru, &zone->inactive_list);
		pgmoved++;
		if (!pagevec_add(&pvec, page)) {
			zone->nr_inactive += pgmoved;
			spin_unlock_irq(&zone->lru_lock);
			pgdeactivate += pgmoved;
			pgmoved = 0;
			if (buffer_heads_over_limit)
				pagevec_strip(&pvec);
			__pagevec_release(&pvec);
			spin_lock_irq(&zone->lru_lock);
		}
	}
	zone->nr_inactive += pgmoved;
	pgdeactivate += pgmoved;
	if (buffer_heads_over_limit) {
		spin_unlock_irq(&zone->lru_lock);
		pagevec_strip(&pvec);
		spin_lock_irq(&zone->lru_lock);
	}

	pgmoved = 0;
	while (!list_empty(&l_active)) {
		page = lru_to_page(&l_active);
		prefetchw_prev_lru_page(page, &l_active, flags);
N
Nick Piggin 已提交
1305 1306
		BUG_ON(PageLRU(page));
		SetPageLRU(page);
L
Linus Torvalds 已提交
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
		BUG_ON(!PageActive(page));
		list_move(&page->lru, &zone->active_list);
		pgmoved++;
		if (!pagevec_add(&pvec, page)) {
			zone->nr_active += pgmoved;
			pgmoved = 0;
			spin_unlock_irq(&zone->lru_lock);
			__pagevec_release(&pvec);
			spin_lock_irq(&zone->lru_lock);
		}
	}
	zone->nr_active += pgmoved;
N
Nick Piggin 已提交
1319 1320 1321 1322 1323
	spin_unlock(&zone->lru_lock);

	__mod_page_state_zone(zone, pgrefill, pgscanned);
	__mod_page_state(pgdeactivate, pgdeactivate);
	local_irq_enable();
L
Linus Torvalds 已提交
1324

N
Nick Piggin 已提交
1325
	pagevec_release(&pvec);
L
Linus Torvalds 已提交
1326 1327 1328 1329 1330
}

/*
 * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
 */
1331 1332
static unsigned long shrink_zone(int priority, struct zone *zone,
				struct scan_control *sc)
L
Linus Torvalds 已提交
1333 1334 1335
{
	unsigned long nr_active;
	unsigned long nr_inactive;
1336
	unsigned long nr_to_scan;
1337
	unsigned long nr_reclaimed = 0;
L
Linus Torvalds 已提交
1338

1339 1340
	atomic_inc(&zone->reclaim_in_progress);

L
Linus Torvalds 已提交
1341 1342 1343 1344
	/*
	 * Add one to `nr_to_scan' just to make sure that the kernel will
	 * slowly sift through the active list.
	 */
1345
	zone->nr_scan_active += (zone->nr_active >> priority) + 1;
L
Linus Torvalds 已提交
1346 1347 1348 1349 1350 1351
	nr_active = zone->nr_scan_active;
	if (nr_active >= sc->swap_cluster_max)
		zone->nr_scan_active = 0;
	else
		nr_active = 0;

1352
	zone->nr_scan_inactive += (zone->nr_inactive >> priority) + 1;
L
Linus Torvalds 已提交
1353 1354 1355 1356 1357 1358 1359 1360
	nr_inactive = zone->nr_scan_inactive;
	if (nr_inactive >= sc->swap_cluster_max)
		zone->nr_scan_inactive = 0;
	else
		nr_inactive = 0;

	while (nr_active || nr_inactive) {
		if (nr_active) {
1361
			nr_to_scan = min(nr_active,
L
Linus Torvalds 已提交
1362
					(unsigned long)sc->swap_cluster_max);
1363
			nr_active -= nr_to_scan;
A
Andrew Morton 已提交
1364
			shrink_active_list(nr_to_scan, zone, sc);
L
Linus Torvalds 已提交
1365 1366 1367
		}

		if (nr_inactive) {
1368
			nr_to_scan = min(nr_inactive,
L
Linus Torvalds 已提交
1369
					(unsigned long)sc->swap_cluster_max);
1370
			nr_inactive -= nr_to_scan;
A
Andrew Morton 已提交
1371 1372
			nr_reclaimed += shrink_inactive_list(nr_to_scan, zone,
								sc);
L
Linus Torvalds 已提交
1373 1374 1375 1376
		}
	}

	throttle_vm_writeout();
1377 1378

	atomic_dec(&zone->reclaim_in_progress);
1379
	return nr_reclaimed;
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
}

/*
 * This is the direct reclaim path, for page-allocating processes.  We only
 * try to reclaim pages from zones which will satisfy the caller's allocation
 * request.
 *
 * We reclaim from a zone even if that zone is over pages_high.  Because:
 * a) The caller may be trying to free *extra* pages to satisfy a higher-order
 *    allocation or
 * b) The zones may be over pages_high but they must go *over* pages_high to
 *    satisfy the `incremental min' zone defense algorithm.
 *
 * Returns the number of reclaimed pages.
 *
 * If a zone is deemed to be full of pinned pages then just give it a light
 * scan then give up on it.
 */
A
Andrew Morton 已提交
1398
static unsigned long shrink_zones(int priority, struct zone **zones,
1399
					struct scan_control *sc)
L
Linus Torvalds 已提交
1400
{
1401
	unsigned long nr_reclaimed = 0;
L
Linus Torvalds 已提交
1402 1403 1404 1405 1406
	int i;

	for (i = 0; zones[i] != NULL; i++) {
		struct zone *zone = zones[i];

1407
		if (!populated_zone(zone))
L
Linus Torvalds 已提交
1408 1409
			continue;

1410
		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
L
Linus Torvalds 已提交
1411 1412
			continue;

1413 1414 1415
		zone->temp_priority = priority;
		if (zone->prev_priority > priority)
			zone->prev_priority = priority;
L
Linus Torvalds 已提交
1416

1417
		if (zone->all_unreclaimable && priority != DEF_PRIORITY)
L
Linus Torvalds 已提交
1418 1419
			continue;	/* Let kswapd poll it */

1420
		nr_reclaimed += shrink_zone(priority, zone, sc);
L
Linus Torvalds 已提交
1421
	}
1422
	return nr_reclaimed;
L
Linus Torvalds 已提交
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
}
 
/*
 * This is the main entry point to direct page reclaim.
 *
 * If a full scan of the inactive list fails to free enough memory then we
 * are "out of memory" and something needs to be killed.
 *
 * If the caller is !__GFP_FS then the probability of a failure is reasonably
 * high - the zone may be full of dirty or under-writeback pages, which this
 * caller can't do much about.  We kick pdflush and take explicit naps in the
 * hope that some of these pages can be written.  But if the allocating task
 * holds filesystem locks which prevent writeout this might not work, and the
 * allocation attempt will fail.
 */
1438
unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1439 1440 1441
{
	int priority;
	int ret = 0;
1442
	unsigned long total_scanned = 0;
1443
	unsigned long nr_reclaimed = 0;
L
Linus Torvalds 已提交
1444 1445 1446
	struct reclaim_state *reclaim_state = current->reclaim_state;
	unsigned long lru_pages = 0;
	int i;
1447 1448 1449 1450 1451 1452
	struct scan_control sc = {
		.gfp_mask = gfp_mask,
		.may_writepage = !laptop_mode,
		.swap_cluster_max = SWAP_CLUSTER_MAX,
		.may_swap = 1,
	};
L
Linus Torvalds 已提交
1453 1454 1455 1456 1457 1458

	inc_page_state(allocstall);

	for (i = 0; zones[i] != NULL; i++) {
		struct zone *zone = zones[i];

1459
		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
L
Linus Torvalds 已提交
1460 1461 1462 1463 1464 1465 1466 1467 1468
			continue;

		zone->temp_priority = DEF_PRIORITY;
		lru_pages += zone->nr_active + zone->nr_inactive;
	}

	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
		sc.nr_mapped = read_page_state(nr_mapped);
		sc.nr_scanned = 0;
1469 1470
		if (!priority)
			disable_swap_token();
A
Andrew Morton 已提交
1471
		nr_reclaimed += shrink_zones(priority, zones, &sc);
L
Linus Torvalds 已提交
1472 1473
		shrink_slab(sc.nr_scanned, gfp_mask, lru_pages);
		if (reclaim_state) {
1474
			nr_reclaimed += reclaim_state->reclaimed_slab;
L
Linus Torvalds 已提交
1475 1476 1477
			reclaim_state->reclaimed_slab = 0;
		}
		total_scanned += sc.nr_scanned;
1478
		if (nr_reclaimed >= sc.swap_cluster_max) {
L
Linus Torvalds 已提交
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
			ret = 1;
			goto out;
		}

		/*
		 * Try to write back as many pages as we just scanned.  This
		 * tends to cause slow streaming writers to write data to the
		 * disk smoothly, at the dirtying rate, which is nice.   But
		 * that's undesirable in laptop mode, where we *want* lumpy
		 * writeout.  So in laptop mode, write out the whole world.
		 */
1490 1491
		if (total_scanned > sc.swap_cluster_max +
					sc.swap_cluster_max / 2) {
1492
			wakeup_pdflush(laptop_mode ? 0 : total_scanned);
L
Linus Torvalds 已提交
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
			sc.may_writepage = 1;
		}

		/* Take a nap, wait for some writeback to complete */
		if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
			blk_congestion_wait(WRITE, HZ/10);
	}
out:
	for (i = 0; zones[i] != 0; i++) {
		struct zone *zone = zones[i];

1504
		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
L
Linus Torvalds 已提交
1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
			continue;

		zone->prev_priority = zone->temp_priority;
	}
	return ret;
}

/*
 * For kswapd, balance_pgdat() will work across all this node's zones until
 * they are all at pages_high.
 *
 * If `nr_pages' is non-zero then it is the number of pages which are to be
 * reclaimed, regardless of the zone occupancies.  This is a software suspend
 * special.
 *
 * Returns the number of pages which were actually freed.
 *
 * There is special handling here for zones which are full of pinned pages.
 * This can happen if the pages are all mlocked, or if they are all used by
 * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
 * What we do is to detect the case where all pages in the zone have been
 * scanned twice and there has been zero successful reclaim.  Mark the zone as
 * dead and from now on, only perform a short scan.  Basically we're polling
 * the zone for when the problem goes away.
 *
 * kswapd scans the zones in the highmem->normal->dma direction.  It skips
 * zones which have free_pages > pages_high, but once a zone is found to have
 * free_pages <= pages_high, we scan that zone and the lower zones regardless
 * of the number of free pages in the lower zones.  This interoperates with
 * the page allocator fallback scheme to ensure that aging of pages is balanced
 * across the zones.
 */
1537 1538
static unsigned long balance_pgdat(pg_data_t *pgdat, unsigned long nr_pages,
				int order)
L
Linus Torvalds 已提交
1539
{
1540
	unsigned long to_free = nr_pages;
L
Linus Torvalds 已提交
1541 1542 1543
	int all_zones_ok;
	int priority;
	int i;
1544
	unsigned long total_scanned;
1545
	unsigned long nr_reclaimed;
L
Linus Torvalds 已提交
1546
	struct reclaim_state *reclaim_state = current->reclaim_state;
1547 1548 1549 1550 1551
	struct scan_control sc = {
		.gfp_mask = GFP_KERNEL,
		.may_swap = 1,
		.swap_cluster_max = nr_pages ? nr_pages : SWAP_CLUSTER_MAX,
	};
L
Linus Torvalds 已提交
1552 1553 1554

loop_again:
	total_scanned = 0;
1555
	nr_reclaimed = 0;
1556
	sc.may_writepage = !laptop_mode,
L
Linus Torvalds 已提交
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	sc.nr_mapped = read_page_state(nr_mapped);

	inc_page_state(pageoutrun);

	for (i = 0; i < pgdat->nr_zones; i++) {
		struct zone *zone = pgdat->node_zones + i;

		zone->temp_priority = DEF_PRIORITY;
	}

	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
		int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
		unsigned long lru_pages = 0;

1571 1572 1573 1574
		/* The swap token gets in the way of swapout... */
		if (!priority)
			disable_swap_token();

L
Linus Torvalds 已提交
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
		all_zones_ok = 1;

		if (nr_pages == 0) {
			/*
			 * Scan in the highmem->dma direction for the highest
			 * zone which needs scanning
			 */
			for (i = pgdat->nr_zones - 1; i >= 0; i--) {
				struct zone *zone = pgdat->node_zones + i;

1585
				if (!populated_zone(zone))
L
Linus Torvalds 已提交
1586 1587 1588 1589 1590 1591 1592
					continue;

				if (zone->all_unreclaimable &&
						priority != DEF_PRIORITY)
					continue;

				if (!zone_watermark_ok(zone, order,
R
Rohit Seth 已提交
1593
						zone->pages_high, 0, 0)) {
L
Linus Torvalds 已提交
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
					end_zone = i;
					goto scan;
				}
			}
			goto out;
		} else {
			end_zone = pgdat->nr_zones - 1;
		}
scan:
		for (i = 0; i <= end_zone; i++) {
			struct zone *zone = pgdat->node_zones + i;

			lru_pages += zone->nr_active + zone->nr_inactive;
		}

		/*
		 * Now scan the zone in the dma->highmem direction, stopping
		 * at the last zone which needs scanning.
		 *
		 * We do this because the page allocator works in the opposite
		 * direction.  This prevents the page allocator from allocating
		 * pages behind kswapd's direction of progress, which would
		 * cause too much scanning of the lower zones.
		 */
		for (i = 0; i <= end_zone; i++) {
			struct zone *zone = pgdat->node_zones + i;
1620
			int nr_slab;
L
Linus Torvalds 已提交
1621

1622
			if (!populated_zone(zone))
L
Linus Torvalds 已提交
1623 1624 1625 1626 1627 1628 1629
				continue;

			if (zone->all_unreclaimable && priority != DEF_PRIORITY)
				continue;

			if (nr_pages == 0) {	/* Not software suspend */
				if (!zone_watermark_ok(zone, order,
R
Rohit Seth 已提交
1630
						zone->pages_high, end_zone, 0))
L
Linus Torvalds 已提交
1631 1632 1633 1634 1635 1636
					all_zones_ok = 0;
			}
			zone->temp_priority = priority;
			if (zone->prev_priority > priority)
				zone->prev_priority = priority;
			sc.nr_scanned = 0;
1637
			nr_reclaimed += shrink_zone(priority, zone, &sc);
L
Linus Torvalds 已提交
1638
			reclaim_state->reclaimed_slab = 0;
1639 1640
			nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
						lru_pages);
1641
			nr_reclaimed += reclaim_state->reclaimed_slab;
L
Linus Torvalds 已提交
1642 1643 1644
			total_scanned += sc.nr_scanned;
			if (zone->all_unreclaimable)
				continue;
1645 1646
			if (nr_slab == 0 && zone->pages_scanned >=
				    (zone->nr_active + zone->nr_inactive) * 4)
L
Linus Torvalds 已提交
1647 1648 1649 1650 1651 1652 1653
				zone->all_unreclaimable = 1;
			/*
			 * If we've done a decent amount of scanning and
			 * the reclaim ratio is low, start doing writepage
			 * even in laptop mode
			 */
			if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
1654
			    total_scanned > nr_reclaimed + nr_reclaimed / 2)
L
Linus Torvalds 已提交
1655 1656
				sc.may_writepage = 1;
		}
1657
		if (nr_pages && to_free > nr_reclaimed)
L
Linus Torvalds 已提交
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673
			continue;	/* swsusp: need to do more work */
		if (all_zones_ok)
			break;		/* kswapd: all done */
		/*
		 * OK, kswapd is getting into trouble.  Take a nap, then take
		 * another pass across the zones.
		 */
		if (total_scanned && priority < DEF_PRIORITY - 2)
			blk_congestion_wait(WRITE, HZ/10);

		/*
		 * We do this so kswapd doesn't build up large priorities for
		 * example when it is freeing in parallel with allocators. It
		 * matches the direct reclaim path behaviour in terms of impact
		 * on zone->*_priority.
		 */
1674
		if ((nr_reclaimed >= SWAP_CLUSTER_MAX) && !nr_pages)
L
Linus Torvalds 已提交
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687
			break;
	}
out:
	for (i = 0; i < pgdat->nr_zones; i++) {
		struct zone *zone = pgdat->node_zones + i;

		zone->prev_priority = zone->temp_priority;
	}
	if (!all_zones_ok) {
		cond_resched();
		goto loop_again;
	}

1688
	return nr_reclaimed;
L
Linus Torvalds 已提交
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
}

/*
 * The background pageout daemon, started as a kernel thread
 * from the init process. 
 *
 * This basically trickles out pages so that we have _some_
 * free memory available even if there is no other activity
 * that frees anything up. This is needed for things like routing
 * etc, where we otherwise might have all activity going on in
 * asynchronous contexts that cannot page things out.
 *
 * If there are applications that are active memory-allocators
 * (most normal use), this basically shouldn't matter.
 */
static int kswapd(void *p)
{
	unsigned long order;
	pg_data_t *pgdat = (pg_data_t*)p;
	struct task_struct *tsk = current;
	DEFINE_WAIT(wait);
	struct reclaim_state reclaim_state = {
		.reclaimed_slab = 0,
	};
	cpumask_t cpumask;

	daemonize("kswapd%d", pgdat->node_id);
	cpumask = node_to_cpumask(pgdat->node_id);
	if (!cpus_empty(cpumask))
		set_cpus_allowed(tsk, cpumask);
	current->reclaim_state = &reclaim_state;

	/*
	 * Tell the memory management that we're a "memory allocator",
	 * and that if we need more memory we should get access to it
	 * regardless (see "__alloc_pages()"). "kswapd" should
	 * never get caught in the normal page freeing logic.
	 *
	 * (Kswapd normally doesn't need memory anyway, but sometimes
	 * you need a small amount of memory in order to be able to
	 * page out something else, and this flag essentially protects
	 * us from recursively trying to free more memory as we're
	 * trying to free the first piece of memory in the first place).
	 */
1733
	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
L
Linus Torvalds 已提交
1734 1735 1736 1737

	order = 0;
	for ( ; ; ) {
		unsigned long new_order;
1738 1739

		try_to_freeze();
L
Linus Torvalds 已提交
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767

		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
		new_order = pgdat->kswapd_max_order;
		pgdat->kswapd_max_order = 0;
		if (order < new_order) {
			/*
			 * Don't sleep if someone wants a larger 'order'
			 * allocation
			 */
			order = new_order;
		} else {
			schedule();
			order = pgdat->kswapd_max_order;
		}
		finish_wait(&pgdat->kswapd_wait, &wait);

		balance_pgdat(pgdat, 0, order);
	}
	return 0;
}

/*
 * A zone is low on free memory, so wake its kswapd task to service it.
 */
void wakeup_kswapd(struct zone *zone, int order)
{
	pg_data_t *pgdat;

1768
	if (!populated_zone(zone))
L
Linus Torvalds 已提交
1769 1770 1771
		return;

	pgdat = zone->zone_pgdat;
R
Rohit Seth 已提交
1772
	if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0))
L
Linus Torvalds 已提交
1773 1774 1775
		return;
	if (pgdat->kswapd_max_order < order)
		pgdat->kswapd_max_order = order;
1776
	if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
L
Linus Torvalds 已提交
1777
		return;
1778
	if (!waitqueue_active(&pgdat->kswapd_wait))
L
Linus Torvalds 已提交
1779
		return;
1780
	wake_up_interruptible(&pgdat->kswapd_wait);
L
Linus Torvalds 已提交
1781 1782 1783 1784 1785 1786 1787
}

#ifdef CONFIG_PM
/*
 * Try to free `nr_pages' of memory, system-wide.  Returns the number of freed
 * pages.
 */
1788
unsigned long shrink_all_memory(unsigned long nr_pages)
L
Linus Torvalds 已提交
1789 1790
{
	pg_data_t *pgdat;
1791 1792
	unsigned long nr_to_free = nr_pages;
	unsigned long ret = 0;
L
Linus Torvalds 已提交
1793 1794 1795 1796 1797 1798
	struct reclaim_state reclaim_state = {
		.reclaimed_slab = 0,
	};

	current->reclaim_state = &reclaim_state;
	for_each_pgdat(pgdat) {
1799 1800
		unsigned long freed;

L
Linus Torvalds 已提交
1801 1802 1803
		freed = balance_pgdat(pgdat, nr_to_free, 0);
		ret += freed;
		nr_to_free -= freed;
1804
		if ((long)nr_to_free <= 0)
L
Linus Torvalds 已提交
1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
			break;
	}
	current->reclaim_state = NULL;
	return ret;
}
#endif

#ifdef CONFIG_HOTPLUG_CPU
/* It's optimal to keep kswapds on the same CPUs as their memory, but
   not required for correctness.  So if the last cpu in a node goes
   away, we get changed to run anywhere: as the first one comes back,
   restore their cpu bindings. */
static int __devinit cpu_callback(struct notifier_block *nfb,
1818
				  unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
{
	pg_data_t *pgdat;
	cpumask_t mask;

	if (action == CPU_ONLINE) {
		for_each_pgdat(pgdat) {
			mask = node_to_cpumask(pgdat->node_id);
			if (any_online_cpu(mask) != NR_CPUS)
				/* One of our CPUs online: restore mask */
				set_cpus_allowed(pgdat->kswapd, mask);
		}
	}
	return NOTIFY_OK;
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __init kswapd_init(void)
{
	pg_data_t *pgdat;
1838

L
Linus Torvalds 已提交
1839
	swap_setup();
1840 1841 1842 1843 1844 1845 1846
	for_each_pgdat(pgdat) {
		pid_t pid;

		pid = kernel_thread(kswapd, pgdat, CLONE_KERNEL);
		BUG_ON(pid < 0);
		pgdat->kswapd = find_task_by_pid(pid);
	}
L
Linus Torvalds 已提交
1847 1848 1849 1850 1851 1852
	total_memory = nr_free_pagecache_pages();
	hotcpu_notifier(cpu_callback, 0);
	return 0;
}

module_init(kswapd_init)
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866

#ifdef CONFIG_NUMA
/*
 * Zone reclaim mode
 *
 * If non-zero call zone_reclaim when the number of free pages falls below
 * the watermarks.
 *
 * In the future we may add flags to the mode. However, the page allocator
 * should only have to check that zone_reclaim_mode != 0 before calling
 * zone_reclaim().
 */
int zone_reclaim_mode __read_mostly;

1867 1868 1869 1870
#define RECLAIM_OFF 0
#define RECLAIM_ZONE (1<<0)	/* Run shrink_cache on the zone */
#define RECLAIM_WRITE (1<<1)	/* Writeout pages during reclaim */
#define RECLAIM_SWAP (1<<2)	/* Swap pages out during reclaim */
1871
#define RECLAIM_SLAB (1<<3)	/* Do a global slab shrink if the zone is out of memory */
1872

1873 1874 1875
/*
 * Mininum time between zone reclaim scans
 */
1876
int zone_reclaim_interval __read_mostly = 30*HZ;
1877 1878 1879 1880 1881 1882 1883 1884

/*
 * Priority for ZONE_RECLAIM. This determines the fraction of pages
 * of a node considered for each zone_reclaim. 4 scans 1/16th of
 * a zone.
 */
#define ZONE_RECLAIM_PRIORITY 4

1885 1886 1887
/*
 * Try to free up some pages from this zone through reclaim.
 */
1888
static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
1889
{
1890
	/* Minimum pages needed in order to stay on node */
1891
	const unsigned long nr_pages = 1 << order;
1892 1893
	struct task_struct *p = current;
	struct reclaim_state reclaim_state;
1894
	int priority;
1895
	unsigned long nr_reclaimed = 0;
1896 1897 1898 1899
	struct scan_control sc = {
		.may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
		.may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP),
		.nr_mapped = read_page_state(nr_mapped),
1900 1901
		.swap_cluster_max = max_t(unsigned long, nr_pages,
					SWAP_CLUSTER_MAX),
1902 1903
		.gfp_mask = gfp_mask,
	};
1904 1905 1906

	disable_swap_token();
	cond_resched();
1907 1908 1909 1910 1911 1912
	/*
	 * We need to be able to allocate from the reserves for RECLAIM_SWAP
	 * and we also need to be able to write out pages for RECLAIM_WRITE
	 * and RECLAIM_SWAP.
	 */
	p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
1913 1914
	reclaim_state.reclaimed_slab = 0;
	p->reclaim_state = &reclaim_state;
1915

1916 1917 1918 1919
	/*
	 * Free memory by calling shrink zone with increasing priorities
	 * until we have enough memory freed.
	 */
1920
	priority = ZONE_RECLAIM_PRIORITY;
1921
	do {
1922
		nr_reclaimed += shrink_zone(priority, zone, &sc);
1923
		priority--;
1924
	} while (priority >= 0 && nr_reclaimed < nr_pages);
1925

1926
	if (nr_reclaimed < nr_pages && (zone_reclaim_mode & RECLAIM_SLAB)) {
1927
		/*
1928 1929 1930 1931 1932 1933
		 * shrink_slab() does not currently allow us to determine how
		 * many pages were freed in this zone. So we just shake the slab
		 * a bit and then go off node for this particular allocation
		 * despite possibly having freed enough memory to allocate in
		 * this zone.  If we freed local memory then the next
		 * allocations will be local again.
1934 1935 1936 1937 1938 1939 1940
		 *
		 * shrink_slab will free memory on all zones and may take
		 * a long time.
		 */
		shrink_slab(sc.nr_scanned, gfp_mask, order);
	}

1941
	p->reclaim_state = NULL;
1942
	current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
1943

1944 1945 1946 1947 1948 1949
	if (nr_reclaimed == 0) {
		/*
		 * We were unable to reclaim enough pages to stay on node.  We
		 * now allow off node accesses for a certain time period before
		 * trying again to reclaim pages from the local zone.
		 */
1950
		zone->last_unsuccessful_zone_reclaim = jiffies;
1951
	}
1952

1953
	return nr_reclaimed >= nr_pages;
1954
}
1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993

int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
{
	cpumask_t mask;
	int node_id;

	/*
	 * Do not reclaim if there was a recent unsuccessful attempt at zone
	 * reclaim.  In that case we let allocations go off node for the
	 * zone_reclaim_interval.  Otherwise we would scan for each off-node
	 * page allocation.
	 */
	if (time_before(jiffies,
		zone->last_unsuccessful_zone_reclaim + zone_reclaim_interval))
			return 0;

	/*
	 * Avoid concurrent zone reclaims, do not reclaim in a zone that does
	 * not have reclaimable pages and if we should not delay the allocation
	 * then do not scan.
	 */
	if (!(gfp_mask & __GFP_WAIT) ||
		zone->all_unreclaimable ||
		atomic_read(&zone->reclaim_in_progress) > 0 ||
		(current->flags & PF_MEMALLOC))
			return 0;

	/*
	 * Only run zone reclaim on the local zone or on zones that do not
	 * have associated processors. This will favor the local processor
	 * over remote processors and spread off node memory allocations
	 * as wide as possible.
	 */
	node_id = zone->zone_pgdat->node_id;
	mask = node_to_cpumask(node_id);
	if (!cpus_empty(mask) && node_id != numa_node_id())
		return 0;
	return __zone_reclaim(zone, gfp_mask, order);
}
1994
#endif