vmscan.c 110.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 *  linux/mm/vmscan.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  Swap reorganised 29.12.95, Stephen Tweedie.
 *  kswapd added: 7.1.96  sct
 *  Removed kswapd_ctl limits, and swap out as many pages as needed
 *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
 *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
 *  Multiqueue VM started 5.8.00, Rik van Riel.
 */

14 15
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

L
Linus Torvalds 已提交
16 17
#include <linux/mm.h>
#include <linux/module.h>
18
#include <linux/gfp.h>
L
Linus Torvalds 已提交
19 20 21 22 23
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/highmem.h>
24
#include <linux/vmpressure.h>
25
#include <linux/vmstat.h>
L
Linus Torvalds 已提交
26 27 28 29 30 31 32 33 34 35 36
#include <linux/file.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>	/* for try_to_release_page(),
					buffer_heads_over_limit */
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
37
#include <linux/compaction.h>
L
Linus Torvalds 已提交
38 39
#include <linux/notifier.h>
#include <linux/rwsem.h>
40
#include <linux/delay.h>
41
#include <linux/kthread.h>
42
#include <linux/freezer.h>
43
#include <linux/memcontrol.h>
44
#include <linux/delayacct.h>
45
#include <linux/sysctl.h>
46
#include <linux/oom.h>
47
#include <linux/prefetch.h>
48
#include <linux/printk.h>
49
#include <linux/dax.h>
L
Linus Torvalds 已提交
50 51 52 53 54

#include <asm/tlbflush.h>
#include <asm/div64.h>

#include <linux/swapops.h>
55
#include <linux/balloon_compaction.h>
L
Linus Torvalds 已提交
56

57 58
#include "internal.h"

59 60 61
#define CREATE_TRACE_POINTS
#include <trace/events/vmscan.h>

L
Linus Torvalds 已提交
62
struct scan_control {
63 64 65
	/* How many pages shrink_list() should reclaim */
	unsigned long nr_to_reclaim;

L
Linus Torvalds 已提交
66
	/* This context's GFP mask */
A
Al Viro 已提交
67
	gfp_t gfp_mask;
L
Linus Torvalds 已提交
68

69
	/* Allocation order */
A
Andy Whitcroft 已提交
70
	int order;
71

72 73 74 75 76
	/*
	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
	 * are scanned.
	 */
	nodemask_t	*nodemask;
77

78 79 80 81 82
	/*
	 * The memory cgroup that hit its limit and as a result is the
	 * primary target of this reclaim invocation.
	 */
	struct mem_cgroup *target_mem_cgroup;
83

84 85 86
	/* Scan (total_size >> priority) pages at once */
	int priority;

87 88 89
	/* The highest zone to isolate pages for reclaim from */
	enum zone_type reclaim_idx;

90 91 92 93 94 95 96 97
	unsigned int may_writepage:1;

	/* Can mapped pages be reclaimed? */
	unsigned int may_unmap:1;

	/* Can pages be swapped as part of reclaim? */
	unsigned int may_swap:1;

98 99 100
	/* Can cgroups be reclaimed below their normal consumption range? */
	unsigned int may_thrash:1;

101 102 103 104 105 106 107 108 109 110
	unsigned int hibernation_mode:1;

	/* One of the zones is ready for compaction */
	unsigned int compaction_ready:1;

	/* Incremented by the number of inactive pages that were scanned */
	unsigned long nr_scanned;

	/* Number of pages freed so far during a call to shrink_zones() */
	unsigned long nr_reclaimed;
L
Linus Torvalds 已提交
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
};

#ifdef ARCH_HAS_PREFETCH
#define prefetch_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetch(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

#ifdef ARCH_HAS_PREFETCHW
#define prefetchw_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetchw(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

/*
 * From 0 .. 100.  Higher means more swappy.
 */
int vm_swappiness = 60;
145 146 147 148 149
/*
 * The total number of pages which are beyond the high watermark within all
 * zones.
 */
unsigned long vm_total_pages;
L
Linus Torvalds 已提交
150 151 152 153

static LIST_HEAD(shrinker_list);
static DECLARE_RWSEM(shrinker_rwsem);

A
Andrew Morton 已提交
154
#ifdef CONFIG_MEMCG
155 156
static bool global_reclaim(struct scan_control *sc)
{
157
	return !sc->target_mem_cgroup;
158
}
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179

/**
 * sane_reclaim - is the usual dirty throttling mechanism operational?
 * @sc: scan_control in question
 *
 * The normal page dirty throttling mechanism in balance_dirty_pages() is
 * completely broken with the legacy memcg and direct stalling in
 * shrink_page_list() is used for throttling instead, which lacks all the
 * niceties such as fairness, adaptive pausing, bandwidth proportional
 * allocation and configurability.
 *
 * This function tests whether the vmscan currently in progress can assume
 * that the normal dirty throttling mechanism is operational.
 */
static bool sane_reclaim(struct scan_control *sc)
{
	struct mem_cgroup *memcg = sc->target_mem_cgroup;

	if (!memcg)
		return true;
#ifdef CONFIG_CGROUP_WRITEBACK
180
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
181 182 183 184
		return true;
#endif
	return false;
}
185
#else
186 187 188 189
static bool global_reclaim(struct scan_control *sc)
{
	return true;
}
190 191 192 193 194

static bool sane_reclaim(struct scan_control *sc)
{
	return true;
}
195 196
#endif

197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
/*
 * This misses isolated pages which are not accounted for to save counters.
 * As the data only determines if reclaim or compaction continues, it is
 * not expected that isolated pages will be a dominating factor.
 */
unsigned long zone_reclaimable_pages(struct zone *zone)
{
	unsigned long nr;

	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
	if (get_nr_swap_pages() > 0)
		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);

	return nr;
}

M
Mel Gorman 已提交
215 216 217 218 219 220 221
unsigned long pgdat_reclaimable_pages(struct pglist_data *pgdat)
{
	unsigned long nr;

	nr = node_page_state_snapshot(pgdat, NR_ACTIVE_FILE) +
	     node_page_state_snapshot(pgdat, NR_INACTIVE_FILE) +
	     node_page_state_snapshot(pgdat, NR_ISOLATED_FILE);
222 223

	if (get_nr_swap_pages() > 0)
M
Mel Gorman 已提交
224 225 226
		nr += node_page_state_snapshot(pgdat, NR_ACTIVE_ANON) +
		      node_page_state_snapshot(pgdat, NR_INACTIVE_ANON) +
		      node_page_state_snapshot(pgdat, NR_ISOLATED_ANON);
227 228 229 230

	return nr;
}

M
Mel Gorman 已提交
231
bool pgdat_reclaimable(struct pglist_data *pgdat)
232
{
M
Mel Gorman 已提交
233 234
	return node_page_state_snapshot(pgdat, NR_PAGES_SCANNED) <
		pgdat_reclaimable_pages(pgdat) * 6;
235 236
}

237
unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru)
238
{
239
	if (!mem_cgroup_disabled())
240
		return mem_cgroup_get_lru_size(lruvec, lru);
241

M
Mel Gorman 已提交
242
	return node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
243 244
}

L
Linus Torvalds 已提交
245
/*
G
Glauber Costa 已提交
246
 * Add a shrinker callback to be called from the vm.
L
Linus Torvalds 已提交
247
 */
G
Glauber Costa 已提交
248
int register_shrinker(struct shrinker *shrinker)
L
Linus Torvalds 已提交
249
{
G
Glauber Costa 已提交
250 251 252 253 254 255 256 257 258
	size_t size = sizeof(*shrinker->nr_deferred);

	if (shrinker->flags & SHRINKER_NUMA_AWARE)
		size *= nr_node_ids;

	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
	if (!shrinker->nr_deferred)
		return -ENOMEM;

259 260 261
	down_write(&shrinker_rwsem);
	list_add_tail(&shrinker->list, &shrinker_list);
	up_write(&shrinker_rwsem);
G
Glauber Costa 已提交
262
	return 0;
L
Linus Torvalds 已提交
263
}
264
EXPORT_SYMBOL(register_shrinker);
L
Linus Torvalds 已提交
265 266 267 268

/*
 * Remove one
 */
269
void unregister_shrinker(struct shrinker *shrinker)
L
Linus Torvalds 已提交
270 271 272 273
{
	down_write(&shrinker_rwsem);
	list_del(&shrinker->list);
	up_write(&shrinker_rwsem);
274
	kfree(shrinker->nr_deferred);
L
Linus Torvalds 已提交
275
}
276
EXPORT_SYMBOL(unregister_shrinker);
L
Linus Torvalds 已提交
277 278

#define SHRINK_BATCH 128
G
Glauber Costa 已提交
279

280 281 282 283
static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
				    struct shrinker *shrinker,
				    unsigned long nr_scanned,
				    unsigned long nr_eligible)
G
Glauber Costa 已提交
284 285 286 287
{
	unsigned long freed = 0;
	unsigned long long delta;
	long total_scan;
288
	long freeable;
G
Glauber Costa 已提交
289 290 291 292 293 294
	long nr;
	long new_nr;
	int nid = shrinkctl->nid;
	long batch_size = shrinker->batch ? shrinker->batch
					  : SHRINK_BATCH;

295 296
	freeable = shrinker->count_objects(shrinker, shrinkctl);
	if (freeable == 0)
G
Glauber Costa 已提交
297 298 299 300 301 302 303 304 305 306
		return 0;

	/*
	 * copy the current shrinker scan count into a local variable
	 * and zero it so that other concurrent shrinker invocations
	 * don't also do this scanning work.
	 */
	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);

	total_scan = nr;
307
	delta = (4 * nr_scanned) / shrinker->seeks;
308
	delta *= freeable;
309
	do_div(delta, nr_eligible + 1);
G
Glauber Costa 已提交
310 311
	total_scan += delta;
	if (total_scan < 0) {
312
		pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
D
Dave Chinner 已提交
313
		       shrinker->scan_objects, total_scan);
314
		total_scan = freeable;
G
Glauber Costa 已提交
315 316 317 318 319 320 321 322
	}

	/*
	 * We need to avoid excessive windup on filesystem shrinkers
	 * due to large numbers of GFP_NOFS allocations causing the
	 * shrinkers to return -1 all the time. This results in a large
	 * nr being built up so when a shrink that can do some work
	 * comes along it empties the entire cache due to nr >>>
323
	 * freeable. This is bad for sustaining a working set in
G
Glauber Costa 已提交
324 325 326 327 328
	 * memory.
	 *
	 * Hence only allow the shrinker to scan the entire cache when
	 * a large delta change is calculated directly.
	 */
329 330
	if (delta < freeable / 4)
		total_scan = min(total_scan, freeable / 2);
G
Glauber Costa 已提交
331 332 333 334 335 336

	/*
	 * Avoid risking looping forever due to too large nr value:
	 * never try to free more than twice the estimate number of
	 * freeable entries.
	 */
337 338
	if (total_scan > freeable * 2)
		total_scan = freeable * 2;
G
Glauber Costa 已提交
339 340

	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
341 342
				   nr_scanned, nr_eligible,
				   freeable, delta, total_scan);
G
Glauber Costa 已提交
343

344 345 346 347 348 349 350 351 352 353 354
	/*
	 * Normally, we should not scan less than batch_size objects in one
	 * pass to avoid too frequent shrinker calls, but if the slab has less
	 * than batch_size objects in total and we are really tight on memory,
	 * we will try to reclaim all available objects, otherwise we can end
	 * up failing allocations although there are plenty of reclaimable
	 * objects spread over several slabs with usage less than the
	 * batch_size.
	 *
	 * We detect the "tight on memory" situations by looking at the total
	 * number of objects we want to scan (total_scan). If it is greater
355
	 * than the total number of objects on slab (freeable), we must be
356 357 358 359
	 * scanning at high prio and therefore should try to reclaim as much as
	 * possible.
	 */
	while (total_scan >= batch_size ||
360
	       total_scan >= freeable) {
D
Dave Chinner 已提交
361
		unsigned long ret;
362
		unsigned long nr_to_scan = min(batch_size, total_scan);
G
Glauber Costa 已提交
363

364
		shrinkctl->nr_to_scan = nr_to_scan;
D
Dave Chinner 已提交
365 366 367 368
		ret = shrinker->scan_objects(shrinker, shrinkctl);
		if (ret == SHRINK_STOP)
			break;
		freed += ret;
G
Glauber Costa 已提交
369

370 371
		count_vm_events(SLABS_SCANNED, nr_to_scan);
		total_scan -= nr_to_scan;
G
Glauber Costa 已提交
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386

		cond_resched();
	}

	/*
	 * move the unused scan count back into the shrinker in a
	 * manner that handles concurrent updates. If we exhausted the
	 * scan, there is no need to do an update.
	 */
	if (total_scan > 0)
		new_nr = atomic_long_add_return(total_scan,
						&shrinker->nr_deferred[nid]);
	else
		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);

387
	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
G
Glauber Costa 已提交
388
	return freed;
389 390
}

391
/**
392
 * shrink_slab - shrink slab caches
393 394
 * @gfp_mask: allocation context
 * @nid: node whose slab caches to target
395
 * @memcg: memory cgroup whose slab caches to target
396 397
 * @nr_scanned: pressure numerator
 * @nr_eligible: pressure denominator
L
Linus Torvalds 已提交
398
 *
399
 * Call the shrink functions to age shrinkable caches.
L
Linus Torvalds 已提交
400
 *
401 402
 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
 * unaware shrinkers will receive a node id of 0 instead.
L
Linus Torvalds 已提交
403
 *
404 405
 * @memcg specifies the memory cgroup to target. If it is not NULL,
 * only shrinkers with SHRINKER_MEMCG_AWARE set will be called to scan
406 407
 * objects from the memory cgroup specified. Otherwise, only unaware
 * shrinkers are called.
408
 *
409 410 411 412 413 414 415
 * @nr_scanned and @nr_eligible form a ratio that indicate how much of
 * the available objects should be scanned.  Page reclaim for example
 * passes the number of pages scanned and the number of pages on the
 * LRU lists that it considered on @nid, plus a bias in @nr_scanned
 * when it encountered mapped pages.  The ratio is further biased by
 * the ->seeks setting of the shrink function, which indicates the
 * cost to recreate an object relative to that of an LRU page.
416
 *
417
 * Returns the number of reclaimed slab objects.
L
Linus Torvalds 已提交
418
 */
419 420 421 422
static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
				 struct mem_cgroup *memcg,
				 unsigned long nr_scanned,
				 unsigned long nr_eligible)
L
Linus Torvalds 已提交
423 424
{
	struct shrinker *shrinker;
D
Dave Chinner 已提交
425
	unsigned long freed = 0;
L
Linus Torvalds 已提交
426

427
	if (memcg && (!memcg_kmem_enabled() || !mem_cgroup_online(memcg)))
428 429
		return 0;

430 431
	if (nr_scanned == 0)
		nr_scanned = SWAP_CLUSTER_MAX;
L
Linus Torvalds 已提交
432

433
	if (!down_read_trylock(&shrinker_rwsem)) {
D
Dave Chinner 已提交
434 435 436 437 438 439 440
		/*
		 * If we would return 0, our callers would understand that we
		 * have nothing else to shrink and give up trying. By returning
		 * 1 we keep it going and assume we'll be able to shrink next
		 * time.
		 */
		freed = 1;
441 442
		goto out;
	}
L
Linus Torvalds 已提交
443 444

	list_for_each_entry(shrinker, &shrinker_list, list) {
445 446 447
		struct shrink_control sc = {
			.gfp_mask = gfp_mask,
			.nid = nid,
448
			.memcg = memcg,
449
		};
450

451 452 453 454 455 456 457
		/*
		 * If kernel memory accounting is disabled, we ignore
		 * SHRINKER_MEMCG_AWARE flag and call all shrinkers
		 * passing NULL for memcg.
		 */
		if (memcg_kmem_enabled() &&
		    !!memcg != !!(shrinker->flags & SHRINKER_MEMCG_AWARE))
458 459
			continue;

460 461
		if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
			sc.nid = 0;
L
Linus Torvalds 已提交
462

463
		freed += do_shrink_slab(&sc, shrinker, nr_scanned, nr_eligible);
L
Linus Torvalds 已提交
464
	}
465

L
Linus Torvalds 已提交
466
	up_read(&shrinker_rwsem);
467 468
out:
	cond_resched();
D
Dave Chinner 已提交
469
	return freed;
L
Linus Torvalds 已提交
470 471
}

472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
void drop_slab_node(int nid)
{
	unsigned long freed;

	do {
		struct mem_cgroup *memcg = NULL;

		freed = 0;
		do {
			freed += shrink_slab(GFP_KERNEL, nid, memcg,
					     1000, 1000);
		} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
	} while (freed > 10);
}

void drop_slab(void)
{
	int nid;

	for_each_online_node(nid)
		drop_slab_node(nid);
}

L
Linus Torvalds 已提交
495 496
static inline int is_page_cache_freeable(struct page *page)
{
497 498 499 500 501
	/*
	 * A freeable page cache page is referenced only by the caller
	 * that isolated the page, the page cache radix tree and
	 * optional buffer heads at page->private.
	 */
502
	return page_count(page) - page_has_private(page) == 2;
L
Linus Torvalds 已提交
503 504
}

505
static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
L
Linus Torvalds 已提交
506
{
507
	if (current->flags & PF_SWAPWRITE)
L
Linus Torvalds 已提交
508
		return 1;
509
	if (!inode_write_congested(inode))
L
Linus Torvalds 已提交
510
		return 1;
511
	if (inode_to_bdi(inode) == current->backing_dev_info)
L
Linus Torvalds 已提交
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
		return 1;
	return 0;
}

/*
 * We detected a synchronous write error writing a page out.  Probably
 * -ENOSPC.  We need to propagate that into the address_space for a subsequent
 * fsync(), msync() or close().
 *
 * The tricky part is that after writepage we cannot touch the mapping: nothing
 * prevents it from being freed up.  But we have a ref on the page and once
 * that page is locked, the mapping is pinned.
 *
 * We're allowed to run sleeping lock_page() here because we know the caller has
 * __GFP_FS.
 */
static void handle_write_error(struct address_space *mapping,
				struct page *page, int error)
{
J
Jens Axboe 已提交
531
	lock_page(page);
532 533
	if (page_mapping(page) == mapping)
		mapping_set_error(mapping, error);
L
Linus Torvalds 已提交
534 535 536
	unlock_page(page);
}

537 538 539 540 541 542 543 544 545 546 547 548
/* possible outcome of pageout() */
typedef enum {
	/* failed to write page out, page is locked */
	PAGE_KEEP,
	/* move page to the active list, page is locked */
	PAGE_ACTIVATE,
	/* page has been sent to the disk successfully, page is unlocked */
	PAGE_SUCCESS,
	/* page is clean and locked */
	PAGE_CLEAN,
} pageout_t;

L
Linus Torvalds 已提交
549
/*
A
Andrew Morton 已提交
550 551
 * pageout is called by shrink_page_list() for each dirty page.
 * Calls ->writepage().
L
Linus Torvalds 已提交
552
 */
553
static pageout_t pageout(struct page *page, struct address_space *mapping,
554
			 struct scan_control *sc)
L
Linus Torvalds 已提交
555 556 557 558 559 560 561 562
{
	/*
	 * If the page is dirty, only perform writeback if that write
	 * will be non-blocking.  To prevent this allocation from being
	 * stalled by pagecache activity.  But note that there may be
	 * stalls if we need to run get_block().  We could test
	 * PagePrivate for that.
	 *
563
	 * If this process is currently in __generic_file_write_iter() against
L
Linus Torvalds 已提交
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
	 * this page's queue, we can perform writeback even if that
	 * will block.
	 *
	 * If the page is swapcache, write it back even if that would
	 * block, for some throttling. This happens by accident, because
	 * swap_backing_dev_info is bust: it doesn't reflect the
	 * congestion state of the swapdevs.  Easy to fix, if needed.
	 */
	if (!is_page_cache_freeable(page))
		return PAGE_KEEP;
	if (!mapping) {
		/*
		 * Some data journaling orphaned pages can have
		 * page->mapping == NULL while being dirty with clean buffers.
		 */
579
		if (page_has_private(page)) {
L
Linus Torvalds 已提交
580 581
			if (try_to_free_buffers(page)) {
				ClearPageDirty(page);
582
				pr_info("%s: orphaned page\n", __func__);
L
Linus Torvalds 已提交
583 584 585 586 587 588 589
				return PAGE_CLEAN;
			}
		}
		return PAGE_KEEP;
	}
	if (mapping->a_ops->writepage == NULL)
		return PAGE_ACTIVATE;
590
	if (!may_write_to_inode(mapping->host, sc))
L
Linus Torvalds 已提交
591 592 593 594 595 596 597
		return PAGE_KEEP;

	if (clear_page_dirty_for_io(page)) {
		int res;
		struct writeback_control wbc = {
			.sync_mode = WB_SYNC_NONE,
			.nr_to_write = SWAP_CLUSTER_MAX,
598 599
			.range_start = 0,
			.range_end = LLONG_MAX,
L
Linus Torvalds 已提交
600 601 602 603 604 605 606
			.for_reclaim = 1,
		};

		SetPageReclaim(page);
		res = mapping->a_ops->writepage(page, &wbc);
		if (res < 0)
			handle_write_error(mapping, page, res);
607
		if (res == AOP_WRITEPAGE_ACTIVATE) {
L
Linus Torvalds 已提交
608 609 610
			ClearPageReclaim(page);
			return PAGE_ACTIVATE;
		}
611

L
Linus Torvalds 已提交
612 613 614 615
		if (!PageWriteback(page)) {
			/* synchronous write or broken a_ops? */
			ClearPageReclaim(page);
		}
616
		trace_mm_vmscan_writepage(page);
617
		inc_node_page_state(page, NR_VMSCAN_WRITE);
L
Linus Torvalds 已提交
618 619 620 621 622 623
		return PAGE_SUCCESS;
	}

	return PAGE_CLEAN;
}

624
/*
N
Nick Piggin 已提交
625 626
 * Same as remove_mapping, but if the page is removed from the mapping, it
 * gets returned with a refcount of 0.
627
 */
628 629
static int __remove_mapping(struct address_space *mapping, struct page *page,
			    bool reclaimed)
630
{
631 632
	unsigned long flags;

633 634
	BUG_ON(!PageLocked(page));
	BUG_ON(mapping != page_mapping(page));
635

636
	spin_lock_irqsave(&mapping->tree_lock, flags);
637
	/*
N
Nick Piggin 已提交
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
	 * The non racy check for a busy page.
	 *
	 * Must be careful with the order of the tests. When someone has
	 * a ref to the page, it may be possible that they dirty it then
	 * drop the reference. So if PageDirty is tested before page_count
	 * here, then the following race may occur:
	 *
	 * get_user_pages(&page);
	 * [user mapping goes away]
	 * write_to(page);
	 *				!PageDirty(page)    [good]
	 * SetPageDirty(page);
	 * put_page(page);
	 *				!page_count(page)   [good, discard it]
	 *
	 * [oops, our write_to data is lost]
	 *
	 * Reversing the order of the tests ensures such a situation cannot
	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
657
	 * load is not satisfied before that of page->_refcount.
N
Nick Piggin 已提交
658 659 660
	 *
	 * Note that if SetPageDirty is always performed via set_page_dirty,
	 * and thus under tree_lock, then this ordering is not required.
661
	 */
662
	if (!page_ref_freeze(page, 2))
663
		goto cannot_free;
N
Nick Piggin 已提交
664 665
	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
	if (unlikely(PageDirty(page))) {
666
		page_ref_unfreeze(page, 2);
667
		goto cannot_free;
N
Nick Piggin 已提交
668
	}
669 670 671

	if (PageSwapCache(page)) {
		swp_entry_t swap = { .val = page_private(page) };
672
		mem_cgroup_swapout(page, swap);
673
		__delete_from_swap_cache(page);
674
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
675
		swapcache_free(swap);
N
Nick Piggin 已提交
676
	} else {
677
		void (*freepage)(struct page *);
678
		void *shadow = NULL;
679 680

		freepage = mapping->a_ops->freepage;
681 682 683 684 685 686 687 688 689
		/*
		 * Remember a shadow entry for reclaimed file cache in
		 * order to detect refaults, thus thrashing, later on.
		 *
		 * But don't store shadows in an address space that is
		 * already exiting.  This is not just an optizimation,
		 * inode reclaim needs to empty out the radix tree or
		 * the nodes are lost.  Don't plant shadows behind its
		 * back.
690 691 692 693 694 695
		 *
		 * We also don't store shadows for DAX mappings because the
		 * only page cache pages found in these are zero pages
		 * covering holes, and because we don't want to mix DAX
		 * exceptional entries and shadow exceptional entries in the
		 * same page_tree.
696 697
		 */
		if (reclaimed && page_is_file_cache(page) &&
698
		    !mapping_exiting(mapping) && !dax_mapping(mapping))
699
			shadow = workingset_eviction(mapping, page);
J
Johannes Weiner 已提交
700
		__delete_from_page_cache(page, shadow);
701
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
702 703 704

		if (freepage != NULL)
			freepage(page);
705 706 707 708 709
	}

	return 1;

cannot_free:
710
	spin_unlock_irqrestore(&mapping->tree_lock, flags);
711 712 713
	return 0;
}

N
Nick Piggin 已提交
714 715 716 717 718 719 720 721
/*
 * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
 * someone else has a ref on the page, abort and return 0.  If it was
 * successfully detached, return 1.  Assumes the caller has a single ref on
 * this page.
 */
int remove_mapping(struct address_space *mapping, struct page *page)
{
722
	if (__remove_mapping(mapping, page, false)) {
N
Nick Piggin 已提交
723 724 725 726 727
		/*
		 * Unfreezing the refcount with 1 rather than 2 effectively
		 * drops the pagecache ref for us without requiring another
		 * atomic operation.
		 */
728
		page_ref_unfreeze(page, 1);
N
Nick Piggin 已提交
729 730 731 732 733
		return 1;
	}
	return 0;
}

L
Lee Schermerhorn 已提交
734 735 736 737 738 739 740 741 742 743 744
/**
 * putback_lru_page - put previously isolated page onto appropriate LRU list
 * @page: page to be put back to appropriate lru list
 *
 * Add previously isolated @page to appropriate LRU list.
 * Page may still be unevictable for other reasons.
 *
 * lru_lock must not be held, interrupts must be enabled.
 */
void putback_lru_page(struct page *page)
{
745
	bool is_unevictable;
746
	int was_unevictable = PageUnevictable(page);
L
Lee Schermerhorn 已提交
747

748
	VM_BUG_ON_PAGE(PageLRU(page), page);
L
Lee Schermerhorn 已提交
749 750 751 752

redo:
	ClearPageUnevictable(page);

753
	if (page_evictable(page)) {
L
Lee Schermerhorn 已提交
754 755 756 757 758 759
		/*
		 * For evictable pages, we can use the cache.
		 * In event of a race, worst case is we end up with an
		 * unevictable page on [in]active list.
		 * We know how to handle that.
		 */
760
		is_unevictable = false;
761
		lru_cache_add(page);
L
Lee Schermerhorn 已提交
762 763 764 765 766
	} else {
		/*
		 * Put unevictable pages directly on zone's unevictable
		 * list.
		 */
767
		is_unevictable = true;
L
Lee Schermerhorn 已提交
768
		add_page_to_unevictable_list(page);
769
		/*
770 771 772
		 * When racing with an mlock or AS_UNEVICTABLE clearing
		 * (page is unlocked) make sure that if the other thread
		 * does not observe our setting of PG_lru and fails
773
		 * isolation/check_move_unevictable_pages,
774
		 * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
775 776
		 * the page back to the evictable list.
		 *
777
		 * The other side is TestClearPageMlocked() or shmem_lock().
778 779
		 */
		smp_mb();
L
Lee Schermerhorn 已提交
780 781 782 783 784 785 786
	}

	/*
	 * page's status can change while we move it among lru. If an evictable
	 * page is on unevictable list, it never be freed. To avoid that,
	 * check after we added it to the list, again.
	 */
787
	if (is_unevictable && page_evictable(page)) {
L
Lee Schermerhorn 已提交
788 789 790 791 792 793 794 795 796 797
		if (!isolate_lru_page(page)) {
			put_page(page);
			goto redo;
		}
		/* This means someone else dropped this page from LRU
		 * So, it will be freed or putback to LRU again. There is
		 * nothing to do here.
		 */
	}

798
	if (was_unevictable && !is_unevictable)
799
		count_vm_event(UNEVICTABLE_PGRESCUED);
800
	else if (!was_unevictable && is_unevictable)
801 802
		count_vm_event(UNEVICTABLE_PGCULLED);

L
Lee Schermerhorn 已提交
803 804 805
	put_page(page);		/* drop ref from isolate */
}

806 807 808
enum page_references {
	PAGEREF_RECLAIM,
	PAGEREF_RECLAIM_CLEAN,
809
	PAGEREF_KEEP,
810 811 812 813 814 815
	PAGEREF_ACTIVATE,
};

static enum page_references page_check_references(struct page *page,
						  struct scan_control *sc)
{
816
	int referenced_ptes, referenced_page;
817 818
	unsigned long vm_flags;

819 820
	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
					  &vm_flags);
821
	referenced_page = TestClearPageReferenced(page);
822 823 824 825 826 827 828 829

	/*
	 * Mlock lost the isolation race with us.  Let try_to_unmap()
	 * move the page to the unevictable list.
	 */
	if (vm_flags & VM_LOCKED)
		return PAGEREF_RECLAIM;

830
	if (referenced_ptes) {
831
		if (PageSwapBacked(page))
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
			return PAGEREF_ACTIVATE;
		/*
		 * All mapped pages start out with page table
		 * references from the instantiating fault, so we need
		 * to look twice if a mapped file page is used more
		 * than once.
		 *
		 * Mark it and spare it for another trip around the
		 * inactive list.  Another page table reference will
		 * lead to its activation.
		 *
		 * Note: the mark is set for activated pages as well
		 * so that recently deactivated but used pages are
		 * quickly recovered.
		 */
		SetPageReferenced(page);

849
		if (referenced_page || referenced_ptes > 1)
850 851
			return PAGEREF_ACTIVATE;

852 853 854 855 856 857
		/*
		 * Activate file-backed executable pages after first usage.
		 */
		if (vm_flags & VM_EXEC)
			return PAGEREF_ACTIVATE;

858 859
		return PAGEREF_KEEP;
	}
860 861

	/* Reclaim if clean, defer dirty pages to writeback */
862
	if (referenced_page && !PageSwapBacked(page))
863 864 865
		return PAGEREF_RECLAIM_CLEAN;

	return PAGEREF_RECLAIM;
866 867
}

868 869 870 871
/* Check if a page is dirty or under writeback */
static void page_check_dirty_writeback(struct page *page,
				       bool *dirty, bool *writeback)
{
872 873
	struct address_space *mapping;

874 875 876 877 878 879 880 881 882 883 884 885 886
	/*
	 * Anonymous pages are not handled by flushers and must be written
	 * from reclaim context. Do not stall reclaim based on them
	 */
	if (!page_is_file_cache(page)) {
		*dirty = false;
		*writeback = false;
		return;
	}

	/* By default assume that the page flags are accurate */
	*dirty = PageDirty(page);
	*writeback = PageWriteback(page);
887 888 889 890 891 892 893 894

	/* Verify dirty/writeback state if the filesystem supports it */
	if (!page_has_private(page))
		return;

	mapping = page_mapping(page);
	if (mapping && mapping->a_ops->is_dirty_writeback)
		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
895 896
}

L
Linus Torvalds 已提交
897
/*
A
Andrew Morton 已提交
898
 * shrink_page_list() returns the number of reclaimed pages
L
Linus Torvalds 已提交
899
 */
A
Andrew Morton 已提交
900
static unsigned long shrink_page_list(struct list_head *page_list,
M
Mel Gorman 已提交
901
				      struct pglist_data *pgdat,
902
				      struct scan_control *sc,
903
				      enum ttu_flags ttu_flags,
904
				      unsigned long *ret_nr_dirty,
905
				      unsigned long *ret_nr_unqueued_dirty,
906
				      unsigned long *ret_nr_congested,
907
				      unsigned long *ret_nr_writeback,
908
				      unsigned long *ret_nr_immediate,
909
				      bool force_reclaim)
L
Linus Torvalds 已提交
910 911
{
	LIST_HEAD(ret_pages);
912
	LIST_HEAD(free_pages);
L
Linus Torvalds 已提交
913
	int pgactivate = 0;
914
	unsigned long nr_unqueued_dirty = 0;
915 916
	unsigned long nr_dirty = 0;
	unsigned long nr_congested = 0;
917
	unsigned long nr_reclaimed = 0;
918
	unsigned long nr_writeback = 0;
919
	unsigned long nr_immediate = 0;
L
Linus Torvalds 已提交
920 921 922 923 924 925 926

	cond_resched();

	while (!list_empty(page_list)) {
		struct address_space *mapping;
		struct page *page;
		int may_enter_fs;
927
		enum page_references references = PAGEREF_RECLAIM_CLEAN;
928
		bool dirty, writeback;
M
Minchan Kim 已提交
929 930
		bool lazyfree = false;
		int ret = SWAP_SUCCESS;
L
Linus Torvalds 已提交
931 932 933 934 935 936

		cond_resched();

		page = lru_to_page(page_list);
		list_del(&page->lru);

N
Nick Piggin 已提交
937
		if (!trylock_page(page))
L
Linus Torvalds 已提交
938 939
			goto keep;

940
		VM_BUG_ON_PAGE(PageActive(page), page);
L
Linus Torvalds 已提交
941 942

		sc->nr_scanned++;
943

944
		if (unlikely(!page_evictable(page)))
N
Nick Piggin 已提交
945
			goto cull_mlocked;
L
Lee Schermerhorn 已提交
946

947
		if (!sc->may_unmap && page_mapped(page))
948 949
			goto keep_locked;

L
Linus Torvalds 已提交
950 951 952 953
		/* Double the slab pressure for mapped and swapcache pages */
		if (page_mapped(page) || PageSwapCache(page))
			sc->nr_scanned++;

954 955 956
		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));

957 958 959 960 961 962 963 964 965 966 967 968 969
		/*
		 * The number of dirty pages determines if a zone is marked
		 * reclaim_congested which affects wait_iff_congested. kswapd
		 * will stall and start writing pages if the tail of the LRU
		 * is all dirty unqueued pages.
		 */
		page_check_dirty_writeback(page, &dirty, &writeback);
		if (dirty || writeback)
			nr_dirty++;

		if (dirty && !writeback)
			nr_unqueued_dirty++;

970 971 972 973 974 975
		/*
		 * Treat this page as congested if the underlying BDI is or if
		 * pages are cycling through the LRU so quickly that the
		 * pages marked for immediate reclaim are making it to the
		 * end of the LRU a second time.
		 */
976
		mapping = page_mapping(page);
977
		if (((dirty || writeback) && mapping &&
978
		     inode_write_congested(mapping->host)) ||
979
		    (writeback && PageReclaim(page)))
980 981
			nr_congested++;

982 983 984 985 986 987 988 989 990 991 992
		/*
		 * If a page at the tail of the LRU is under writeback, there
		 * are three cases to consider.
		 *
		 * 1) If reclaim is encountering an excessive number of pages
		 *    under writeback and this page is both under writeback and
		 *    PageReclaim then it indicates that pages are being queued
		 *    for IO but are being recycled through the LRU before the
		 *    IO can complete. Waiting on the page itself risks an
		 *    indefinite stall if it is impossible to writeback the
		 *    page due to IO error or disconnected storage so instead
993 994
		 *    note that the LRU is being scanned too quickly and the
		 *    caller can stall after page list has been processed.
995
		 *
996
		 * 2) Global or new memcg reclaim encounters a page that is
997 998 999
		 *    not marked for immediate reclaim, or the caller does not
		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
		 *    not to fs). In this case mark the page for immediate
1000
		 *    reclaim and continue scanning.
1001
		 *
1002 1003
		 *    Require may_enter_fs because we would wait on fs, which
		 *    may not have submitted IO yet. And the loop driver might
1004 1005 1006 1007 1008
		 *    enter reclaim, and deadlock if it waits on a page for
		 *    which it is needed to do the write (loop masks off
		 *    __GFP_IO|__GFP_FS for this reason); but more thought
		 *    would probably show more reasons.
		 *
1009
		 * 3) Legacy memcg encounters a page that is already marked
1010 1011 1012 1013 1014
		 *    PageReclaim. memcg does not have any dirty pages
		 *    throttling so we could easily OOM just because too many
		 *    pages are in writeback and there is nothing else to
		 *    reclaim. Wait for the writeback to complete.
		 */
1015
		if (PageWriteback(page)) {
1016 1017 1018
			/* Case 1 above */
			if (current_is_kswapd() &&
			    PageReclaim(page) &&
M
Mel Gorman 已提交
1019
			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1020 1021
				nr_immediate++;
				goto keep_locked;
1022 1023

			/* Case 2 above */
1024
			} else if (sane_reclaim(sc) ||
1025
			    !PageReclaim(page) || !may_enter_fs) {
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
				/*
				 * This is slightly racy - end_page_writeback()
				 * might have just cleared PageReclaim, then
				 * setting PageReclaim here end up interpreted
				 * as PageReadahead - but that does not matter
				 * enough to care.  What we do want is for this
				 * page to have PageReclaim set next time memcg
				 * reclaim reaches the tests above, so it will
				 * then wait_on_page_writeback() to avoid OOM;
				 * and it's also appropriate in global reclaim.
				 */
				SetPageReclaim(page);
1038
				nr_writeback++;
1039
				goto keep_locked;
1040 1041 1042

			/* Case 3 above */
			} else {
1043
				unlock_page(page);
1044
				wait_on_page_writeback(page);
1045 1046 1047
				/* then go back and try same page again */
				list_add_tail(&page->lru, page_list);
				continue;
1048
			}
1049
		}
L
Linus Torvalds 已提交
1050

1051 1052 1053
		if (!force_reclaim)
			references = page_check_references(page, sc);

1054 1055
		switch (references) {
		case PAGEREF_ACTIVATE:
L
Linus Torvalds 已提交
1056
			goto activate_locked;
1057 1058
		case PAGEREF_KEEP:
			goto keep_locked;
1059 1060 1061 1062
		case PAGEREF_RECLAIM:
		case PAGEREF_RECLAIM_CLEAN:
			; /* try to reclaim the page below */
		}
L
Linus Torvalds 已提交
1063 1064 1065 1066 1067

		/*
		 * Anonymous process memory has backing store?
		 * Try to allocate it some swap space here.
		 */
N
Nick Piggin 已提交
1068
		if (PageAnon(page) && !PageSwapCache(page)) {
1069 1070
			if (!(sc->gfp_mask & __GFP_IO))
				goto keep_locked;
1071
			if (!add_to_swap(page, page_list))
L
Linus Torvalds 已提交
1072
				goto activate_locked;
M
Minchan Kim 已提交
1073
			lazyfree = true;
1074
			may_enter_fs = 1;
L
Linus Torvalds 已提交
1075

1076 1077
			/* Adding to swap updated mapping */
			mapping = page_mapping(page);
1078 1079 1080 1081
		} else if (unlikely(PageTransHuge(page))) {
			/* Split file THP */
			if (split_huge_page_to_list(page, page_list))
				goto keep_locked;
1082
		}
L
Linus Torvalds 已提交
1083

1084 1085
		VM_BUG_ON_PAGE(PageTransHuge(page), page);

L
Linus Torvalds 已提交
1086 1087 1088 1089 1090
		/*
		 * The page is mapped into the page tables of one or more
		 * processes. Try to unmap it here.
		 */
		if (page_mapped(page) && mapping) {
M
Minchan Kim 已提交
1091 1092 1093
			switch (ret = try_to_unmap(page, lazyfree ?
				(ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) :
				(ttu_flags | TTU_BATCH_FLUSH))) {
L
Linus Torvalds 已提交
1094 1095 1096 1097
			case SWAP_FAIL:
				goto activate_locked;
			case SWAP_AGAIN:
				goto keep_locked;
N
Nick Piggin 已提交
1098 1099
			case SWAP_MLOCK:
				goto cull_mlocked;
M
Minchan Kim 已提交
1100 1101
			case SWAP_LZFREE:
				goto lazyfree;
L
Linus Torvalds 已提交
1102 1103 1104 1105 1106 1107
			case SWAP_SUCCESS:
				; /* try to free the page below */
			}
		}

		if (PageDirty(page)) {
1108 1109
			/*
			 * Only kswapd can writeback filesystem pages to
1110 1111
			 * avoid risk of stack overflow but only writeback
			 * if many dirty pages have been encountered.
1112
			 */
1113
			if (page_is_file_cache(page) &&
1114
					(!current_is_kswapd() ||
M
Mel Gorman 已提交
1115
					 !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1116 1117 1118 1119 1120 1121
				/*
				 * Immediately reclaim when written back.
				 * Similar in principal to deactivate_page()
				 * except we already have the page isolated
				 * and know it's dirty
				 */
1122
				inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1123 1124
				SetPageReclaim(page);

1125 1126 1127
				goto keep_locked;
			}

1128
			if (references == PAGEREF_RECLAIM_CLEAN)
L
Linus Torvalds 已提交
1129
				goto keep_locked;
1130
			if (!may_enter_fs)
L
Linus Torvalds 已提交
1131
				goto keep_locked;
1132
			if (!sc->may_writepage)
L
Linus Torvalds 已提交
1133 1134
				goto keep_locked;

1135 1136 1137 1138 1139 1140
			/*
			 * Page is dirty. Flush the TLB if a writable entry
			 * potentially exists to avoid CPU writes after IO
			 * starts and then write it out here.
			 */
			try_to_unmap_flush_dirty();
1141
			switch (pageout(page, mapping, sc)) {
L
Linus Torvalds 已提交
1142 1143 1144 1145 1146
			case PAGE_KEEP:
				goto keep_locked;
			case PAGE_ACTIVATE:
				goto activate_locked;
			case PAGE_SUCCESS:
1147
				if (PageWriteback(page))
1148
					goto keep;
1149
				if (PageDirty(page))
L
Linus Torvalds 已提交
1150
					goto keep;
1151

L
Linus Torvalds 已提交
1152 1153 1154 1155
				/*
				 * A synchronous write - probably a ramdisk.  Go
				 * ahead and try to reclaim the page.
				 */
N
Nick Piggin 已提交
1156
				if (!trylock_page(page))
L
Linus Torvalds 已提交
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
					goto keep;
				if (PageDirty(page) || PageWriteback(page))
					goto keep_locked;
				mapping = page_mapping(page);
			case PAGE_CLEAN:
				; /* try to free the page below */
			}
		}

		/*
		 * If the page has buffers, try to free the buffer mappings
		 * associated with this page. If we succeed we try to free
		 * the page as well.
		 *
		 * We do this even if the page is PageDirty().
		 * try_to_release_page() does not perform I/O, but it is
		 * possible for a page to have PageDirty set, but it is actually
		 * clean (all its buffers are clean).  This happens if the
		 * buffers were written out directly, with submit_bh(). ext3
L
Lee Schermerhorn 已提交
1176
		 * will do this, as well as the blockdev mapping.
L
Linus Torvalds 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
		 * try_to_release_page() will discover that cleanness and will
		 * drop the buffers and mark the page clean - it can be freed.
		 *
		 * Rarely, pages can have buffers and no ->mapping.  These are
		 * the pages which were not successfully invalidated in
		 * truncate_complete_page().  We try to drop those buffers here
		 * and if that worked, and the page is no longer mapped into
		 * process address space (page_count == 1) it can be freed.
		 * Otherwise, leave the page on the LRU so it is swappable.
		 */
1187
		if (page_has_private(page)) {
L
Linus Torvalds 已提交
1188 1189
			if (!try_to_release_page(page, sc->gfp_mask))
				goto activate_locked;
N
Nick Piggin 已提交
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
			if (!mapping && page_count(page) == 1) {
				unlock_page(page);
				if (put_page_testzero(page))
					goto free_it;
				else {
					/*
					 * rare race with speculative reference.
					 * the speculative reference will free
					 * this page shortly, so we may
					 * increment nr_reclaimed here (and
					 * leave it off the LRU).
					 */
					nr_reclaimed++;
					continue;
				}
			}
L
Linus Torvalds 已提交
1206 1207
		}

M
Minchan Kim 已提交
1208
lazyfree:
1209
		if (!mapping || !__remove_mapping(mapping, page, true))
1210
			goto keep_locked;
L
Linus Torvalds 已提交
1211

N
Nick Piggin 已提交
1212 1213 1214 1215 1216 1217 1218
		/*
		 * At this point, we have no other references and there is
		 * no way to pick any more up (removed from LRU, removed
		 * from pagecache). Can use non-atomic bitops now (and
		 * we obviously don't have to worry about waking up a process
		 * waiting on the page lock, because there are no references.
		 */
1219
		__ClearPageLocked(page);
N
Nick Piggin 已提交
1220
free_it:
M
Minchan Kim 已提交
1221 1222 1223
		if (ret == SWAP_LZFREE)
			count_vm_event(PGLAZYFREED);

1224
		nr_reclaimed++;
1225 1226 1227 1228 1229 1230

		/*
		 * Is there need to periodically free_page_list? It would
		 * appear not as the counts should be low
		 */
		list_add(&page->lru, &free_pages);
L
Linus Torvalds 已提交
1231 1232
		continue;

N
Nick Piggin 已提交
1233
cull_mlocked:
1234 1235
		if (PageSwapCache(page))
			try_to_free_swap(page);
N
Nick Piggin 已提交
1236
		unlock_page(page);
1237
		list_add(&page->lru, &ret_pages);
N
Nick Piggin 已提交
1238 1239
		continue;

L
Linus Torvalds 已提交
1240
activate_locked:
1241
		/* Not a candidate for swapping, so reclaim swap space. */
1242
		if (PageSwapCache(page) && mem_cgroup_swap_full(page))
1243
			try_to_free_swap(page);
1244
		VM_BUG_ON_PAGE(PageActive(page), page);
L
Linus Torvalds 已提交
1245 1246 1247 1248 1249 1250
		SetPageActive(page);
		pgactivate++;
keep_locked:
		unlock_page(page);
keep:
		list_add(&page->lru, &ret_pages);
1251
		VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
L
Linus Torvalds 已提交
1252
	}
1253

1254
	mem_cgroup_uncharge_list(&free_pages);
1255
	try_to_unmap_flush();
1256
	free_hot_cold_page_list(&free_pages, true);
1257

L
Linus Torvalds 已提交
1258
	list_splice(&ret_pages, page_list);
1259
	count_vm_events(PGACTIVATE, pgactivate);
1260

1261 1262
	*ret_nr_dirty += nr_dirty;
	*ret_nr_congested += nr_congested;
1263
	*ret_nr_unqueued_dirty += nr_unqueued_dirty;
1264
	*ret_nr_writeback += nr_writeback;
1265
	*ret_nr_immediate += nr_immediate;
1266
	return nr_reclaimed;
L
Linus Torvalds 已提交
1267 1268
}

1269 1270 1271 1272 1273 1274 1275 1276
unsigned long reclaim_clean_pages_from_list(struct zone *zone,
					    struct list_head *page_list)
{
	struct scan_control sc = {
		.gfp_mask = GFP_KERNEL,
		.priority = DEF_PRIORITY,
		.may_unmap = 1,
	};
1277
	unsigned long ret, dummy1, dummy2, dummy3, dummy4, dummy5;
1278 1279 1280 1281
	struct page *page, *next;
	LIST_HEAD(clean_pages);

	list_for_each_entry_safe(page, next, page_list, lru) {
1282
		if (page_is_file_cache(page) && !PageDirty(page) &&
1283
		    !__PageMovable(page)) {
1284 1285 1286 1287 1288
			ClearPageActive(page);
			list_move(&page->lru, &clean_pages);
		}
	}

M
Mel Gorman 已提交
1289
	ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1290 1291
			TTU_UNMAP|TTU_IGNORE_ACCESS,
			&dummy1, &dummy2, &dummy3, &dummy4, &dummy5, true);
1292
	list_splice(&clean_pages, page_list);
M
Mel Gorman 已提交
1293
	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1294 1295 1296
	return ret;
}

A
Andy Whitcroft 已提交
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
/*
 * Attempt to remove the specified page from its LRU.  Only take this page
 * if it is of the appropriate PageActive status.  Pages which are being
 * freed elsewhere are also ignored.
 *
 * page:	page to consider
 * mode:	one of the LRU isolation modes defined above
 *
 * returns 0 on success, -ve errno on failure.
 */
1307
int __isolate_lru_page(struct page *page, isolate_mode_t mode)
A
Andy Whitcroft 已提交
1308 1309 1310 1311 1312 1313 1314
{
	int ret = -EINVAL;

	/* Only take pages on the LRU. */
	if (!PageLRU(page))
		return ret;

M
Minchan Kim 已提交
1315 1316
	/* Compaction should not handle unevictable pages but CMA can do so */
	if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
L
Lee Schermerhorn 已提交
1317 1318
		return ret;

A
Andy Whitcroft 已提交
1319
	ret = -EBUSY;
K
KAMEZAWA Hiroyuki 已提交
1320

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
	/*
	 * To minimise LRU disruption, the caller can indicate that it only
	 * wants to isolate pages it will be able to operate on without
	 * blocking - clean pages for the most part.
	 *
	 * ISOLATE_CLEAN means that only clean pages should be isolated. This
	 * is used by reclaim when it is cannot write to backing storage
	 *
	 * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
	 * that it is possible to migrate without blocking
	 */
	if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
		/* All the caller can do on PageWriteback is block */
		if (PageWriteback(page))
			return ret;

		if (PageDirty(page)) {
			struct address_space *mapping;

			/* ISOLATE_CLEAN means only clean pages */
			if (mode & ISOLATE_CLEAN)
				return ret;

			/*
			 * Only pages without mappings or that have a
			 * ->migratepage callback are possible to migrate
			 * without blocking
			 */
			mapping = page_mapping(page);
			if (mapping && !mapping->a_ops->migratepage)
				return ret;
		}
	}
1354

1355 1356 1357
	if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
		return ret;

A
Andy Whitcroft 已提交
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
	if (likely(get_page_unless_zero(page))) {
		/*
		 * Be careful not to clear PageLRU until after we're
		 * sure the page is not being freed elsewhere -- the
		 * page release code relies on it.
		 */
		ClearPageLRU(page);
		ret = 0;
	}

	return ret;
}

1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393

/*
 * Update LRU sizes after isolating pages. The LRU size updates must
 * be complete before mem_cgroup_update_lru_size due to a santity check.
 */
static __always_inline void update_lru_sizes(struct lruvec *lruvec,
			enum lru_list lru, unsigned long *nr_zone_taken,
			unsigned long nr_taken)
{
	int zid;

	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		if (!nr_zone_taken[zid])
			continue;

		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
	}

#ifdef CONFIG_MEMCG
	mem_cgroup_update_lru_size(lruvec, lru, -nr_taken);
#endif
}

L
Linus Torvalds 已提交
1394
/*
1395
 * zone_lru_lock is heavily contended.  Some of the functions that
L
Linus Torvalds 已提交
1396 1397 1398 1399 1400 1401 1402 1403 1404
 * shrink the lists perform better by taking out a batch of pages
 * and working on them outside the LRU lock.
 *
 * For pagecache intensive workloads, this function is the hottest
 * spot in the kernel (apart from copy_*_user functions).
 *
 * Appropriate locks must be held before calling this function.
 *
 * @nr_to_scan:	The number of pages to look through on the list.
1405
 * @lruvec:	The LRU vector to pull pages from.
L
Linus Torvalds 已提交
1406
 * @dst:	The temp list to put pages on to.
H
Hugh Dickins 已提交
1407
 * @nr_scanned:	The number of pages that were scanned.
1408
 * @sc:		The scan_control struct for this reclaim session
A
Andy Whitcroft 已提交
1409
 * @mode:	One of the LRU isolation modes
1410
 * @lru:	LRU list id for isolating
L
Linus Torvalds 已提交
1411 1412 1413
 *
 * returns how many pages were moved onto *@dst.
 */
1414
static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1415
		struct lruvec *lruvec, struct list_head *dst,
1416
		unsigned long *nr_scanned, struct scan_control *sc,
1417
		isolate_mode_t mode, enum lru_list lru)
L
Linus Torvalds 已提交
1418
{
H
Hugh Dickins 已提交
1419
	struct list_head *src = &lruvec->lists[lru];
1420
	unsigned long nr_taken = 0;
M
Mel Gorman 已提交
1421
	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1422
	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
M
Mel Gorman 已提交
1423
	unsigned long scan, nr_pages;
1424
	LIST_HEAD(pages_skipped);
L
Linus Torvalds 已提交
1425

1426
	for (scan = 0; scan < nr_to_scan && nr_taken < nr_to_scan &&
1427
					!list_empty(src);) {
A
Andy Whitcroft 已提交
1428 1429
		struct page *page;

L
Linus Torvalds 已提交
1430 1431 1432
		page = lru_to_page(src);
		prefetchw_prev_lru_page(page, src, flags);

1433
		VM_BUG_ON_PAGE(!PageLRU(page), page);
N
Nick Piggin 已提交
1434

1435 1436
		if (page_zonenum(page) > sc->reclaim_idx) {
			list_move(&page->lru, &pages_skipped);
1437
			nr_skipped[page_zonenum(page)]++;
1438 1439 1440
			continue;
		}

1441 1442 1443 1444 1445 1446
		/*
		 * Account for scanned and skipped separetly to avoid the pgdat
		 * being prematurely marked unreclaimable by pgdat_reclaimable.
		 */
		scan++;

1447
		switch (__isolate_lru_page(page, mode)) {
A
Andy Whitcroft 已提交
1448
		case 0:
M
Mel Gorman 已提交
1449 1450 1451
			nr_pages = hpage_nr_pages(page);
			nr_taken += nr_pages;
			nr_zone_taken[page_zonenum(page)] += nr_pages;
A
Andy Whitcroft 已提交
1452 1453 1454 1455 1456 1457 1458
			list_move(&page->lru, dst);
			break;

		case -EBUSY:
			/* else it is being freed elsewhere */
			list_move(&page->lru, src);
			continue;
1459

A
Andy Whitcroft 已提交
1460 1461 1462
		default:
			BUG();
		}
L
Linus Torvalds 已提交
1463 1464
	}

1465 1466 1467 1468 1469 1470 1471
	/*
	 * Splice any skipped pages to the start of the LRU list. Note that
	 * this disrupts the LRU order when reclaiming for lower zones but
	 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
	 * scanning would soon rescan the same pages to skip and put the
	 * system at risk of premature OOM.
	 */
1472 1473
	if (!list_empty(&pages_skipped)) {
		int zid;
1474
		unsigned long total_skipped = 0;
1475 1476 1477 1478 1479 1480

		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			if (!nr_skipped[zid])
				continue;

			__count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1481
			total_skipped += nr_skipped[zid];
1482
		}
1483 1484 1485 1486 1487 1488 1489 1490 1491

		/*
		 * Account skipped pages as a partial scan as the pgdat may be
		 * close to unreclaimable. If the LRU list is empty, account
		 * skipped pages as a full scan.
		 */
		scan += list_empty(src) ? total_skipped : total_skipped >> 2;

		list_splice(&pages_skipped, src);
1492
	}
H
Hugh Dickins 已提交
1493
	*nr_scanned = scan;
1494
	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, scan,
H
Hugh Dickins 已提交
1495
				    nr_taken, mode, is_file_lru(lru));
1496
	update_lru_sizes(lruvec, lru, nr_zone_taken, nr_taken);
L
Linus Torvalds 已提交
1497 1498 1499
	return nr_taken;
}

1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
/**
 * isolate_lru_page - tries to isolate a page from its LRU list
 * @page: page to isolate from its LRU list
 *
 * Isolates a @page from an LRU list, clears PageLRU and adjusts the
 * vmstat statistic corresponding to whatever LRU list the page was on.
 *
 * Returns 0 if the page was removed from an LRU list.
 * Returns -EBUSY if the page was not on an LRU list.
 *
 * The returned page will have PageLRU() cleared.  If it was found on
L
Lee Schermerhorn 已提交
1511 1512 1513
 * the active list, it will have PageActive set.  If it was found on
 * the unevictable list, it will have the PageUnevictable bit set. That flag
 * may need to be cleared by the caller before letting the page go.
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
 *
 * The vmstat statistic corresponding to the list on which the page was
 * found will be decremented.
 *
 * Restrictions:
 * (1) Must be called with an elevated refcount on the page. This is a
 *     fundamentnal difference from isolate_lru_pages (which is called
 *     without a stable reference).
 * (2) the lru_lock must not be held.
 * (3) interrupts must be enabled.
 */
int isolate_lru_page(struct page *page)
{
	int ret = -EBUSY;

1529
	VM_BUG_ON_PAGE(!page_count(page), page);
1530
	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1531

1532 1533
	if (PageLRU(page)) {
		struct zone *zone = page_zone(page);
1534
		struct lruvec *lruvec;
1535

1536
		spin_lock_irq(zone_lru_lock(zone));
M
Mel Gorman 已提交
1537
		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
1538
		if (PageLRU(page)) {
L
Lee Schermerhorn 已提交
1539
			int lru = page_lru(page);
1540
			get_page(page);
1541
			ClearPageLRU(page);
1542 1543
			del_page_from_lru_list(page, lruvec, lru);
			ret = 0;
1544
		}
1545
		spin_unlock_irq(zone_lru_lock(zone));
1546 1547 1548 1549
	}
	return ret;
}

1550
/*
F
Fengguang Wu 已提交
1551 1552 1553 1554 1555
 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
 * then get resheduled. When there are massive number of tasks doing page
 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
 * the LRU list will go small and be scanned faster than necessary, leading to
 * unnecessary swapping, thrashing and OOM.
1556
 */
M
Mel Gorman 已提交
1557
static int too_many_isolated(struct pglist_data *pgdat, int file,
1558 1559 1560 1561 1562 1563 1564
		struct scan_control *sc)
{
	unsigned long inactive, isolated;

	if (current_is_kswapd())
		return 0;

1565
	if (!sane_reclaim(sc))
1566 1567 1568
		return 0;

	if (file) {
M
Mel Gorman 已提交
1569 1570
		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
1571
	} else {
M
Mel Gorman 已提交
1572 1573
		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
1574 1575
	}

1576 1577 1578 1579 1580
	/*
	 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
	 * won't get blocked by normal direct-reclaimers, forming a circular
	 * deadlock.
	 */
1581
	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
1582 1583
		inactive >>= 3;

1584 1585 1586
	return isolated > inactive;
}

1587
static noinline_for_stack void
H
Hugh Dickins 已提交
1588
putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1589
{
1590
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
M
Mel Gorman 已提交
1591
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1592
	LIST_HEAD(pages_to_free);
1593 1594 1595 1596 1597

	/*
	 * Put back any unfreeable pages.
	 */
	while (!list_empty(page_list)) {
1598
		struct page *page = lru_to_page(page_list);
1599
		int lru;
1600

1601
		VM_BUG_ON_PAGE(PageLRU(page), page);
1602
		list_del(&page->lru);
1603
		if (unlikely(!page_evictable(page))) {
M
Mel Gorman 已提交
1604
			spin_unlock_irq(&pgdat->lru_lock);
1605
			putback_lru_page(page);
M
Mel Gorman 已提交
1606
			spin_lock_irq(&pgdat->lru_lock);
1607 1608
			continue;
		}
1609

M
Mel Gorman 已提交
1610
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
1611

1612
		SetPageLRU(page);
1613
		lru = page_lru(page);
1614 1615
		add_page_to_lru_list(page, lruvec, lru);

1616 1617
		if (is_active_lru(lru)) {
			int file = is_file_lru(lru);
1618 1619
			int numpages = hpage_nr_pages(page);
			reclaim_stat->recent_rotated[file] += numpages;
1620
		}
1621 1622 1623
		if (put_page_testzero(page)) {
			__ClearPageLRU(page);
			__ClearPageActive(page);
1624
			del_page_from_lru_list(page, lruvec, lru);
1625 1626

			if (unlikely(PageCompound(page))) {
M
Mel Gorman 已提交
1627
				spin_unlock_irq(&pgdat->lru_lock);
1628
				mem_cgroup_uncharge(page);
1629
				(*get_compound_page_dtor(page))(page);
M
Mel Gorman 已提交
1630
				spin_lock_irq(&pgdat->lru_lock);
1631 1632
			} else
				list_add(&page->lru, &pages_to_free);
1633 1634 1635
		}
	}

1636 1637 1638 1639
	/*
	 * To save our caller's stack, now use input list for pages to free.
	 */
	list_splice(&pages_to_free, page_list);
1640 1641
}

1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
/*
 * If a kernel thread (such as nfsd for loop-back mounts) services
 * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
 * In that case we should only throttle if the backing device it is
 * writing to is congested.  In other cases it is safe to throttle.
 */
static int current_may_throttle(void)
{
	return !(current->flags & PF_LESS_THROTTLE) ||
		current->backing_dev_info == NULL ||
		bdi_write_congested(current->backing_dev_info);
}

L
Linus Torvalds 已提交
1655
/*
1656
 * shrink_inactive_list() is a helper for shrink_node().  It returns the number
A
Andrew Morton 已提交
1657
 * of reclaimed pages
L
Linus Torvalds 已提交
1658
 */
1659
static noinline_for_stack unsigned long
1660
shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1661
		     struct scan_control *sc, enum lru_list lru)
L
Linus Torvalds 已提交
1662 1663
{
	LIST_HEAD(page_list);
1664
	unsigned long nr_scanned;
1665
	unsigned long nr_reclaimed = 0;
1666
	unsigned long nr_taken;
1667 1668
	unsigned long nr_dirty = 0;
	unsigned long nr_congested = 0;
1669
	unsigned long nr_unqueued_dirty = 0;
1670
	unsigned long nr_writeback = 0;
1671
	unsigned long nr_immediate = 0;
1672
	isolate_mode_t isolate_mode = 0;
1673
	int file = is_file_lru(lru);
M
Mel Gorman 已提交
1674
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1675
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1676

M
Mel Gorman 已提交
1677
	while (unlikely(too_many_isolated(pgdat, file, sc))) {
1678
		congestion_wait(BLK_RW_ASYNC, HZ/10);
1679 1680 1681 1682 1683 1684

		/* We are about to die and free our memory. Return now. */
		if (fatal_signal_pending(current))
			return SWAP_CLUSTER_MAX;
	}

L
Linus Torvalds 已提交
1685
	lru_add_drain();
1686 1687

	if (!sc->may_unmap)
1688
		isolate_mode |= ISOLATE_UNMAPPED;
1689
	if (!sc->may_writepage)
1690
		isolate_mode |= ISOLATE_CLEAN;
1691

M
Mel Gorman 已提交
1692
	spin_lock_irq(&pgdat->lru_lock);
1693

1694 1695
	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
				     &nr_scanned, sc, isolate_mode, lru);
1696

M
Mel Gorman 已提交
1697
	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1698
	reclaim_stat->recent_scanned[file] += nr_taken;
1699

1700
	if (global_reclaim(sc)) {
M
Mel Gorman 已提交
1701
		__mod_node_page_state(pgdat, NR_PAGES_SCANNED, nr_scanned);
1702
		if (current_is_kswapd())
M
Mel Gorman 已提交
1703
			__count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1704
		else
M
Mel Gorman 已提交
1705
			__count_vm_events(PGSCAN_DIRECT, nr_scanned);
1706
	}
M
Mel Gorman 已提交
1707
	spin_unlock_irq(&pgdat->lru_lock);
1708

1709
	if (nr_taken == 0)
1710
		return 0;
A
Andy Whitcroft 已提交
1711

M
Mel Gorman 已提交
1712
	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, TTU_UNMAP,
1713 1714 1715
				&nr_dirty, &nr_unqueued_dirty, &nr_congested,
				&nr_writeback, &nr_immediate,
				false);
1716

M
Mel Gorman 已提交
1717
	spin_lock_irq(&pgdat->lru_lock);
1718

Y
Ying Han 已提交
1719 1720
	if (global_reclaim(sc)) {
		if (current_is_kswapd())
M
Mel Gorman 已提交
1721
			__count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
Y
Ying Han 已提交
1722
		else
M
Mel Gorman 已提交
1723
			__count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
Y
Ying Han 已提交
1724
	}
N
Nick Piggin 已提交
1725

1726
	putback_inactive_pages(lruvec, &page_list);
1727

M
Mel Gorman 已提交
1728
	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1729

M
Mel Gorman 已提交
1730
	spin_unlock_irq(&pgdat->lru_lock);
1731

1732
	mem_cgroup_uncharge_list(&page_list);
1733
	free_hot_cold_page_list(&page_list, true);
1734

1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
	/*
	 * If reclaim is isolating dirty pages under writeback, it implies
	 * that the long-lived page allocation rate is exceeding the page
	 * laundering rate. Either the global limits are not being effective
	 * at throttling processes due to the page distribution throughout
	 * zones or there is heavy usage of a slow backing device. The
	 * only option is to throttle from reclaim context which is not ideal
	 * as there is no guarantee the dirtying process is throttled in the
	 * same way balance_dirty_pages() manages.
	 *
1745 1746 1747
	 * Once a zone is flagged ZONE_WRITEBACK, kswapd will count the number
	 * of pages under pages flagged for immediate reclaim and stall if any
	 * are encountered in the nr_immediate check below.
1748
	 */
1749
	if (nr_writeback && nr_writeback == nr_taken)
M
Mel Gorman 已提交
1750
		set_bit(PGDAT_WRITEBACK, &pgdat->flags);
1751

1752
	/*
1753 1754
	 * Legacy memcg will stall in page writeback so avoid forcibly
	 * stalling here.
1755
	 */
1756
	if (sane_reclaim(sc)) {
1757 1758 1759 1760 1761
		/*
		 * Tag a zone as congested if all the dirty pages scanned were
		 * backed by a congested BDI and wait_iff_congested will stall.
		 */
		if (nr_dirty && nr_dirty == nr_congested)
M
Mel Gorman 已提交
1762
			set_bit(PGDAT_CONGESTED, &pgdat->flags);
1763

1764 1765 1766
		/*
		 * If dirty pages are scanned that are not queued for IO, it
		 * implies that flushers are not keeping up. In this case, flag
M
Mel Gorman 已提交
1767
		 * the pgdat PGDAT_DIRTY and kswapd will start writing pages from
J
Johannes Weiner 已提交
1768
		 * reclaim context.
1769 1770
		 */
		if (nr_unqueued_dirty == nr_taken)
M
Mel Gorman 已提交
1771
			set_bit(PGDAT_DIRTY, &pgdat->flags);
1772 1773

		/*
1774 1775 1776
		 * If kswapd scans pages marked marked for immediate
		 * reclaim and under writeback (nr_immediate), it implies
		 * that pages are cycling through the LRU faster than
1777 1778
		 * they are written so also forcibly stall.
		 */
1779
		if (nr_immediate && current_may_throttle())
1780
			congestion_wait(BLK_RW_ASYNC, HZ/10);
1781
	}
1782

1783 1784 1785 1786 1787
	/*
	 * Stall direct reclaim for IO completions if underlying BDIs or zone
	 * is congested. Allow kswapd to continue until it starts encountering
	 * unqueued dirty pages or cycling through the LRU too quickly.
	 */
1788 1789
	if (!sc->hibernation_mode && !current_is_kswapd() &&
	    current_may_throttle())
M
Mel Gorman 已提交
1790
		wait_iff_congested(pgdat, BLK_RW_ASYNC, HZ/10);
1791

M
Mel Gorman 已提交
1792 1793
	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
			nr_scanned, nr_reclaimed,
1794
			sc->priority, file);
1795
	return nr_reclaimed;
L
Linus Torvalds 已提交
1796 1797 1798 1799 1800 1801 1802 1803 1804
}

/*
 * This moves pages from the active list to the inactive list.
 *
 * We move them the other way if the page is referenced by one or more
 * processes, from rmap.
 *
 * If the pages are mostly unmapped, the processing is fast and it is
1805
 * appropriate to hold zone_lru_lock across the whole operation.  But if
L
Linus Torvalds 已提交
1806
 * the pages are mapped, the processing is slow (page_referenced()) so we
1807
 * should drop zone_lru_lock around each page.  It's impossible to balance
L
Linus Torvalds 已提交
1808 1809 1810 1811
 * this, so instead we remove the pages from the LRU while processing them.
 * It is safe to rely on PG_active against the non-LRU pages in here because
 * nobody will play with that bit on a non-LRU page.
 *
1812
 * The downside is that we have to touch page->_refcount against each page.
L
Linus Torvalds 已提交
1813 1814
 * But we had to alter page->flags anyway.
 */
1815

1816
static void move_active_pages_to_lru(struct lruvec *lruvec,
1817
				     struct list_head *list,
1818
				     struct list_head *pages_to_free,
1819 1820
				     enum lru_list lru)
{
M
Mel Gorman 已提交
1821
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1822 1823
	unsigned long pgmoved = 0;
	struct page *page;
1824
	int nr_pages;
1825 1826 1827

	while (!list_empty(list)) {
		page = lru_to_page(list);
M
Mel Gorman 已提交
1828
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
1829

1830
		VM_BUG_ON_PAGE(PageLRU(page), page);
1831 1832
		SetPageLRU(page);

1833
		nr_pages = hpage_nr_pages(page);
M
Mel Gorman 已提交
1834
		update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
1835
		list_move(&page->lru, &lruvec->lists[lru]);
1836
		pgmoved += nr_pages;
1837

1838 1839 1840
		if (put_page_testzero(page)) {
			__ClearPageLRU(page);
			__ClearPageActive(page);
1841
			del_page_from_lru_list(page, lruvec, lru);
1842 1843

			if (unlikely(PageCompound(page))) {
M
Mel Gorman 已提交
1844
				spin_unlock_irq(&pgdat->lru_lock);
1845
				mem_cgroup_uncharge(page);
1846
				(*get_compound_page_dtor(page))(page);
M
Mel Gorman 已提交
1847
				spin_lock_irq(&pgdat->lru_lock);
1848 1849
			} else
				list_add(&page->lru, pages_to_free);
1850 1851
		}
	}
1852

1853 1854 1855
	if (!is_active_lru(lru))
		__count_vm_events(PGDEACTIVATE, pgmoved);
}
1856

H
Hugh Dickins 已提交
1857
static void shrink_active_list(unsigned long nr_to_scan,
1858
			       struct lruvec *lruvec,
1859
			       struct scan_control *sc,
1860
			       enum lru_list lru)
L
Linus Torvalds 已提交
1861
{
1862
	unsigned long nr_taken;
H
Hugh Dickins 已提交
1863
	unsigned long nr_scanned;
1864
	unsigned long vm_flags;
L
Linus Torvalds 已提交
1865
	LIST_HEAD(l_hold);	/* The pages which were snipped off */
1866
	LIST_HEAD(l_active);
1867
	LIST_HEAD(l_inactive);
L
Linus Torvalds 已提交
1868
	struct page *page;
1869
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1870
	unsigned long nr_rotated = 0;
1871
	isolate_mode_t isolate_mode = 0;
1872
	int file = is_file_lru(lru);
M
Mel Gorman 已提交
1873
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
L
Linus Torvalds 已提交
1874 1875

	lru_add_drain();
1876 1877

	if (!sc->may_unmap)
1878
		isolate_mode |= ISOLATE_UNMAPPED;
1879
	if (!sc->may_writepage)
1880
		isolate_mode |= ISOLATE_CLEAN;
1881

M
Mel Gorman 已提交
1882
	spin_lock_irq(&pgdat->lru_lock);
1883

1884 1885
	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
				     &nr_scanned, sc, isolate_mode, lru);
1886

M
Mel Gorman 已提交
1887
	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1888
	reclaim_stat->recent_scanned[file] += nr_taken;
1889

1890
	if (global_reclaim(sc))
M
Mel Gorman 已提交
1891 1892
		__mod_node_page_state(pgdat, NR_PAGES_SCANNED, nr_scanned);
	__count_vm_events(PGREFILL, nr_scanned);
1893

M
Mel Gorman 已提交
1894
	spin_unlock_irq(&pgdat->lru_lock);
L
Linus Torvalds 已提交
1895 1896 1897 1898 1899

	while (!list_empty(&l_hold)) {
		cond_resched();
		page = lru_to_page(&l_hold);
		list_del(&page->lru);
1900

1901
		if (unlikely(!page_evictable(page))) {
L
Lee Schermerhorn 已提交
1902 1903 1904 1905
			putback_lru_page(page);
			continue;
		}

1906 1907 1908 1909 1910 1911 1912 1913
		if (unlikely(buffer_heads_over_limit)) {
			if (page_has_private(page) && trylock_page(page)) {
				if (page_has_private(page))
					try_to_release_page(page, 0);
				unlock_page(page);
			}
		}

1914 1915
		if (page_referenced(page, 0, sc->target_mem_cgroup,
				    &vm_flags)) {
1916
			nr_rotated += hpage_nr_pages(page);
1917 1918 1919 1920 1921 1922 1923 1924 1925
			/*
			 * Identify referenced, file-backed active pages and
			 * give them one more trip around the active list. So
			 * that executable code get better chances to stay in
			 * memory under moderate memory pressure.  Anon pages
			 * are not likely to be evicted by use-once streaming
			 * IO, plus JVM can create lots of anon VM_EXEC pages,
			 * so we ignore them here.
			 */
1926
			if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
1927 1928 1929 1930
				list_add(&page->lru, &l_active);
				continue;
			}
		}
1931

1932
		ClearPageActive(page);	/* we are de-activating */
L
Linus Torvalds 已提交
1933 1934 1935
		list_add(&page->lru, &l_inactive);
	}

1936
	/*
1937
	 * Move pages back to the lru list.
1938
	 */
M
Mel Gorman 已提交
1939
	spin_lock_irq(&pgdat->lru_lock);
1940
	/*
1941 1942 1943
	 * Count referenced pages from currently used mappings as rotated,
	 * even though only some of them are actually re-activated.  This
	 * helps balance scan pressure between file and anonymous pages in
1944
	 * get_scan_count.
1945
	 */
1946
	reclaim_stat->recent_rotated[file] += nr_rotated;
1947

1948 1949
	move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
	move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
M
Mel Gorman 已提交
1950 1951
	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
	spin_unlock_irq(&pgdat->lru_lock);
1952

1953
	mem_cgroup_uncharge_list(&l_hold);
1954
	free_hot_cold_page_list(&l_hold, true);
L
Linus Torvalds 已提交
1955 1956
}

1957 1958 1959
/*
 * The inactive anon list should be small enough that the VM never has
 * to do too much work.
1960
 *
1961 1962 1963
 * The inactive file list should be small enough to leave most memory
 * to the established workingset on the scan-resistant active list,
 * but large enough to avoid thrashing the aggregate readahead window.
1964
 *
1965 1966
 * Both inactive lists should also be large enough that each inactive
 * page has a chance to be referenced again before it is reclaimed.
1967
 *
1968 1969 1970
 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
 * on this LRU, maintained by the pageout code. A zone->inactive_ratio
 * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
1971
 *
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
 * total     target    max
 * memory    ratio     inactive
 * -------------------------------------
 *   10MB       1         5MB
 *  100MB       1        50MB
 *    1GB       3       250MB
 *   10GB      10       0.9GB
 *  100GB      31         3GB
 *    1TB     101        10GB
 *   10TB     320        32GB
1982
 */
1983 1984
static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
						struct scan_control *sc)
1985
{
1986
	unsigned long inactive_ratio;
1987 1988
	unsigned long inactive;
	unsigned long active;
1989
	unsigned long gb;
1990 1991
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
	int zid;
1992

1993 1994 1995 1996 1997 1998
	/*
	 * If we don't have swap space, anonymous page deactivation
	 * is pointless.
	 */
	if (!file && !total_swap_pages)
		return false;
1999

2000 2001
	inactive = lruvec_lru_size(lruvec, file * LRU_FILE);
	active = lruvec_lru_size(lruvec, file * LRU_FILE + LRU_ACTIVE);
2002

2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
	/*
	 * For zone-constrained allocations, it is necessary to check if
	 * deactivations are required for lowmem to be reclaimed. This
	 * calculates the inactive/active pages available in eligible zones.
	 */
	for (zid = sc->reclaim_idx + 1; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &pgdat->node_zones[zid];
		unsigned long inactive_zone, active_zone;

		if (!populated_zone(zone))
			continue;

		inactive_zone = zone_page_state(zone,
				NR_ZONE_LRU_BASE + (file * LRU_FILE));
		active_zone = zone_page_state(zone,
				NR_ZONE_LRU_BASE + (file * LRU_FILE) + LRU_ACTIVE);

		inactive -= min(inactive, inactive_zone);
		active -= min(active, active_zone);
	}

2024 2025 2026
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
2027
	else
2028 2029 2030
		inactive_ratio = 1;

	return inactive * inactive_ratio < active;
2031 2032
}

2033
static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2034
				 struct lruvec *lruvec, struct scan_control *sc)
2035
{
2036
	if (is_active_lru(lru)) {
2037
		if (inactive_list_is_low(lruvec, is_file_lru(lru), sc))
2038
			shrink_active_list(nr_to_scan, lruvec, sc, lru);
2039 2040 2041
		return 0;
	}

2042
	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2043 2044
}

2045 2046 2047 2048 2049 2050 2051
enum scan_balance {
	SCAN_EQUAL,
	SCAN_FRACT,
	SCAN_ANON,
	SCAN_FILE,
};

2052 2053 2054 2055 2056 2057
/*
 * Determine how aggressively the anon and file LRU lists should be
 * scanned.  The relative value of each set of LRU lists is determined
 * by looking at the fraction of the pages scanned we did rotate back
 * onto the active list instead of evict.
 *
W
Wanpeng Li 已提交
2058 2059
 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2060
 */
2061
static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
2062 2063
			   struct scan_control *sc, unsigned long *nr,
			   unsigned long *lru_pages)
2064
{
2065
	int swappiness = mem_cgroup_swappiness(memcg);
2066 2067 2068
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
	u64 fraction[2];
	u64 denominator = 0;	/* gcc */
M
Mel Gorman 已提交
2069
	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2070
	unsigned long anon_prio, file_prio;
2071
	enum scan_balance scan_balance;
2072
	unsigned long anon, file;
2073
	bool force_scan = false;
2074
	unsigned long ap, fp;
H
Hugh Dickins 已提交
2075
	enum lru_list lru;
2076 2077
	bool some_scanned;
	int pass;
2078

2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
	/*
	 * If the zone or memcg is small, nr[l] can be 0.  This
	 * results in no scanning on this priority and a potential
	 * priority drop.  Global direct reclaim can go to the next
	 * zone and tends to have no problems. Global kswapd is for
	 * zone balancing and it needs to scan a minimum amount. When
	 * reclaiming for a memcg, a priority drop can cause high
	 * latencies, so it's better to scan a minimum amount there as
	 * well.
	 */
2089
	if (current_is_kswapd()) {
M
Mel Gorman 已提交
2090
		if (!pgdat_reclaimable(pgdat))
2091
			force_scan = true;
2092
		if (!mem_cgroup_online(memcg))
2093 2094
			force_scan = true;
	}
2095
	if (!global_reclaim(sc))
2096
		force_scan = true;
2097 2098

	/* If we have no swap space, do not bother scanning anon pages. */
2099
	if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
2100
		scan_balance = SCAN_FILE;
2101 2102
		goto out;
	}
2103

2104 2105 2106 2107 2108 2109 2110
	/*
	 * Global reclaim will swap to prevent OOM even with no
	 * swappiness, but memcg users want to use this knob to
	 * disable swapping for individual groups completely when
	 * using the memory controller's swap limit feature would be
	 * too expensive.
	 */
2111
	if (!global_reclaim(sc) && !swappiness) {
2112
		scan_balance = SCAN_FILE;
2113 2114 2115 2116 2117 2118 2119 2120
		goto out;
	}

	/*
	 * Do not apply any pressure balancing cleverness when the
	 * system is close to OOM, scan both anon and file equally
	 * (unless the swappiness setting disagrees with swapping).
	 */
2121
	if (!sc->priority && swappiness) {
2122
		scan_balance = SCAN_EQUAL;
2123 2124 2125
		goto out;
	}

2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
	/*
	 * Prevent the reclaimer from falling into the cache trap: as
	 * cache pages start out inactive, every cache fault will tip
	 * the scan balance towards the file LRU.  And as the file LRU
	 * shrinks, so does the window for rotation from references.
	 * This means we have a runaway feedback loop where a tiny
	 * thrashing file LRU becomes infinitely more attractive than
	 * anon pages.  Try to detect this based on file LRU size.
	 */
	if (global_reclaim(sc)) {
M
Mel Gorman 已提交
2136 2137 2138 2139
		unsigned long pgdatfile;
		unsigned long pgdatfree;
		int z;
		unsigned long total_high_wmark = 0;
2140

M
Mel Gorman 已提交
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
		pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
		pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
			   node_page_state(pgdat, NR_INACTIVE_FILE);

		for (z = 0; z < MAX_NR_ZONES; z++) {
			struct zone *zone = &pgdat->node_zones[z];
			if (!populated_zone(zone))
				continue;

			total_high_wmark += high_wmark_pages(zone);
		}
2152

M
Mel Gorman 已提交
2153
		if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2154 2155 2156 2157 2158
			scan_balance = SCAN_ANON;
			goto out;
		}
	}

2159
	/*
2160 2161 2162 2163 2164 2165 2166
	 * If there is enough inactive page cache, i.e. if the size of the
	 * inactive list is greater than that of the active list *and* the
	 * inactive list actually has some pages to scan on this priority, we
	 * do not reclaim anything from the anonymous working set right now.
	 * Without the second condition we could end up never scanning an
	 * lruvec even if it has plenty of old anonymous pages unless the
	 * system is under heavy pressure.
2167
	 */
2168
	if (!inactive_list_is_low(lruvec, true, sc) &&
2169
	    lruvec_lru_size(lruvec, LRU_INACTIVE_FILE) >> sc->priority) {
2170
		scan_balance = SCAN_FILE;
2171 2172 2173
		goto out;
	}

2174 2175
	scan_balance = SCAN_FRACT;

2176 2177 2178 2179
	/*
	 * With swappiness at 100, anonymous and file have the same priority.
	 * This scanning priority is essentially the inverse of IO cost.
	 */
2180
	anon_prio = swappiness;
H
Hugh Dickins 已提交
2181
	file_prio = 200 - anon_prio;
2182

2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
	/*
	 * OK, so we have swap space and a fair amount of page cache
	 * pages.  We use the recently rotated / recently scanned
	 * ratios to determine how valuable each cache is.
	 *
	 * Because workloads change over time (and to avoid overflow)
	 * we keep these statistics as a floating average, which ends
	 * up weighing recent references more than old ones.
	 *
	 * anon in [0], file in [1]
	 */
2194

2195 2196 2197 2198
	anon  = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON) +
		lruvec_lru_size(lruvec, LRU_INACTIVE_ANON);
	file  = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE) +
		lruvec_lru_size(lruvec, LRU_INACTIVE_FILE);
2199

M
Mel Gorman 已提交
2200
	spin_lock_irq(&pgdat->lru_lock);
2201 2202 2203
	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
		reclaim_stat->recent_scanned[0] /= 2;
		reclaim_stat->recent_rotated[0] /= 2;
2204 2205
	}

2206 2207 2208
	if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
		reclaim_stat->recent_scanned[1] /= 2;
		reclaim_stat->recent_rotated[1] /= 2;
2209 2210 2211
	}

	/*
2212 2213 2214
	 * The amount of pressure on anon vs file pages is inversely
	 * proportional to the fraction of recently scanned pages on
	 * each list that were recently referenced and in active use.
2215
	 */
2216
	ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2217
	ap /= reclaim_stat->recent_rotated[0] + 1;
2218

2219
	fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2220
	fp /= reclaim_stat->recent_rotated[1] + 1;
M
Mel Gorman 已提交
2221
	spin_unlock_irq(&pgdat->lru_lock);
2222

2223 2224 2225 2226
	fraction[0] = ap;
	fraction[1] = fp;
	denominator = ap + fp + 1;
out:
2227 2228 2229
	some_scanned = false;
	/* Only use force_scan on second pass. */
	for (pass = 0; !some_scanned && pass < 2; pass++) {
2230
		*lru_pages = 0;
2231 2232 2233 2234
		for_each_evictable_lru(lru) {
			int file = is_file_lru(lru);
			unsigned long size;
			unsigned long scan;
2235

2236
			size = lruvec_lru_size(lruvec, lru);
2237
			scan = size >> sc->priority;
2238

2239 2240
			if (!scan && pass && force_scan)
				scan = min(size, SWAP_CLUSTER_MAX);
2241

2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
			switch (scan_balance) {
			case SCAN_EQUAL:
				/* Scan lists relative to size */
				break;
			case SCAN_FRACT:
				/*
				 * Scan types proportional to swappiness and
				 * their relative recent reclaim efficiency.
				 */
				scan = div64_u64(scan * fraction[file],
							denominator);
				break;
			case SCAN_FILE:
			case SCAN_ANON:
				/* Scan one type exclusively */
2257 2258
				if ((scan_balance == SCAN_FILE) != file) {
					size = 0;
2259
					scan = 0;
2260
				}
2261 2262 2263 2264 2265
				break;
			default:
				/* Look ma, no brain */
				BUG();
			}
2266 2267

			*lru_pages += size;
2268
			nr[lru] = scan;
2269

2270
			/*
2271 2272
			 * Skip the second pass and don't force_scan,
			 * if we found something to scan.
2273
			 */
2274
			some_scanned |= !!scan;
2275
		}
2276
	}
2277
}
2278

2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
static void init_tlb_ubc(void)
{
	/*
	 * This deliberately does not clear the cpumask as it's expensive
	 * and unnecessary. If there happens to be data in there then the
	 * first SWAP_CLUSTER_MAX pages will send an unnecessary IPI and
	 * then will be cleared.
	 */
	current->tlb_ubc.flush_required = false;
}
#else
static inline void init_tlb_ubc(void)
{
}
#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */

2296
/*
2297
 * This is a basic per-node page freer.  Used by both kswapd and direct reclaim.
2298
 */
2299
static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
2300
			      struct scan_control *sc, unsigned long *lru_pages)
2301
{
2302
	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2303
	unsigned long nr[NR_LRU_LISTS];
2304
	unsigned long targets[NR_LRU_LISTS];
2305 2306 2307 2308 2309
	unsigned long nr_to_scan;
	enum lru_list lru;
	unsigned long nr_reclaimed = 0;
	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
	struct blk_plug plug;
2310
	bool scan_adjusted;
2311

2312
	get_scan_count(lruvec, memcg, sc, nr, lru_pages);
2313

2314 2315 2316
	/* Record the original scan target for proportional adjustments later */
	memcpy(targets, nr, sizeof(nr));

2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330
	/*
	 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
	 * event that can occur when there is little memory pressure e.g.
	 * multiple streaming readers/writers. Hence, we do not abort scanning
	 * when the requested number of pages are reclaimed when scanning at
	 * DEF_PRIORITY on the assumption that the fact we are direct
	 * reclaiming implies that kswapd is not keeping up and it is best to
	 * do a batch of work at once. For memcg reclaim one check is made to
	 * abort proportional reclaim if either the file or anon lru has already
	 * dropped to zero at the first pass.
	 */
	scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
			 sc->priority == DEF_PRIORITY);

2331 2332
	init_tlb_ubc();

2333 2334 2335
	blk_start_plug(&plug);
	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
					nr[LRU_INACTIVE_FILE]) {
2336 2337 2338
		unsigned long nr_anon, nr_file, percentage;
		unsigned long nr_scanned;

2339 2340 2341 2342 2343 2344 2345 2346 2347
		for_each_evictable_lru(lru) {
			if (nr[lru]) {
				nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
				nr[lru] -= nr_to_scan;

				nr_reclaimed += shrink_list(lru, nr_to_scan,
							    lruvec, sc);
			}
		}
2348 2349 2350 2351 2352 2353

		if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
			continue;

		/*
		 * For kswapd and memcg, reclaim at least the number of pages
2354
		 * requested. Ensure that the anon and file LRUs are scanned
2355 2356 2357 2358 2359 2360 2361
		 * proportionally what was requested by get_scan_count(). We
		 * stop reclaiming one LRU and reduce the amount scanning
		 * proportional to the original scan target.
		 */
		nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
		nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];

2362 2363 2364 2365 2366 2367 2368 2369 2370
		/*
		 * It's just vindictive to attack the larger once the smaller
		 * has gone to zero.  And given the way we stop scanning the
		 * smaller below, this makes sure that we only make one nudge
		 * towards proportionality once we've got nr_to_reclaim.
		 */
		if (!nr_file || !nr_anon)
			break;

2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
		if (nr_file > nr_anon) {
			unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
						targets[LRU_ACTIVE_ANON] + 1;
			lru = LRU_BASE;
			percentage = nr_anon * 100 / scan_target;
		} else {
			unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
						targets[LRU_ACTIVE_FILE] + 1;
			lru = LRU_FILE;
			percentage = nr_file * 100 / scan_target;
		}

		/* Stop scanning the smaller of the LRU */
		nr[lru] = 0;
		nr[lru + LRU_ACTIVE] = 0;

		/*
		 * Recalculate the other LRU scan count based on its original
		 * scan target and the percentage scanning already complete
		 */
		lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
		nr_scanned = targets[lru] - nr[lru];
		nr[lru] = targets[lru] * (100 - percentage) / 100;
		nr[lru] -= min(nr[lru], nr_scanned);

		lru += LRU_ACTIVE;
		nr_scanned = targets[lru] - nr[lru];
		nr[lru] = targets[lru] * (100 - percentage) / 100;
		nr[lru] -= min(nr[lru], nr_scanned);

		scan_adjusted = true;
2402 2403 2404 2405 2406 2407 2408 2409
	}
	blk_finish_plug(&plug);
	sc->nr_reclaimed += nr_reclaimed;

	/*
	 * Even if we did not try to evict anon pages at all, we want to
	 * rebalance the anon lru active/inactive ratio.
	 */
2410
	if (inactive_list_is_low(lruvec, false, sc))
2411 2412 2413 2414 2415 2416
		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
				   sc, LRU_ACTIVE_ANON);

	throttle_vm_writeout(sc->gfp_mask);
}

M
Mel Gorman 已提交
2417
/* Use reclaim/compaction for costly allocs or under memory pressure */
2418
static bool in_reclaim_compaction(struct scan_control *sc)
M
Mel Gorman 已提交
2419
{
2420
	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
M
Mel Gorman 已提交
2421
			(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2422
			 sc->priority < DEF_PRIORITY - 2))
M
Mel Gorman 已提交
2423 2424 2425 2426 2427
		return true;

	return false;
}

2428
/*
M
Mel Gorman 已提交
2429 2430 2431 2432 2433
 * Reclaim/compaction is used for high-order allocation requests. It reclaims
 * order-0 pages before compacting the zone. should_continue_reclaim() returns
 * true if more pages should be reclaimed such that when the page allocator
 * calls try_to_compact_zone() that it will have enough free pages to succeed.
 * It will give up earlier than that if there is difficulty reclaiming pages.
2434
 */
2435
static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2436 2437 2438 2439 2440 2441
					unsigned long nr_reclaimed,
					unsigned long nr_scanned,
					struct scan_control *sc)
{
	unsigned long pages_for_compaction;
	unsigned long inactive_lru_pages;
2442
	int z;
2443 2444

	/* If not in reclaim/compaction mode, stop */
2445
	if (!in_reclaim_compaction(sc))
2446 2447
		return false;

2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
	/* Consider stopping depending on scan and reclaim activity */
	if (sc->gfp_mask & __GFP_REPEAT) {
		/*
		 * For __GFP_REPEAT allocations, stop reclaiming if the
		 * full LRU list has been scanned and we are still failing
		 * to reclaim pages. This full LRU scan is potentially
		 * expensive but a __GFP_REPEAT caller really wants to succeed
		 */
		if (!nr_reclaimed && !nr_scanned)
			return false;
	} else {
		/*
		 * For non-__GFP_REPEAT allocations which can presumably
		 * fail without consequence, stop if we failed to reclaim
		 * any pages from the last SWAP_CLUSTER_MAX number of
		 * pages that were scanned. This will return to the
		 * caller faster at the risk reclaim/compaction and
		 * the resulting allocation attempt fails
		 */
		if (!nr_reclaimed)
			return false;
	}
2470 2471 2472 2473 2474 2475

	/*
	 * If we have not reclaimed enough pages for compaction and the
	 * inactive lists are large enough, continue reclaiming
	 */
	pages_for_compaction = (2UL << sc->order);
2476
	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2477
	if (get_nr_swap_pages() > 0)
2478
		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2479 2480 2481 2482 2483
	if (sc->nr_reclaimed < pages_for_compaction &&
			inactive_lru_pages > pages_for_compaction)
		return true;

	/* If compaction would go ahead or the allocation would succeed, stop */
2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496
	for (z = 0; z <= sc->reclaim_idx; z++) {
		struct zone *zone = &pgdat->node_zones[z];
		if (!populated_zone(zone))
			continue;

		switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
		case COMPACT_PARTIAL:
		case COMPACT_CONTINUE:
			return false;
		default:
			/* check next zone */
			;
		}
2497
	}
2498
	return true;
2499 2500
}

2501
static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
L
Linus Torvalds 已提交
2502
{
2503
	struct reclaim_state *reclaim_state = current->reclaim_state;
2504
	unsigned long nr_reclaimed, nr_scanned;
2505
	bool reclaimable = false;
L
Linus Torvalds 已提交
2506

2507 2508 2509
	do {
		struct mem_cgroup *root = sc->target_mem_cgroup;
		struct mem_cgroup_reclaim_cookie reclaim = {
2510
			.pgdat = pgdat,
2511 2512
			.priority = sc->priority,
		};
2513
		unsigned long node_lru_pages = 0;
2514
		struct mem_cgroup *memcg;
2515

2516 2517
		nr_reclaimed = sc->nr_reclaimed;
		nr_scanned = sc->nr_scanned;
L
Linus Torvalds 已提交
2518

2519 2520
		memcg = mem_cgroup_iter(root, NULL, &reclaim);
		do {
2521
			unsigned long lru_pages;
2522
			unsigned long reclaimed;
2523
			unsigned long scanned;
2524

2525 2526 2527 2528 2529 2530
			if (mem_cgroup_low(root, memcg)) {
				if (!sc->may_thrash)
					continue;
				mem_cgroup_events(memcg, MEMCG_LOW, 1);
			}

2531
			reclaimed = sc->nr_reclaimed;
2532
			scanned = sc->nr_scanned;
2533

2534 2535
			shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
			node_lru_pages += lru_pages;
2536

2537
			if (!global_reclaim(sc))
2538
				shrink_slab(sc->gfp_mask, pgdat->node_id,
2539 2540 2541
					    memcg, sc->nr_scanned - scanned,
					    lru_pages);

2542 2543 2544 2545 2546
			/* Record the group's reclaim efficiency */
			vmpressure(sc->gfp_mask, memcg, false,
				   sc->nr_scanned - scanned,
				   sc->nr_reclaimed - reclaimed);

2547
			/*
2548 2549
			 * Direct reclaim and kswapd have to scan all memory
			 * cgroups to fulfill the overall scan target for the
2550
			 * node.
2551 2552 2553 2554 2555
			 *
			 * Limit reclaim, on the other hand, only cares about
			 * nr_to_reclaim pages to be reclaimed and it will
			 * retry with decreasing priority if one round over the
			 * whole hierarchy is not sufficient.
2556
			 */
2557 2558
			if (!global_reclaim(sc) &&
					sc->nr_reclaimed >= sc->nr_to_reclaim) {
2559 2560 2561
				mem_cgroup_iter_break(root, memcg);
				break;
			}
2562
		} while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2563

2564 2565 2566 2567
		/*
		 * Shrink the slab caches in the same proportion that
		 * the eligible LRU pages were scanned.
		 */
2568
		if (global_reclaim(sc))
2569
			shrink_slab(sc->gfp_mask, pgdat->node_id, NULL,
2570
				    sc->nr_scanned - nr_scanned,
2571
				    node_lru_pages);
2572 2573 2574 2575

		if (reclaim_state) {
			sc->nr_reclaimed += reclaim_state->reclaimed_slab;
			reclaim_state->reclaimed_slab = 0;
2576 2577
		}

2578 2579
		/* Record the subtree's reclaim efficiency */
		vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2580 2581 2582
			   sc->nr_scanned - nr_scanned,
			   sc->nr_reclaimed - nr_reclaimed);

2583 2584 2585
		if (sc->nr_reclaimed - nr_reclaimed)
			reclaimable = true;

2586
	} while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2587
					 sc->nr_scanned - nr_scanned, sc));
2588 2589

	return reclaimable;
2590 2591
}

2592 2593 2594 2595
/*
 * Returns true if compaction should go ahead for a high-order request, or
 * the high-order allocation would succeed without compaction.
 */
2596
static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2597
{
M
Mel Gorman 已提交
2598
	unsigned long watermark;
2599 2600 2601 2602 2603 2604 2605 2606
	bool watermark_ok;

	/*
	 * Compaction takes time to run and there are potentially other
	 * callers using the pages just freed. Continue reclaiming until
	 * there is a buffer of free pages available to give compaction
	 * a reasonable chance of completing and allocating the page
	 */
2607 2608
	watermark = high_wmark_pages(zone) + (2UL << sc->order);
	watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
2609 2610 2611 2612 2613

	/*
	 * If compaction is deferred, reclaim up to a point where
	 * compaction will have a chance of success when re-enabled
	 */
2614
	if (compaction_deferred(zone, sc->order))
2615 2616
		return watermark_ok;

2617 2618 2619 2620
	/*
	 * If compaction is not ready to start and allocation is not likely
	 * to succeed without it, then keep reclaiming.
	 */
2621
	if (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx) == COMPACT_SKIPPED)
2622 2623 2624 2625 2626
		return false;

	return watermark_ok;
}

L
Linus Torvalds 已提交
2627 2628 2629 2630 2631 2632 2633 2634
/*
 * This is the direct reclaim path, for page-allocating processes.  We only
 * try to reclaim pages from zones which will satisfy the caller's allocation
 * request.
 *
 * If a zone is deemed to be full of pinned pages then just give it a light
 * scan then give up on it.
 */
M
Michal Hocko 已提交
2635
static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
L
Linus Torvalds 已提交
2636
{
2637
	struct zoneref *z;
2638
	struct zone *zone;
2639 2640
	unsigned long nr_soft_reclaimed;
	unsigned long nr_soft_scanned;
2641
	gfp_t orig_mask;
2642
	pg_data_t *last_pgdat = NULL;
2643

2644 2645 2646 2647 2648
	/*
	 * If the number of buffer_heads in the machine exceeds the maximum
	 * allowed level, force direct reclaim to scan the highmem zone as
	 * highmem pages could be pinning lowmem pages storing buffer_heads
	 */
2649
	orig_mask = sc->gfp_mask;
2650
	if (buffer_heads_over_limit) {
2651
		sc->gfp_mask |= __GFP_HIGHMEM;
2652
		sc->reclaim_idx = gfp_zone(sc->gfp_mask);
2653
	}
2654

2655
	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2656
					sc->reclaim_idx, sc->nodemask) {
2657 2658 2659 2660
		/*
		 * Take care memory controller reclaiming has small influence
		 * to global LRU.
		 */
2661
		if (global_reclaim(sc)) {
2662 2663
			if (!cpuset_zone_allowed(zone,
						 GFP_KERNEL | __GFP_HARDWALL))
2664
				continue;
2665

2666
			if (sc->priority != DEF_PRIORITY &&
M
Mel Gorman 已提交
2667
			    !pgdat_reclaimable(zone->zone_pgdat))
2668
				continue;	/* Let kswapd poll it */
2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680

			/*
			 * If we already have plenty of memory free for
			 * compaction in this zone, don't free any more.
			 * Even though compaction is invoked for any
			 * non-zero order, only frequent costly order
			 * reclamation is disruptive enough to become a
			 * noticeable problem, like transparent huge
			 * page allocations.
			 */
			if (IS_ENABLED(CONFIG_COMPACTION) &&
			    sc->order > PAGE_ALLOC_COSTLY_ORDER &&
2681
			    compaction_ready(zone, sc)) {
2682 2683
				sc->compaction_ready = true;
				continue;
2684
			}
2685

2686 2687 2688 2689 2690 2691 2692 2693 2694
			/*
			 * Shrink each node in the zonelist once. If the
			 * zonelist is ordered by zone (not the default) then a
			 * node may be shrunk multiple times but in that case
			 * the user prefers lower zones being preserved.
			 */
			if (zone->zone_pgdat == last_pgdat)
				continue;

2695 2696 2697 2698 2699 2700 2701
			/*
			 * This steals pages from memory cgroups over softlimit
			 * and returns the number of reclaimed pages and
			 * scanned pages. This works for global memory pressure
			 * and balancing, not for a memcg's limit.
			 */
			nr_soft_scanned = 0;
2702
			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
2703 2704 2705 2706
						sc->order, sc->gfp_mask,
						&nr_soft_scanned);
			sc->nr_reclaimed += nr_soft_reclaimed;
			sc->nr_scanned += nr_soft_scanned;
2707
			/* need some check for avoid more shrink_zone() */
2708
		}
2709

2710 2711 2712 2713
		/* See comment about same check for global reclaim above */
		if (zone->zone_pgdat == last_pgdat)
			continue;
		last_pgdat = zone->zone_pgdat;
2714
		shrink_node(zone->zone_pgdat, sc);
L
Linus Torvalds 已提交
2715
	}
2716

2717 2718 2719 2720 2721
	/*
	 * Restore to original mask to avoid the impact on the caller if we
	 * promoted it to __GFP_HIGHMEM.
	 */
	sc->gfp_mask = orig_mask;
L
Linus Torvalds 已提交
2722
}
2723

L
Linus Torvalds 已提交
2724 2725 2726 2727 2728 2729 2730 2731
/*
 * This is the main entry point to direct page reclaim.
 *
 * If a full scan of the inactive list fails to free enough memory then we
 * are "out of memory" and something needs to be killed.
 *
 * If the caller is !__GFP_FS then the probability of a failure is reasonably
 * high - the zone may be full of dirty or under-writeback pages, which this
2732 2733 2734 2735
 * caller can't do much about.  We kick the writeback threads and take explicit
 * naps in the hope that some of these pages can be written.  But if the
 * allocating task holds filesystem locks which prevent writeout this might not
 * work, and the allocation attempt will fail.
2736 2737 2738
 *
 * returns:	0, if no pages reclaimed
 * 		else, the number of pages reclaimed
L
Linus Torvalds 已提交
2739
 */
2740
static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2741
					  struct scan_control *sc)
L
Linus Torvalds 已提交
2742
{
2743
	int initial_priority = sc->priority;
2744
	unsigned long total_scanned = 0;
2745
	unsigned long writeback_threshold;
2746
retry:
2747 2748
	delayacct_freepages_start();

2749
	if (global_reclaim(sc))
2750
		__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
L
Linus Torvalds 已提交
2751

2752
	do {
2753 2754
		vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
				sc->priority);
2755
		sc->nr_scanned = 0;
M
Michal Hocko 已提交
2756
		shrink_zones(zonelist, sc);
2757

2758
		total_scanned += sc->nr_scanned;
2759
		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
2760 2761 2762 2763
			break;

		if (sc->compaction_ready)
			break;
L
Linus Torvalds 已提交
2764

2765 2766 2767 2768 2769 2770 2771
		/*
		 * If we're getting trouble reclaiming, start doing
		 * writepage even in laptop mode.
		 */
		if (sc->priority < DEF_PRIORITY - 2)
			sc->may_writepage = 1;

L
Linus Torvalds 已提交
2772 2773 2774 2775 2776 2777 2778
		/*
		 * Try to write back as many pages as we just scanned.  This
		 * tends to cause slow streaming writers to write data to the
		 * disk smoothly, at the dirtying rate, which is nice.   But
		 * that's undesirable in laptop mode, where we *want* lumpy
		 * writeout.  So in laptop mode, write out the whole world.
		 */
2779 2780
		writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
		if (total_scanned > writeback_threshold) {
2781 2782
			wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
						WB_REASON_TRY_TO_FREE_PAGES);
2783
			sc->may_writepage = 1;
L
Linus Torvalds 已提交
2784
		}
2785
	} while (--sc->priority >= 0);
2786

2787 2788
	delayacct_freepages_end();

2789 2790 2791
	if (sc->nr_reclaimed)
		return sc->nr_reclaimed;

2792
	/* Aborted reclaim to try compaction? don't OOM, then */
2793
	if (sc->compaction_ready)
2794 2795
		return 1;

2796 2797 2798 2799 2800 2801 2802
	/* Untapped cgroup reserves?  Don't OOM, retry. */
	if (!sc->may_thrash) {
		sc->priority = initial_priority;
		sc->may_thrash = 1;
		goto retry;
	}

2803
	return 0;
L
Linus Torvalds 已提交
2804 2805
}

2806 2807 2808 2809 2810 2811 2812 2813 2814 2815
static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
{
	struct zone *zone;
	unsigned long pfmemalloc_reserve = 0;
	unsigned long free_pages = 0;
	int i;
	bool wmark_ok;

	for (i = 0; i <= ZONE_NORMAL; i++) {
		zone = &pgdat->node_zones[i];
2816
		if (!populated_zone(zone) ||
M
Mel Gorman 已提交
2817
		    pgdat_reclaimable_pages(pgdat) == 0)
2818 2819
			continue;

2820 2821 2822 2823
		pfmemalloc_reserve += min_wmark_pages(zone);
		free_pages += zone_page_state(zone, NR_FREE_PAGES);
	}

2824 2825 2826 2827
	/* If there are no reserves (unexpected config) then do not throttle */
	if (!pfmemalloc_reserve)
		return true;

2828 2829 2830 2831
	wmark_ok = free_pages > pfmemalloc_reserve / 2;

	/* kswapd must be awake if processes are being throttled */
	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
2832
		pgdat->kswapd_classzone_idx = min(pgdat->kswapd_classzone_idx,
2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843
						(enum zone_type)ZONE_NORMAL);
		wake_up_interruptible(&pgdat->kswapd_wait);
	}

	return wmark_ok;
}

/*
 * Throttle direct reclaimers if backing storage is backed by the network
 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
 * depleted. kswapd will continue to make progress and wake the processes
2844 2845 2846 2847
 * when the low watermark is reached.
 *
 * Returns true if a fatal signal was delivered during throttling. If this
 * happens, the page allocator should not consider triggering the OOM killer.
2848
 */
2849
static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
2850 2851
					nodemask_t *nodemask)
{
2852
	struct zoneref *z;
2853
	struct zone *zone;
2854
	pg_data_t *pgdat = NULL;
2855 2856 2857 2858 2859 2860 2861 2862 2863

	/*
	 * Kernel threads should not be throttled as they may be indirectly
	 * responsible for cleaning pages necessary for reclaim to make forward
	 * progress. kjournald for example may enter direct reclaim while
	 * committing a transaction where throttling it could forcing other
	 * processes to block on log_wait_commit().
	 */
	if (current->flags & PF_KTHREAD)
2864 2865 2866 2867 2868 2869 2870 2871
		goto out;

	/*
	 * If a fatal signal is pending, this process should not throttle.
	 * It should return quickly so it can exit and free its memory
	 */
	if (fatal_signal_pending(current))
		goto out;
2872

2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
	/*
	 * Check if the pfmemalloc reserves are ok by finding the first node
	 * with a usable ZONE_NORMAL or lower zone. The expectation is that
	 * GFP_KERNEL will be required for allocating network buffers when
	 * swapping over the network so ZONE_HIGHMEM is unusable.
	 *
	 * Throttling is based on the first usable node and throttled processes
	 * wait on a queue until kswapd makes progress and wakes them. There
	 * is an affinity then between processes waking up and where reclaim
	 * progress has been made assuming the process wakes on the same node.
	 * More importantly, processes running on remote nodes will not compete
	 * for remote pfmemalloc reserves and processes on different nodes
	 * should make reasonable progress.
	 */
	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2888
					gfp_zone(gfp_mask), nodemask) {
2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
		if (zone_idx(zone) > ZONE_NORMAL)
			continue;

		/* Throttle based on the first usable node */
		pgdat = zone->zone_pgdat;
		if (pfmemalloc_watermark_ok(pgdat))
			goto out;
		break;
	}

	/* If no zone was usable by the allocation flags then do not throttle */
	if (!pgdat)
2901
		goto out;
2902

2903 2904 2905
	/* Account for the throttling */
	count_vm_event(PGSCAN_DIRECT_THROTTLE);

2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916
	/*
	 * If the caller cannot enter the filesystem, it's possible that it
	 * is due to the caller holding an FS lock or performing a journal
	 * transaction in the case of a filesystem like ext[3|4]. In this case,
	 * it is not safe to block on pfmemalloc_wait as kswapd could be
	 * blocked waiting on the same lock. Instead, throttle for up to a
	 * second before continuing.
	 */
	if (!(gfp_mask & __GFP_FS)) {
		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
			pfmemalloc_watermark_ok(pgdat), HZ);
2917 2918

		goto check_pending;
2919 2920 2921 2922 2923
	}

	/* Throttle until kswapd wakes the process */
	wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
		pfmemalloc_watermark_ok(pgdat));
2924 2925 2926 2927 2928 2929 2930

check_pending:
	if (fatal_signal_pending(current))
		return true;

out:
	return false;
2931 2932
}

2933
unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
2934
				gfp_t gfp_mask, nodemask_t *nodemask)
2935
{
2936
	unsigned long nr_reclaimed;
2937
	struct scan_control sc = {
2938
		.nr_to_reclaim = SWAP_CLUSTER_MAX,
2939
		.gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
2940
		.reclaim_idx = gfp_zone(gfp_mask),
2941 2942 2943
		.order = order,
		.nodemask = nodemask,
		.priority = DEF_PRIORITY,
2944
		.may_writepage = !laptop_mode,
2945
		.may_unmap = 1,
2946
		.may_swap = 1,
2947 2948
	};

2949
	/*
2950 2951 2952
	 * Do not enter reclaim if fatal signal was delivered while throttled.
	 * 1 is returned so that the page allocator does not OOM kill at this
	 * point.
2953
	 */
2954
	if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask))
2955 2956
		return 1;

2957 2958
	trace_mm_vmscan_direct_reclaim_begin(order,
				sc.may_writepage,
2959 2960
				gfp_mask,
				sc.reclaim_idx);
2961

2962
	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
2963 2964 2965 2966

	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);

	return nr_reclaimed;
2967 2968
}

A
Andrew Morton 已提交
2969
#ifdef CONFIG_MEMCG
2970

2971
unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
2972
						gfp_t gfp_mask, bool noswap,
2973
						pg_data_t *pgdat,
2974
						unsigned long *nr_scanned)
2975 2976
{
	struct scan_control sc = {
2977
		.nr_to_reclaim = SWAP_CLUSTER_MAX,
2978
		.target_mem_cgroup = memcg,
2979 2980
		.may_writepage = !laptop_mode,
		.may_unmap = 1,
2981
		.reclaim_idx = MAX_NR_ZONES - 1,
2982 2983
		.may_swap = !noswap,
	};
2984
	unsigned long lru_pages;
2985

2986 2987
	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
2988

2989
	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
2990
						      sc.may_writepage,
2991 2992
						      sc.gfp_mask,
						      sc.reclaim_idx);
2993

2994 2995 2996
	/*
	 * NOTE: Although we can get the priority field, using it
	 * here is not a good idea, since it limits the pages we can scan.
2997
	 * if we don't reclaim here, the shrink_node from balance_pgdat
2998 2999 3000
	 * will pick up pages from other mem cgroup's as well. We hack
	 * the priority and make it zero.
	 */
3001
	shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
3002 3003 3004

	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);

3005
	*nr_scanned = sc.nr_scanned;
3006 3007 3008
	return sc.nr_reclaimed;
}

3009
unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3010
					   unsigned long nr_pages,
K
KOSAKI Motohiro 已提交
3011
					   gfp_t gfp_mask,
3012
					   bool may_swap)
3013
{
3014
	struct zonelist *zonelist;
3015
	unsigned long nr_reclaimed;
3016
	int nid;
3017
	struct scan_control sc = {
3018
		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3019 3020
		.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
				(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3021
		.reclaim_idx = MAX_NR_ZONES - 1,
3022 3023 3024 3025
		.target_mem_cgroup = memcg,
		.priority = DEF_PRIORITY,
		.may_writepage = !laptop_mode,
		.may_unmap = 1,
3026
		.may_swap = may_swap,
3027
	};
3028

3029 3030 3031 3032 3033
	/*
	 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
	 * take care of from where we get pages. So the node where we start the
	 * scan does not need to be the current node.
	 */
3034
	nid = mem_cgroup_select_victim_node(memcg);
3035 3036

	zonelist = NODE_DATA(nid)->node_zonelists;
3037 3038 3039

	trace_mm_vmscan_memcg_reclaim_begin(0,
					    sc.may_writepage,
3040 3041
					    sc.gfp_mask,
					    sc.reclaim_idx);
3042

3043
	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3044 3045 3046 3047

	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);

	return nr_reclaimed;
3048 3049 3050
}
#endif

3051
static void age_active_anon(struct pglist_data *pgdat,
3052
				struct scan_control *sc)
3053
{
3054
	struct mem_cgroup *memcg;
3055

3056 3057 3058 3059 3060
	if (!total_swap_pages)
		return;

	memcg = mem_cgroup_iter(NULL, NULL, NULL);
	do {
3061
		struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3062

3063
		if (inactive_list_is_low(lruvec, false, sc))
3064
			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3065
					   sc, LRU_ACTIVE_ANON);
3066 3067 3068

		memcg = mem_cgroup_iter(NULL, memcg, NULL);
	} while (memcg);
3069 3070
}

M
Mel Gorman 已提交
3071
static bool zone_balanced(struct zone *zone, int order, int classzone_idx)
3072
{
M
Mel Gorman 已提交
3073
	unsigned long mark = high_wmark_pages(zone);
3074

3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
	if (!zone_watermark_ok_safe(zone, order, mark, classzone_idx))
		return false;

	/*
	 * If any eligible zone is balanced then the node is not considered
	 * to be congested or dirty
	 */
	clear_bit(PGDAT_CONGESTED, &zone->zone_pgdat->flags);
	clear_bit(PGDAT_DIRTY, &zone->zone_pgdat->flags);

	return true;
3086 3087
}

3088 3089 3090 3091 3092 3093
/*
 * Prepare kswapd for sleeping. This verifies that there are no processes
 * waiting in throttle_direct_reclaim() and that watermarks have been met.
 *
 * Returns true if kswapd is ready to sleep
 */
3094
static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3095
{
3096 3097
	int i;

3098
	/*
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109
	 * The throttled processes are normally woken up in balance_pgdat() as
	 * soon as pfmemalloc_watermark_ok() is true. But there is a potential
	 * race between when kswapd checks the watermarks and a process gets
	 * throttled. There is also a potential race if processes get
	 * throttled, kswapd wakes, a large process exits thereby balancing the
	 * zones, which causes kswapd to exit balance_pgdat() before reaching
	 * the wake up checks. If kswapd is going to sleep, no process should
	 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
	 * the wake up is premature, processes will wake kswapd and get
	 * throttled again. The difference from wake ups in balance_pgdat() is
	 * that here we are under prepare_to_wait().
3110
	 */
3111 3112
	if (waitqueue_active(&pgdat->pfmemalloc_wait))
		wake_up_all(&pgdat->pfmemalloc_wait);
3113

3114 3115 3116 3117 3118 3119
	for (i = 0; i <= classzone_idx; i++) {
		struct zone *zone = pgdat->node_zones + i;

		if (!populated_zone(zone))
			continue;

3120 3121
		if (!zone_balanced(zone, order, classzone_idx))
			return false;
3122 3123
	}

3124
	return true;
3125 3126
}

3127
/*
3128 3129
 * kswapd shrinks a node of pages that are at or below the highest usable
 * zone that is currently unbalanced.
3130 3131
 *
 * Returns true if kswapd scanned at least the requested number of pages to
3132 3133
 * reclaim or if the lack of progress was due to pages under writeback.
 * This is used to determine if the scanning priority needs to be raised.
3134
 */
3135
static bool kswapd_shrink_node(pg_data_t *pgdat,
3136
			       struct scan_control *sc)
3137
{
3138 3139
	struct zone *zone;
	int z;
3140

3141 3142
	/* Reclaim a number of pages proportional to the number of zones */
	sc->nr_to_reclaim = 0;
3143
	for (z = 0; z <= sc->reclaim_idx; z++) {
3144 3145 3146
		zone = pgdat->node_zones + z;
		if (!populated_zone(zone))
			continue;
3147

3148 3149
		sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
	}
3150 3151

	/*
3152 3153
	 * Historically care was taken to put equal pressure on all zones but
	 * now pressure is applied based on node LRU order.
3154
	 */
3155
	shrink_node(pgdat, sc);
3156

3157
	/*
3158 3159 3160 3161 3162
	 * Fragmentation may mean that the system cannot be rebalanced for
	 * high-order allocations. If twice the allocation size has been
	 * reclaimed then recheck watermarks only at order-0 to prevent
	 * excessive reclaim. Assume that a process requested a high-order
	 * can direct reclaim/compact.
3163
	 */
3164 3165
	if (sc->order && sc->nr_reclaimed >= 2UL << sc->order)
		sc->order = 0;
3166

3167
	return sc->nr_scanned >= sc->nr_to_reclaim;
3168 3169
}

L
Linus Torvalds 已提交
3170
/*
3171 3172 3173
 * For kswapd, balance_pgdat() will reclaim pages across a node from zones
 * that are eligible for use by the caller until at least one zone is
 * balanced.
L
Linus Torvalds 已提交
3174
 *
3175
 * Returns the order kswapd finished reclaiming at.
L
Linus Torvalds 已提交
3176 3177
 *
 * kswapd scans the zones in the highmem->normal->dma direction.  It skips
3178
 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
3179 3180 3181
 * found to have free_pages <= high_wmark_pages(zone), any page is that zone
 * or lower is eligible for reclaim until at least one usable zone is
 * balanced.
L
Linus Torvalds 已提交
3182
 */
3183
static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
L
Linus Torvalds 已提交
3184 3185
{
	int i;
3186 3187
	unsigned long nr_soft_reclaimed;
	unsigned long nr_soft_scanned;
3188
	struct zone *zone;
3189 3190
	struct scan_control sc = {
		.gfp_mask = GFP_KERNEL,
3191
		.order = order,
3192
		.priority = DEF_PRIORITY,
3193
		.may_writepage = !laptop_mode,
3194
		.may_unmap = 1,
3195
		.may_swap = 1,
3196
	};
3197
	count_vm_event(PAGEOUTRUN);
L
Linus Torvalds 已提交
3198

3199
	do {
3200 3201 3202
		bool raise_priority = true;

		sc.nr_reclaimed = 0;
3203
		sc.reclaim_idx = classzone_idx;
L
Linus Torvalds 已提交
3204

3205
		/*
3206 3207 3208 3209 3210 3211 3212 3213
		 * If the number of buffer_heads exceeds the maximum allowed
		 * then consider reclaiming from all zones. This has a dual
		 * purpose -- on 64-bit systems it is expected that
		 * buffer_heads are stripped during active rotation. On 32-bit
		 * systems, highmem pages can pin lowmem memory and shrinking
		 * buffers can relieve lowmem pressure. Reclaim may still not
		 * go ahead if all eligible zones for the original allocation
		 * request are balanced to avoid excessive reclaim from kswapd.
3214 3215 3216 3217 3218 3219
		 */
		if (buffer_heads_over_limit) {
			for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
				zone = pgdat->node_zones + i;
				if (!populated_zone(zone))
					continue;
3220

3221
				sc.reclaim_idx = i;
A
Andrew Morton 已提交
3222
				break;
L
Linus Torvalds 已提交
3223 3224
			}
		}
3225

3226 3227 3228 3229 3230 3231
		/*
		 * Only reclaim if there are no eligible zones. Check from
		 * high to low zone as allocations prefer higher zones.
		 * Scanning from low to high zone would allow congestion to be
		 * cleared during a very small window when a small low
		 * zone was balanced even under extreme pressure when the
3232 3233 3234
		 * overall node may be congested. Note that sc.reclaim_idx
		 * is not used as buffer_heads_over_limit may have adjusted
		 * it.
3235
		 */
3236
		for (i = classzone_idx; i >= 0; i--) {
3237 3238 3239 3240
			zone = pgdat->node_zones + i;
			if (!populated_zone(zone))
				continue;

3241
			if (zone_balanced(zone, sc.order, classzone_idx))
3242 3243
				goto out;
		}
A
Andrew Morton 已提交
3244

3245 3246 3247 3248 3249 3250
		/*
		 * Do some background aging of the anon list, to give
		 * pages a chance to be referenced before reclaiming. All
		 * pages are rotated regardless of classzone as this is
		 * about consistent aging.
		 */
3251
		age_active_anon(pgdat, &sc);
3252

3253 3254 3255 3256
		/*
		 * If we're getting trouble reclaiming, start doing writepage
		 * even in laptop mode.
		 */
3257
		if (sc.priority < DEF_PRIORITY - 2 || !pgdat_reclaimable(pgdat))
3258 3259
			sc.may_writepage = 1;

3260 3261 3262
		/* Call soft limit reclaim before calling shrink_node. */
		sc.nr_scanned = 0;
		nr_soft_scanned = 0;
3263
		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
3264 3265 3266
						sc.gfp_mask, &nr_soft_scanned);
		sc.nr_reclaimed += nr_soft_reclaimed;

L
Linus Torvalds 已提交
3267
		/*
3268 3269 3270
		 * There should be no need to raise the scanning priority if
		 * enough pages are already being scanned that that high
		 * watermark would be met at 100% efficiency.
L
Linus Torvalds 已提交
3271
		 */
3272
		if (kswapd_shrink_node(pgdat, &sc))
3273
			raise_priority = false;
3274 3275 3276 3277 3278 3279 3280 3281

		/*
		 * If the low watermark is met there is no need for processes
		 * to be throttled on pfmemalloc_wait as they should not be
		 * able to safely make forward progress. Wake them
		 */
		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
				pfmemalloc_watermark_ok(pgdat))
3282
			wake_up_all(&pgdat->pfmemalloc_wait);
3283

3284 3285 3286
		/* Check if kswapd should be suspending */
		if (try_to_freeze() || kthread_should_stop())
			break;
3287

3288
		/*
3289 3290
		 * Raise priority if scanning rate is too low or there was no
		 * progress in reclaiming pages
3291
		 */
3292 3293
		if (raise_priority || !sc.nr_reclaimed)
			sc.priority--;
3294
	} while (sc.priority >= 1);
L
Linus Torvalds 已提交
3295

3296
out:
3297
	/*
3298 3299 3300 3301
	 * Return the order kswapd stopped reclaiming at as
	 * prepare_kswapd_sleep() takes it into account. If another caller
	 * entered the allocator slow path while kswapd was awake, order will
	 * remain at the higher level.
3302
	 */
3303
	return sc.order;
L
Linus Torvalds 已提交
3304 3305
}

3306 3307
static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
				unsigned int classzone_idx)
3308 3309 3310 3311 3312 3313 3314 3315 3316 3317
{
	long remaining = 0;
	DEFINE_WAIT(wait);

	if (freezing(current) || kthread_should_stop())
		return;

	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);

	/* Try to sleep for a short interval */
3318
	if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330
		/*
		 * Compaction records what page blocks it recently failed to
		 * isolate pages from and skips them in the future scanning.
		 * When kswapd is going to sleep, it is reasonable to assume
		 * that pages and compaction may succeed so reset the cache.
		 */
		reset_isolation_suitable(pgdat);

		/*
		 * We have freed the memory, now we should compact it to make
		 * allocation of the requested order possible.
		 */
3331
		wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
3332

3333
		remaining = schedule_timeout(HZ/10);
3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344

		/*
		 * If woken prematurely then reset kswapd_classzone_idx and
		 * order. The values will either be from a wakeup request or
		 * the previous request that slept prematurely.
		 */
		if (remaining) {
			pgdat->kswapd_classzone_idx = max(pgdat->kswapd_classzone_idx, classzone_idx);
			pgdat->kswapd_order = max(pgdat->kswapd_order, reclaim_order);
		}

3345 3346 3347 3348 3349 3350 3351 3352
		finish_wait(&pgdat->kswapd_wait, &wait);
		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
	}

	/*
	 * After a short sleep, check if it was a premature sleep. If not, then
	 * go fully to sleep until explicitly woken up.
	 */
3353 3354
	if (!remaining &&
	    prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365
		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);

		/*
		 * vmstat counters are not perfectly accurate and the estimated
		 * value for counters such as NR_FREE_PAGES can deviate from the
		 * true value by nr_online_cpus * threshold. To avoid the zone
		 * watermarks being breached while under pressure, we reduce the
		 * per-cpu vmstat threshold while kswapd is awake and restore
		 * them before going back to sleep.
		 */
		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
3366 3367 3368 3369

		if (!kthread_should_stop())
			schedule();

3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
	} else {
		if (remaining)
			count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
		else
			count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
	}
	finish_wait(&pgdat->kswapd_wait, &wait);
}

L
Linus Torvalds 已提交
3380 3381
/*
 * The background pageout daemon, started as a kernel thread
3382
 * from the init process.
L
Linus Torvalds 已提交
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
 *
 * This basically trickles out pages so that we have _some_
 * free memory available even if there is no other activity
 * that frees anything up. This is needed for things like routing
 * etc, where we otherwise might have all activity going on in
 * asynchronous contexts that cannot page things out.
 *
 * If there are applications that are active memory-allocators
 * (most normal use), this basically shouldn't matter.
 */
static int kswapd(void *p)
{
3395
	unsigned int alloc_order, reclaim_order, classzone_idx;
L
Linus Torvalds 已提交
3396 3397
	pg_data_t *pgdat = (pg_data_t*)p;
	struct task_struct *tsk = current;
3398

L
Linus Torvalds 已提交
3399 3400 3401
	struct reclaim_state reclaim_state = {
		.reclaimed_slab = 0,
	};
3402
	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
L
Linus Torvalds 已提交
3403

3404 3405
	lockdep_set_current_reclaim_state(GFP_KERNEL);

R
Rusty Russell 已提交
3406
	if (!cpumask_empty(cpumask))
3407
		set_cpus_allowed_ptr(tsk, cpumask);
L
Linus Torvalds 已提交
3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421
	current->reclaim_state = &reclaim_state;

	/*
	 * Tell the memory management that we're a "memory allocator",
	 * and that if we need more memory we should get access to it
	 * regardless (see "__alloc_pages()"). "kswapd" should
	 * never get caught in the normal page freeing logic.
	 *
	 * (Kswapd normally doesn't need memory anyway, but sometimes
	 * you need a small amount of memory in order to be able to
	 * page out something else, and this flag essentially protects
	 * us from recursively trying to free more memory as we're
	 * trying to free the first piece of memory in the first place).
	 */
3422
	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3423
	set_freezable();
L
Linus Torvalds 已提交
3424

3425 3426
	pgdat->kswapd_order = alloc_order = reclaim_order = 0;
	pgdat->kswapd_classzone_idx = classzone_idx = 0;
L
Linus Torvalds 已提交
3427
	for ( ; ; ) {
3428
		bool ret;
3429

3430 3431 3432
kswapd_try_sleep:
		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
					classzone_idx);
3433

3434 3435 3436 3437 3438
		/* Read the new order and classzone_idx */
		alloc_order = reclaim_order = pgdat->kswapd_order;
		classzone_idx = pgdat->kswapd_classzone_idx;
		pgdat->kswapd_order = 0;
		pgdat->kswapd_classzone_idx = 0;
L
Linus Torvalds 已提交
3439

3440 3441 3442 3443 3444 3445 3446 3447
		ret = try_to_freeze();
		if (kthread_should_stop())
			break;

		/*
		 * We can speed up thawing tasks if we don't call balance_pgdat
		 * after returning from the refrigerator
		 */
3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458
		if (ret)
			continue;

		/*
		 * Reclaim begins at the requested order but if a high-order
		 * reclaim fails then kswapd falls back to reclaiming for
		 * order-0. If that happens, kswapd will consider sleeping
		 * for the order it finished reclaiming at (reclaim_order)
		 * but kcompactd is woken to compact for the original
		 * request (alloc_order).
		 */
3459 3460
		trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
						alloc_order);
3461 3462 3463
		reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
		if (reclaim_order < alloc_order)
			goto kswapd_try_sleep;
3464

3465 3466
		alloc_order = reclaim_order = pgdat->kswapd_order;
		classzone_idx = pgdat->kswapd_classzone_idx;
L
Linus Torvalds 已提交
3467
	}
3468

3469
	tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3470
	current->reclaim_state = NULL;
3471 3472
	lockdep_clear_current_reclaim_state();

L
Linus Torvalds 已提交
3473 3474 3475 3476 3477 3478
	return 0;
}

/*
 * A zone is low on free memory, so wake its kswapd task to service it.
 */
3479
void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
L
Linus Torvalds 已提交
3480 3481
{
	pg_data_t *pgdat;
3482
	int z;
L
Linus Torvalds 已提交
3483

3484
	if (!populated_zone(zone))
L
Linus Torvalds 已提交
3485 3486
		return;

3487
	if (!cpuset_zone_allowed(zone, GFP_KERNEL | __GFP_HARDWALL))
L
Linus Torvalds 已提交
3488
		return;
3489
	pgdat = zone->zone_pgdat;
3490 3491
	pgdat->kswapd_classzone_idx = max(pgdat->kswapd_classzone_idx, classzone_idx);
	pgdat->kswapd_order = max(pgdat->kswapd_order, order);
3492
	if (!waitqueue_active(&pgdat->kswapd_wait))
L
Linus Torvalds 已提交
3493
		return;
3494 3495 3496 3497 3498 3499 3500 3501 3502 3503

	/* Only wake kswapd if all zones are unbalanced */
	for (z = 0; z <= classzone_idx; z++) {
		zone = pgdat->node_zones + z;
		if (!populated_zone(zone))
			continue;

		if (zone_balanced(zone, order, classzone_idx))
			return;
	}
3504 3505

	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
3506
	wake_up_interruptible(&pgdat->kswapd_wait);
L
Linus Torvalds 已提交
3507 3508
}

3509
#ifdef CONFIG_HIBERNATION
L
Linus Torvalds 已提交
3510
/*
3511
 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3512 3513 3514 3515 3516
 * freed pages.
 *
 * Rather than trying to age LRUs the aim is to preserve the overall
 * LRU order by reclaiming preferentially
 * inactive > active > active referenced > active mapped
L
Linus Torvalds 已提交
3517
 */
3518
unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
L
Linus Torvalds 已提交
3519
{
3520 3521
	struct reclaim_state reclaim_state;
	struct scan_control sc = {
3522
		.nr_to_reclaim = nr_to_reclaim,
3523
		.gfp_mask = GFP_HIGHUSER_MOVABLE,
3524
		.reclaim_idx = MAX_NR_ZONES - 1,
3525
		.priority = DEF_PRIORITY,
3526
		.may_writepage = 1,
3527 3528
		.may_unmap = 1,
		.may_swap = 1,
3529
		.hibernation_mode = 1,
L
Linus Torvalds 已提交
3530
	};
3531
	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3532 3533
	struct task_struct *p = current;
	unsigned long nr_reclaimed;
L
Linus Torvalds 已提交
3534

3535 3536 3537 3538
	p->flags |= PF_MEMALLOC;
	lockdep_set_current_reclaim_state(sc.gfp_mask);
	reclaim_state.reclaimed_slab = 0;
	p->reclaim_state = &reclaim_state;
3539

3540
	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3541

3542 3543 3544
	p->reclaim_state = NULL;
	lockdep_clear_current_reclaim_state();
	p->flags &= ~PF_MEMALLOC;
3545

3546
	return nr_reclaimed;
L
Linus Torvalds 已提交
3547
}
3548
#endif /* CONFIG_HIBERNATION */
L
Linus Torvalds 已提交
3549 3550 3551 3552 3553

/* It's optimal to keep kswapds on the same CPUs as their memory, but
   not required for correctness.  So if the last cpu in a node goes
   away, we get changed to run anywhere: as the first one comes back,
   restore their cpu bindings. */
3554 3555
static int cpu_callback(struct notifier_block *nfb, unsigned long action,
			void *hcpu)
L
Linus Torvalds 已提交
3556
{
3557
	int nid;
L
Linus Torvalds 已提交
3558

3559
	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
3560
		for_each_node_state(nid, N_MEMORY) {
3561
			pg_data_t *pgdat = NODE_DATA(nid);
3562 3563 3564
			const struct cpumask *mask;

			mask = cpumask_of_node(pgdat->node_id);
3565

3566
			if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
L
Linus Torvalds 已提交
3567
				/* One of our CPUs online: restore mask */
3568
				set_cpus_allowed_ptr(pgdat->kswapd, mask);
L
Linus Torvalds 已提交
3569 3570 3571 3572 3573
		}
	}
	return NOTIFY_OK;
}

3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589
/*
 * This kswapd start function will be called by init and node-hot-add.
 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
 */
int kswapd_run(int nid)
{
	pg_data_t *pgdat = NODE_DATA(nid);
	int ret = 0;

	if (pgdat->kswapd)
		return 0;

	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
	if (IS_ERR(pgdat->kswapd)) {
		/* failure at boot is fatal */
		BUG_ON(system_state == SYSTEM_BOOTING);
3590 3591
		pr_err("Failed to start kswapd on node %d\n", nid);
		ret = PTR_ERR(pgdat->kswapd);
3592
		pgdat->kswapd = NULL;
3593 3594 3595 3596
	}
	return ret;
}

3597
/*
3598
 * Called by memory hotplug when all memory in a node is offlined.  Caller must
3599
 * hold mem_hotplug_begin/end().
3600 3601 3602 3603 3604
 */
void kswapd_stop(int nid)
{
	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;

3605
	if (kswapd) {
3606
		kthread_stop(kswapd);
3607 3608
		NODE_DATA(nid)->kswapd = NULL;
	}
3609 3610
}

L
Linus Torvalds 已提交
3611 3612
static int __init kswapd_init(void)
{
3613
	int nid;
3614

L
Linus Torvalds 已提交
3615
	swap_setup();
3616
	for_each_node_state(nid, N_MEMORY)
3617
 		kswapd_run(nid);
L
Linus Torvalds 已提交
3618 3619 3620 3621 3622
	hotcpu_notifier(cpu_callback, 0);
	return 0;
}

module_init(kswapd_init)
3623 3624 3625

#ifdef CONFIG_NUMA
/*
3626
 * Node reclaim mode
3627
 *
3628
 * If non-zero call node_reclaim when the number of free pages falls below
3629 3630
 * the watermarks.
 */
3631
int node_reclaim_mode __read_mostly;
3632

3633
#define RECLAIM_OFF 0
3634
#define RECLAIM_ZONE (1<<0)	/* Run shrink_inactive_list on the zone */
3635
#define RECLAIM_WRITE (1<<1)	/* Writeout pages during reclaim */
3636
#define RECLAIM_UNMAP (1<<2)	/* Unmap pages during reclaim */
3637

3638
/*
3639
 * Priority for NODE_RECLAIM. This determines the fraction of pages
3640 3641 3642
 * of a node considered for each zone_reclaim. 4 scans 1/16th of
 * a zone.
 */
3643
#define NODE_RECLAIM_PRIORITY 4
3644

3645
/*
3646
 * Percentage of pages in a zone that must be unmapped for node_reclaim to
3647 3648 3649 3650
 * occur.
 */
int sysctl_min_unmapped_ratio = 1;

3651 3652 3653 3654 3655 3656
/*
 * If the number of slab pages in a zone grows beyond this percentage then
 * slab reclaim needs to occur.
 */
int sysctl_min_slab_ratio = 5;

3657
static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
3658
{
3659 3660 3661
	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
		node_page_state(pgdat, NR_ACTIVE_FILE);
3662 3663 3664 3665 3666 3667 3668 3669 3670 3671

	/*
	 * It's possible for there to be more file mapped pages than
	 * accounted for by the pages on the file LRU lists because
	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
	 */
	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
}

/* Work out how many page cache pages we can reclaim in this reclaim_mode */
3672
static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
3673
{
3674 3675
	unsigned long nr_pagecache_reclaimable;
	unsigned long delta = 0;
3676 3677

	/*
3678
	 * If RECLAIM_UNMAP is set, then all file pages are considered
3679
	 * potentially reclaimable. Otherwise, we have to worry about
3680
	 * pages like swapcache and node_unmapped_file_pages() provides
3681 3682
	 * a better estimate
	 */
3683 3684
	if (node_reclaim_mode & RECLAIM_UNMAP)
		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
3685
	else
3686
		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
3687 3688

	/* If we can't clean pages, remove dirty pages from consideration */
3689 3690
	if (!(node_reclaim_mode & RECLAIM_WRITE))
		delta += node_page_state(pgdat, NR_FILE_DIRTY);
3691 3692 3693 3694 3695 3696 3697 3698

	/* Watch for any possible underflows due to delta */
	if (unlikely(delta > nr_pagecache_reclaimable))
		delta = nr_pagecache_reclaimable;

	return nr_pagecache_reclaimable - delta;
}

3699
/*
3700
 * Try to free up some pages from this node through reclaim.
3701
 */
3702
static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
3703
{
3704
	/* Minimum pages needed in order to stay on node */
3705
	const unsigned long nr_pages = 1 << order;
3706 3707
	struct task_struct *p = current;
	struct reclaim_state reclaim_state;
3708
	int classzone_idx = gfp_zone(gfp_mask);
3709
	struct scan_control sc = {
3710
		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3711
		.gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
3712
		.order = order,
3713 3714 3715
		.priority = NODE_RECLAIM_PRIORITY,
		.may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
		.may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
3716
		.may_swap = 1,
3717
		.reclaim_idx = classzone_idx,
3718
	};
3719 3720

	cond_resched();
3721
	/*
3722
	 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
3723
	 * and we also need to be able to write out pages for RECLAIM_WRITE
3724
	 * and RECLAIM_UNMAP.
3725 3726
	 */
	p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
3727
	lockdep_set_current_reclaim_state(gfp_mask);
3728 3729
	reclaim_state.reclaimed_slab = 0;
	p->reclaim_state = &reclaim_state;
3730

3731
	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
3732 3733 3734 3735 3736
		/*
		 * Free memory by calling shrink zone with increasing
		 * priorities until we have enough memory freed.
		 */
		do {
3737
			shrink_node(pgdat, &sc);
3738
		} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
3739
	}
3740

3741
	p->reclaim_state = NULL;
3742
	current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
3743
	lockdep_clear_current_reclaim_state();
3744
	return sc.nr_reclaimed >= nr_pages;
3745
}
3746

3747
int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
3748
{
3749
	int ret;
3750 3751

	/*
3752
	 * Node reclaim reclaims unmapped file backed pages and
3753
	 * slab pages if we are over the defined limits.
3754
	 *
3755 3756
	 * A small portion of unmapped file backed pages is needed for
	 * file I/O otherwise pages read by file I/O will be immediately
3757 3758
	 * thrown out if the node is overallocated. So we do not reclaim
	 * if less than a specified percentage of the node is used by
3759
	 * unmapped file backed pages.
3760
	 */
3761 3762 3763
	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
	    sum_zone_node_page_state(pgdat->node_id, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
		return NODE_RECLAIM_FULL;
3764

3765 3766
	if (!pgdat_reclaimable(pgdat))
		return NODE_RECLAIM_FULL;
3767

3768
	/*
3769
	 * Do not scan if the allocation should not be delayed.
3770
	 */
3771
	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
3772
		return NODE_RECLAIM_NOSCAN;
3773 3774

	/*
3775
	 * Only run node reclaim on the local node or on nodes that do not
3776 3777 3778 3779
	 * have associated processors. This will favor the local processor
	 * over remote processors and spread off node memory allocations
	 * as wide as possible.
	 */
3780 3781
	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
		return NODE_RECLAIM_NOSCAN;
3782

3783 3784
	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
		return NODE_RECLAIM_NOSCAN;
3785

3786 3787
	ret = __node_reclaim(pgdat, gfp_mask, order);
	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
3788

3789 3790 3791
	if (!ret)
		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);

3792
	return ret;
3793
}
3794
#endif
L
Lee Schermerhorn 已提交
3795 3796 3797 3798 3799 3800

/*
 * page_evictable - test whether a page is evictable
 * @page: the page to test
 *
 * Test whether page is evictable--i.e., should be placed on active/inactive
3801
 * lists vs unevictable list.
L
Lee Schermerhorn 已提交
3802 3803
 *
 * Reasons page might not be evictable:
3804
 * (1) page's mapping marked unevictable
N
Nick Piggin 已提交
3805
 * (2) page is part of an mlocked VMA
3806
 *
L
Lee Schermerhorn 已提交
3807
 */
3808
int page_evictable(struct page *page)
L
Lee Schermerhorn 已提交
3809
{
3810
	return !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
L
Lee Schermerhorn 已提交
3811
}
3812

3813
#ifdef CONFIG_SHMEM
3814
/**
3815 3816 3817
 * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
 * @pages:	array of pages to check
 * @nr_pages:	number of pages to check
3818
 *
3819
 * Checks pages for evictability and moves them to the appropriate lru list.
3820 3821
 *
 * This function is only used for SysV IPC SHM_UNLOCK.
3822
 */
3823
void check_move_unevictable_pages(struct page **pages, int nr_pages)
3824
{
3825
	struct lruvec *lruvec;
3826
	struct pglist_data *pgdat = NULL;
3827 3828 3829
	int pgscanned = 0;
	int pgrescued = 0;
	int i;
3830

3831 3832
	for (i = 0; i < nr_pages; i++) {
		struct page *page = pages[i];
3833
		struct pglist_data *pagepgdat = page_pgdat(page);
3834

3835
		pgscanned++;
3836 3837 3838 3839 3840
		if (pagepgdat != pgdat) {
			if (pgdat)
				spin_unlock_irq(&pgdat->lru_lock);
			pgdat = pagepgdat;
			spin_lock_irq(&pgdat->lru_lock);
3841
		}
3842
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
3843

3844 3845
		if (!PageLRU(page) || !PageUnevictable(page))
			continue;
3846

3847
		if (page_evictable(page)) {
3848 3849
			enum lru_list lru = page_lru_base_type(page);

3850
			VM_BUG_ON_PAGE(PageActive(page), page);
3851
			ClearPageUnevictable(page);
3852 3853
			del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
			add_page_to_lru_list(page, lruvec, lru);
3854
			pgrescued++;
3855
		}
3856
	}
3857

3858
	if (pgdat) {
3859 3860
		__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
		__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
3861
		spin_unlock_irq(&pgdat->lru_lock);
3862 3863
	}
}
3864
#endif /* CONFIG_SHMEM */