swap.c 31.0 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
L
Linus Torvalds 已提交
2 3 4 5 6 7 8
/*
 *  linux/mm/swap.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 */

/*
S
Simon Arlott 已提交
9
 * This file contains the default values for the operation of the
L
Linus Torvalds 已提交
10
 * Linux VM subsystem. Fine-tuning documentation can be found in
11
 * Documentation/admin-guide/sysctl/vm.rst.
L
Linus Torvalds 已提交
12 13 14 15 16 17 18 19 20 21 22 23 24
 * Started 18.12.91
 * Swap aging added 23.2.95, Stephen Tweedie.
 * Buffermem limits added 12.3.98, Rik van Riel.
 */

#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/init.h>
25
#include <linux/export.h>
L
Linus Torvalds 已提交
26 27
#include <linux/mm_inline.h>
#include <linux/percpu_counter.h>
28
#include <linux/memremap.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
P
Peter Zijlstra 已提交
32
#include <linux/backing-dev.h>
33
#include <linux/memcontrol.h>
34
#include <linux/gfp.h>
35
#include <linux/uio.h>
36
#include <linux/hugetlb.h>
37
#include <linux/page_idle.h>
L
Linus Torvalds 已提交
38

39 40
#include "internal.h"

41 42 43
#define CREATE_TRACE_POINTS
#include <trace/events/pagemap.h>

L
Linus Torvalds 已提交
44 45 46
/* How many pages do we try to swap or page in/out together? */
int page_cluster;

47
static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
48
static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
49
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
M
Minchan Kim 已提交
50
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
51
static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
52 53 54
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
#endif
55

A
Adrian Bunk 已提交
56 57 58 59
/*
 * This path almost never happens for VM activity - pages are normally
 * freed via pagevecs.  But it gets used by networking.
 */
H
Harvey Harrison 已提交
60
static void __page_cache_release(struct page *page)
A
Adrian Bunk 已提交
61 62
{
	if (PageLRU(page)) {
63
		pg_data_t *pgdat = page_pgdat(page);
64 65
		struct lruvec *lruvec;
		unsigned long flags;
A
Adrian Bunk 已提交
66

67 68
		spin_lock_irqsave(&pgdat->lru_lock, flags);
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
69
		VM_BUG_ON_PAGE(!PageLRU(page), page);
A
Adrian Bunk 已提交
70
		__ClearPageLRU(page);
71
		del_page_from_lru_list(page, lruvec, page_off_lru(page));
72
		spin_unlock_irqrestore(&pgdat->lru_lock, flags);
A
Adrian Bunk 已提交
73
	}
74
	__ClearPageWaiters(page);
75 76 77 78 79
}

static void __put_single_page(struct page *page)
{
	__page_cache_release(page);
80
	mem_cgroup_uncharge(page);
81
	free_unref_page(page);
A
Adrian Bunk 已提交
82 83
}

84
static void __put_compound_page(struct page *page)
L
Linus Torvalds 已提交
85
{
86
	compound_page_dtor *dtor;
L
Linus Torvalds 已提交
87

88 89 90 91 92 93 94 95
	/*
	 * __page_cache_release() is supposed to be called for thp, not for
	 * hugetlb. This is because hugetlb page does never have PageLRU set
	 * (it's never listed to any LRU lists) and no memcg routines should
	 * be called for hugetlb (it has a separate hugetlb_cgroup.)
	 */
	if (!PageHuge(page))
		__page_cache_release(page);
96 97 98 99
	dtor = get_compound_page_dtor(page);
	(*dtor)(page);
}

100
void __put_page(struct page *page)
N
Nick Piggin 已提交
101
{
102 103 104 105 106 107 108 109 110 111
	if (is_zone_device_page(page)) {
		put_dev_pagemap(page->pgmap);

		/*
		 * The page belongs to the device that created pgmap. Do
		 * not return it to page allocator.
		 */
		return;
	}

N
Nick Piggin 已提交
112
	if (unlikely(PageCompound(page)))
113 114
		__put_compound_page(page);
	else
115
		__put_single_page(page);
L
Linus Torvalds 已提交
116
}
117
EXPORT_SYMBOL(__put_page);
118

119
/**
120 121
 * put_pages_list() - release a list of pages
 * @pages: list of pages threaded on page->lru
122 123 124 125 126 127 128 129 130
 *
 * Release a list of pages which are strung together on page.lru.  Currently
 * used by read_cache_pages() and related error recovery code.
 */
void put_pages_list(struct list_head *pages)
{
	while (!list_empty(pages)) {
		struct page *victim;

131
		victim = lru_to_page(pages);
132
		list_del(&victim->lru);
133
		put_page(victim);
134 135 136 137
	}
}
EXPORT_SYMBOL(put_pages_list);

138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
/*
 * get_kernel_pages() - pin kernel pages in memory
 * @kiov:	An array of struct kvec structures
 * @nr_segs:	number of segments to pin
 * @write:	pinning for read/write, currently ignored
 * @pages:	array that receives pointers to the pages pinned.
 *		Should be at least nr_segs long.
 *
 * Returns number of pages pinned. This may be fewer than the number
 * requested. If nr_pages is 0 or negative, returns 0. If no pages
 * were pinned, returns -errno. Each page returned must be released
 * with a put_page() call when it is finished with.
 */
int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
		struct page **pages)
{
	int seg;

	for (seg = 0; seg < nr_segs; seg++) {
		if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
			return seg;

160
		pages[seg] = kmap_to_page(kiov[seg].iov_base);
161
		get_page(pages[seg]);
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
	}

	return seg;
}
EXPORT_SYMBOL_GPL(get_kernel_pages);

/*
 * get_kernel_page() - pin a kernel page in memory
 * @start:	starting kernel address
 * @write:	pinning for read/write, currently ignored
 * @pages:	array that receives pointer to the page pinned.
 *		Must be at least nr_segs long.
 *
 * Returns 1 if page is pinned. If the page was not pinned, returns
 * -errno. The page returned must be released with a put_page() call
 * when it is finished with.
 */
int get_kernel_page(unsigned long start, int write, struct page **pages)
{
	const struct kvec kiov = {
		.iov_base = (void *)start,
		.iov_len = PAGE_SIZE
	};

	return get_kernel_pages(&kiov, 1, write, pages);
}
EXPORT_SYMBOL_GPL(get_kernel_page);

S
Shaohua Li 已提交
190
static void pagevec_lru_move_fn(struct pagevec *pvec,
191 192
	void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
	void *arg)
193 194
{
	int i;
195
	struct pglist_data *pgdat = NULL;
196
	struct lruvec *lruvec;
S
Shaohua Li 已提交
197
	unsigned long flags = 0;
198 199 200

	for (i = 0; i < pagevec_count(pvec); i++) {
		struct page *page = pvec->pages[i];
201
		struct pglist_data *pagepgdat = page_pgdat(page);
202

203 204 205 206 207
		if (pagepgdat != pgdat) {
			if (pgdat)
				spin_unlock_irqrestore(&pgdat->lru_lock, flags);
			pgdat = pagepgdat;
			spin_lock_irqsave(&pgdat->lru_lock, flags);
208
		}
S
Shaohua Li 已提交
209

210
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
211
		(*move_fn)(page, lruvec, arg);
212
	}
213 214
	if (pgdat)
		spin_unlock_irqrestore(&pgdat->lru_lock, flags);
215
	release_pages(pvec->pages, pvec->nr);
216
	pagevec_reinit(pvec);
S
Shaohua Li 已提交
217 218
}

219 220
static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
				 void *arg)
S
Shaohua Li 已提交
221 222 223
{
	int *pgmoved = arg;

224 225 226 227
	if (PageLRU(page) && !PageUnevictable(page)) {
		del_page_from_lru_list(page, lruvec, page_lru(page));
		ClearPageActive(page);
		add_page_to_lru_list_tail(page, lruvec, page_lru(page));
S
Shaohua Li 已提交
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
		(*pgmoved)++;
	}
}

/*
 * pagevec_move_tail() must be called with IRQ disabled.
 * Otherwise this may cause nasty races.
 */
static void pagevec_move_tail(struct pagevec *pvec)
{
	int pgmoved = 0;

	pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
	__count_vm_events(PGROTATED, pgmoved);
}

L
Linus Torvalds 已提交
244 245 246
/*
 * Writeback is about to end against a page which has been marked for immediate
 * reclaim.  If it still appears to be reclaimable, move it to the tail of the
247
 * inactive list.
L
Linus Torvalds 已提交
248
 */
S
Shaohua Li 已提交
249
void rotate_reclaimable_page(struct page *page)
L
Linus Torvalds 已提交
250
{
251
	if (!PageLocked(page) && !PageDirty(page) &&
L
Lee Schermerhorn 已提交
252
	    !PageUnevictable(page) && PageLRU(page)) {
253 254 255
		struct pagevec *pvec;
		unsigned long flags;

256
		get_page(page);
257
		local_irq_save(flags);
258
		pvec = this_cpu_ptr(&lru_rotate_pvecs);
259
		if (!pagevec_add(pvec, page) || PageCompound(page))
260 261 262
			pagevec_move_tail(pvec);
		local_irq_restore(flags);
	}
L
Linus Torvalds 已提交
263 264
}

265
static void update_page_reclaim_stat(struct lruvec *lruvec,
K
KOSAKI Motohiro 已提交
266 267
				     int file, int rotated)
{
268
	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
K
KOSAKI Motohiro 已提交
269 270 271 272 273 274

	reclaim_stat->recent_scanned[file]++;
	if (rotated)
		reclaim_stat->recent_rotated[file]++;
}

275 276
static void __activate_page(struct page *page, struct lruvec *lruvec,
			    void *arg)
L
Linus Torvalds 已提交
277
{
278
	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
H
Huang Ying 已提交
279
		int file = page_is_file_lru(page);
280
		int lru = page_lru_base_type(page);
281

282
		del_page_from_lru_list(page, lruvec, lru);
283 284
		SetPageActive(page);
		lru += LRU_ACTIVE;
285
		add_page_to_lru_list(page, lruvec, lru);
286
		trace_mm_lru_activate(page);
287

288 289
		__count_vm_event(PGACTIVATE);
		update_page_reclaim_stat(lruvec, file, 1);
L
Linus Torvalds 已提交
290
	}
291 292 293 294 295 296 297 298 299 300 301
}

#ifdef CONFIG_SMP
static void activate_page_drain(int cpu)
{
	struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);

	if (pagevec_count(pvec))
		pagevec_lru_move_fn(pvec, __activate_page, NULL);
}

302 303 304 305 306
static bool need_activate_page_drain(int cpu)
{
	return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
}

307 308
void activate_page(struct page *page)
{
309
	page = compound_head(page);
310 311 312
	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
		struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);

313
		get_page(page);
314
		if (!pagevec_add(pvec, page) || PageCompound(page))
315 316 317 318 319 320 321 322 323 324 325 326
			pagevec_lru_move_fn(pvec, __activate_page, NULL);
		put_cpu_var(activate_page_pvecs);
	}
}

#else
static inline void activate_page_drain(int cpu)
{
}

void activate_page(struct page *page)
{
327
	pg_data_t *pgdat = page_pgdat(page);
328

329
	page = compound_head(page);
330 331 332
	spin_lock_irq(&pgdat->lru_lock);
	__activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL);
	spin_unlock_irq(&pgdat->lru_lock);
L
Linus Torvalds 已提交
333
}
334
#endif
L
Linus Torvalds 已提交
335

336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
static void __lru_cache_activate_page(struct page *page)
{
	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
	int i;

	/*
	 * Search backwards on the optimistic assumption that the page being
	 * activated has just been added to this pagevec. Note that only
	 * the local pagevec is examined as a !PageLRU page could be in the
	 * process of being released, reclaimed, migrated or on a remote
	 * pagevec that is currently being drained. Furthermore, marking
	 * a remote pagevec's page PageActive potentially hits a race where
	 * a page is marked PageActive just after it is added to the inactive
	 * list causing accounting errors and BUG_ON checks to trigger.
	 */
	for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
		struct page *pagevec_page = pvec->pages[i];

		if (pagevec_page == page) {
			SetPageActive(page);
			break;
		}
	}

	put_cpu_var(lru_add_pvec);
}

L
Linus Torvalds 已提交
363 364 365 366 367 368
/*
 * Mark a page as having seen activity.
 *
 * inactive,unreferenced	->	inactive,referenced
 * inactive,referenced		->	active,unreferenced
 * active,unreferenced		->	active,referenced
369 370 371
 *
 * When a newly allocated page is not yet visible, so safe for non-atomic ops,
 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
L
Linus Torvalds 已提交
372
 */
H
Harvey Harrison 已提交
373
void mark_page_accessed(struct page *page)
L
Linus Torvalds 已提交
374
{
375
	page = compound_head(page);
376

377 378 379 380 381 382 383 384 385
	if (!PageReferenced(page)) {
		SetPageReferenced(page);
	} else if (PageUnevictable(page)) {
		/*
		 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
		 * this list is never rotated or maintained, so marking an
		 * evictable page accessed has no effect.
		 */
	} else if (!PageActive(page)) {
386 387 388 389 390 391 392 393 394 395
		/*
		 * If the page is on the LRU, queue it for activation via
		 * activate_page_pvecs. Otherwise, assume the page is on a
		 * pagevec, mark it active and it'll be moved to the active
		 * LRU on the next drain.
		 */
		if (PageLRU(page))
			activate_page(page);
		else
			__lru_cache_activate_page(page);
L
Linus Torvalds 已提交
396
		ClearPageReferenced(page);
H
Huang Ying 已提交
397
		if (page_is_file_lru(page))
398
			workingset_activation(page);
L
Linus Torvalds 已提交
399
	}
400 401
	if (page_is_idle(page))
		clear_page_idle(page);
L
Linus Torvalds 已提交
402 403 404
}
EXPORT_SYMBOL(mark_page_accessed);

405
static void __lru_cache_add(struct page *page)
L
Linus Torvalds 已提交
406
{
407 408
	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);

409
	get_page(page);
410
	if (!pagevec_add(pvec, page) || PageCompound(page))
411
		__pagevec_lru_add(pvec);
412
	put_cpu_var(lru_add_pvec);
L
Linus Torvalds 已提交
413
}
414 415

/**
416
 * lru_cache_add_anon - add a page to the page lists
417 418 419 420
 * @page: the page to add
 */
void lru_cache_add_anon(struct page *page)
{
421 422
	if (PageActive(page))
		ClearPageActive(page);
423 424 425 426 427
	__lru_cache_add(page);
}

void lru_cache_add_file(struct page *page)
{
428 429
	if (PageActive(page))
		ClearPageActive(page);
430 431 432
	__lru_cache_add(page);
}
EXPORT_SYMBOL(lru_cache_add_file);
L
Linus Torvalds 已提交
433

434
/**
435
 * lru_cache_add - add a page to a page list
436
 * @page: the page to be added to the LRU.
437 438 439 440 441
 *
 * Queue the page for addition to the LRU via pagevec. The decision on whether
 * to add the page to the [in]active [file|anon] list is deferred until the
 * pagevec is drained. This gives a chance for the caller of lru_cache_add()
 * have the page added to the active list using mark_page_accessed().
442
 */
443
void lru_cache_add(struct page *page)
L
Linus Torvalds 已提交
444
{
445 446
	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
	VM_BUG_ON_PAGE(PageLRU(page), page);
447
	__lru_cache_add(page);
L
Linus Torvalds 已提交
448 449
}

450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
/**
 * lru_cache_add_active_or_unevictable
 * @page:  the page to be added to LRU
 * @vma:   vma in which page is mapped for determining reclaimability
 *
 * Place @page on the active or unevictable LRU list, depending on its
 * evictability.  Note that if the page is not evictable, it goes
 * directly back onto it's zone's unevictable list, it does NOT use a
 * per cpu pagevec.
 */
void lru_cache_add_active_or_unevictable(struct page *page,
					 struct vm_area_struct *vma)
{
	VM_BUG_ON_PAGE(PageLRU(page), page);

465
	if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
466
		SetPageActive(page);
467
	else if (!TestSetPageMlocked(page)) {
468 469 470 471 472 473 474 475 476
		/*
		 * We use the irq-unsafe __mod_zone_page_stat because this
		 * counter is not modified from interrupt context, and the pte
		 * lock is held(spinlock), which implies preemption disabled.
		 */
		__mod_zone_page_state(page_zone(page), NR_MLOCK,
				    hpage_nr_pages(page));
		count_vm_event(UNEVICTABLE_PGMLOCKED);
	}
477
	lru_cache_add(page);
478 479
}

M
Minchan Kim 已提交
480 481 482 483 484 485
/*
 * If the page can not be invalidated, it is moved to the
 * inactive list to speed up its reclaim.  It is moved to the
 * head of the list, rather than the tail, to give the flusher
 * threads some time to write it out, as this is much more
 * effective than the single-page writeout from reclaim.
M
Minchan Kim 已提交
486 487 488 489 490 491 492 493 494 495 496 497 498 499
 *
 * If the page isn't page_mapped and dirty/writeback, the page
 * could reclaim asap using PG_reclaim.
 *
 * 1. active, mapped page -> none
 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 * 3. inactive, mapped page -> none
 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 * 5. inactive, clean -> inactive, tail
 * 6. Others -> none
 *
 * In 4, why it moves inactive's head, the VM expects the page would
 * be write it out by flusher threads as this is much more effective
 * than the single-page writeout from reclaim.
M
Minchan Kim 已提交
500
 */
501
static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
502
			      void *arg)
M
Minchan Kim 已提交
503 504
{
	int lru, file;
M
Minchan Kim 已提交
505
	bool active;
M
Minchan Kim 已提交
506

M
Minchan Kim 已提交
507
	if (!PageLRU(page))
M
Minchan Kim 已提交
508 509
		return;

510 511 512
	if (PageUnevictable(page))
		return;

M
Minchan Kim 已提交
513 514 515 516
	/* Some processes are using the page */
	if (page_mapped(page))
		return;

M
Minchan Kim 已提交
517
	active = PageActive(page);
H
Huang Ying 已提交
518
	file = page_is_file_lru(page);
M
Minchan Kim 已提交
519
	lru = page_lru_base_type(page);
520 521

	del_page_from_lru_list(page, lruvec, lru + active);
M
Minchan Kim 已提交
522 523 524
	ClearPageActive(page);
	ClearPageReferenced(page);

M
Minchan Kim 已提交
525 526 527 528 529 530
	if (PageWriteback(page) || PageDirty(page)) {
		/*
		 * PG_reclaim could be raced with end_page_writeback
		 * It can make readahead confusing.  But race window
		 * is _really_ small and  it's non-critical problem.
		 */
531
		add_page_to_lru_list(page, lruvec, lru);
M
Minchan Kim 已提交
532 533 534 535 536 537
		SetPageReclaim(page);
	} else {
		/*
		 * The page's writeback ends up during pagevec
		 * We moves tha page into tail of inactive.
		 */
538
		add_page_to_lru_list_tail(page, lruvec, lru);
M
Minchan Kim 已提交
539 540 541 542 543
		__count_vm_event(PGROTATED);
	}

	if (active)
		__count_vm_event(PGDEACTIVATE);
544
	update_page_reclaim_stat(lruvec, file, 0);
M
Minchan Kim 已提交
545 546
}

M
Minchan Kim 已提交
547 548 549 550
static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
			    void *arg)
{
	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
H
Huang Ying 已提交
551
		int file = page_is_file_lru(page);
M
Minchan Kim 已提交
552 553 554 555 556 557 558 559 560 561 562
		int lru = page_lru_base_type(page);

		del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
		ClearPageActive(page);
		ClearPageReferenced(page);
		add_page_to_lru_list(page, lruvec, lru);

		__count_vm_events(PGDEACTIVATE, hpage_nr_pages(page));
		update_page_reclaim_stat(lruvec, file, 0);
	}
}
563

564
static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
565 566
			    void *arg)
{
567
	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
568
	    !PageSwapCache(page) && !PageUnevictable(page)) {
569
		bool active = PageActive(page);
570

571 572
		del_page_from_lru_list(page, lruvec,
				       LRU_INACTIVE_ANON + active);
573 574
		ClearPageActive(page);
		ClearPageReferenced(page);
575
		/*
H
Huang Ying 已提交
576 577 578
		 * Lazyfree pages are clean anonymous pages.  They have
		 * PG_swapbacked flag cleared, to distinguish them from normal
		 * anonymous pages
579 580 581
		 */
		ClearPageSwapBacked(page);
		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
582

583
		__count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
584
		count_memcg_page_event(page, PGLAZYFREE);
585
		update_page_reclaim_stat(lruvec, 1, 0);
586 587 588
	}
}

589 590 591 592 593
/*
 * Drain pages out of the cpu's pagevecs.
 * Either "cpu" is the current CPU, and preemption has already been
 * disabled; or "cpu" is being hot-unplugged, and is already dead.
 */
594
void lru_add_drain_cpu(int cpu)
L
Linus Torvalds 已提交
595
{
596
	struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
L
Linus Torvalds 已提交
597

598
	if (pagevec_count(pvec))
599
		__pagevec_lru_add(pvec);
600 601 602 603 604 605 606 607 608 609

	pvec = &per_cpu(lru_rotate_pvecs, cpu);
	if (pagevec_count(pvec)) {
		unsigned long flags;

		/* No harm done if a racing interrupt already did this */
		local_irq_save(flags);
		pagevec_move_tail(pvec);
		local_irq_restore(flags);
	}
M
Minchan Kim 已提交
610

611
	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
M
Minchan Kim 已提交
612
	if (pagevec_count(pvec))
613
		pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
614

M
Minchan Kim 已提交
615 616 617 618
	pvec = &per_cpu(lru_deactivate_pvecs, cpu);
	if (pagevec_count(pvec))
		pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);

619
	pvec = &per_cpu(lru_lazyfree_pvecs, cpu);
620
	if (pagevec_count(pvec))
621
		pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
622

623
	activate_page_drain(cpu);
M
Minchan Kim 已提交
624 625 626
}

/**
627
 * deactivate_file_page - forcefully deactivate a file page
M
Minchan Kim 已提交
628 629 630 631 632 633
 * @page: page to deactivate
 *
 * This function hints the VM that @page is a good reclaim candidate,
 * for example if its invalidation fails due to the page being dirty
 * or under writeback.
 */
634
void deactivate_file_page(struct page *page)
M
Minchan Kim 已提交
635
{
636
	/*
637 638
	 * In a workload with many unevictable page such as mprotect,
	 * unevictable page deactivation for accelerating reclaim is pointless.
639 640 641 642
	 */
	if (PageUnevictable(page))
		return;

M
Minchan Kim 已提交
643
	if (likely(get_page_unless_zero(page))) {
644
		struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
M
Minchan Kim 已提交
645

646
		if (!pagevec_add(pvec, page) || PageCompound(page))
647 648
			pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
		put_cpu_var(lru_deactivate_file_pvecs);
M
Minchan Kim 已提交
649
	}
650 651
}

M
Minchan Kim 已提交
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
/*
 * deactivate_page - deactivate a page
 * @page: page to deactivate
 *
 * deactivate_page() moves @page to the inactive list if @page was on the active
 * list and was not an unevictable page.  This is done to accelerate the reclaim
 * of @page.
 */
void deactivate_page(struct page *page)
{
	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);

		get_page(page);
		if (!pagevec_add(pvec, page) || PageCompound(page))
			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
		put_cpu_var(lru_deactivate_pvecs);
	}
}

672
/**
673
 * mark_page_lazyfree - make an anon page lazyfree
674 675
 * @page: page to deactivate
 *
676 677
 * mark_page_lazyfree() moves @page to the inactive file list.
 * This is done to accelerate the reclaim of @page.
678
 */
679
void mark_page_lazyfree(struct page *page)
680
{
681
	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
682
	    !PageSwapCache(page) && !PageUnevictable(page)) {
683
		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);
684

685
		get_page(page);
686
		if (!pagevec_add(pvec, page) || PageCompound(page))
687 688
			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
		put_cpu_var(lru_lazyfree_pvecs);
689 690 691
	}
}

692 693
void lru_add_drain(void)
{
694
	lru_add_drain_cpu(get_cpu());
695
	put_cpu();
L
Linus Torvalds 已提交
696 697
}

698 699 700 701
#ifdef CONFIG_SMP

static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);

D
David Howells 已提交
702
static void lru_add_drain_per_cpu(struct work_struct *dummy)
703 704 705 706
{
	lru_add_drain();
}

707 708 709 710 711 712 713 714
/*
 * Doesn't need any cpu hotplug locking because we do rely on per-cpu
 * kworkers being shut down before our page_alloc_cpu_dead callback is
 * executed on the offlined cpu.
 * Calling this function with cpu hotplug locks held can actually lead
 * to obscure indirect dependencies via WQ context.
 */
void lru_add_drain_all(void)
715
{
716
	static seqcount_t seqcount = SEQCNT_ZERO(seqcount);
717 718
	static DEFINE_MUTEX(lock);
	static struct cpumask has_work;
719
	int cpu, seq;
720

721 722 723 724 725 726 727
	/*
	 * Make sure nobody triggers this path before mm_percpu_wq is fully
	 * initialized.
	 */
	if (WARN_ON(!mm_percpu_wq))
		return;

728 729
	seq = raw_read_seqcount_latch(&seqcount);

730
	mutex_lock(&lock);
731 732 733 734 735 736 737 738 739 740

	/*
	 * Piggyback on drain started and finished while we waited for lock:
	 * all pages pended at the time of our enter were drained from vectors.
	 */
	if (__read_seqcount_retry(&seqcount, seq))
		goto done;

	raw_write_seqcount_latch(&seqcount);

741 742 743 744 745 746 747
	cpumask_clear(&has_work);

	for_each_online_cpu(cpu) {
		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);

		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
748
		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
M
Minchan Kim 已提交
749
		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
750
		    pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) ||
751 752
		    need_activate_page_drain(cpu)) {
			INIT_WORK(work, lru_add_drain_per_cpu);
753
			queue_work_on(cpu, mm_percpu_wq, work);
754 755 756 757 758 759 760
			cpumask_set_cpu(cpu, &has_work);
		}
	}

	for_each_cpu(cpu, &has_work)
		flush_work(&per_cpu(lru_add_drain_work, cpu));

761
done:
762
	mutex_unlock(&lock);
763
}
764 765 766 767 768 769
#else
void lru_add_drain_all(void)
{
	lru_add_drain();
}
#endif
770

771
/**
772
 * release_pages - batched put_page()
773 774
 * @pages: array of pages to release
 * @nr: number of pages
L
Linus Torvalds 已提交
775
 *
776 777
 * Decrement the reference count on all the pages in @pages.  If it
 * fell to zero, remove the page from the LRU and free it.
L
Linus Torvalds 已提交
778
 */
779
void release_pages(struct page **pages, int nr)
L
Linus Torvalds 已提交
780 781
{
	int i;
782
	LIST_HEAD(pages_to_free);
M
Mel Gorman 已提交
783
	struct pglist_data *locked_pgdat = NULL;
784
	struct lruvec *lruvec;
785
	unsigned long uninitialized_var(flags);
786
	unsigned int uninitialized_var(lock_batch);
L
Linus Torvalds 已提交
787 788 789 790

	for (i = 0; i < nr; i++) {
		struct page *page = pages[i];

791 792 793
		/*
		 * Make sure the IRQ-safe lock-holding time does not get
		 * excessive with a continuous string of pages from the
M
Mel Gorman 已提交
794
		 * same pgdat. The lock is held only if pgdat != NULL.
795
		 */
M
Mel Gorman 已提交
796 797 798
		if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) {
			spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
			locked_pgdat = NULL;
799 800
		}

801
		if (is_huge_zero_page(page))
802 803
			continue;

804
		if (is_zone_device_page(page)) {
805 806 807 808 809
			if (locked_pgdat) {
				spin_unlock_irqrestore(&locked_pgdat->lru_lock,
						       flags);
				locked_pgdat = NULL;
			}
810 811 812 813 814 815
			/*
			 * ZONE_DEVICE pages that return 'false' from
			 * put_devmap_managed_page() do not require special
			 * processing, and instead, expect a call to
			 * put_page_testzero().
			 */
816 817
			if (page_is_devmap_managed(page)) {
				put_devmap_managed_page(page);
818
				continue;
819
			}
820 821
		}

822
		page = compound_head(page);
N
Nick Piggin 已提交
823
		if (!put_page_testzero(page))
L
Linus Torvalds 已提交
824 825
			continue;

826
		if (PageCompound(page)) {
M
Mel Gorman 已提交
827 828 829
			if (locked_pgdat) {
				spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
				locked_pgdat = NULL;
830 831 832 833 834
			}
			__put_compound_page(page);
			continue;
		}

835
		if (PageLRU(page)) {
M
Mel Gorman 已提交
836
			struct pglist_data *pgdat = page_pgdat(page);
L
Lee Schermerhorn 已提交
837

M
Mel Gorman 已提交
838 839 840
			if (pgdat != locked_pgdat) {
				if (locked_pgdat)
					spin_unlock_irqrestore(&locked_pgdat->lru_lock,
841
									flags);
842
				lock_batch = 0;
M
Mel Gorman 已提交
843 844
				locked_pgdat = pgdat;
				spin_lock_irqsave(&locked_pgdat->lru_lock, flags);
845
			}
846

M
Mel Gorman 已提交
847
			lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);
848
			VM_BUG_ON_PAGE(!PageLRU(page), page);
N
Nick Piggin 已提交
849
			__ClearPageLRU(page);
850
			del_page_from_lru_list(page, lruvec, page_off_lru(page));
851 852
		}

853
		/* Clear Active bit in case of parallel mark_page_accessed */
854
		__ClearPageActive(page);
855
		__ClearPageWaiters(page);
856

857
		list_add(&page->lru, &pages_to_free);
L
Linus Torvalds 已提交
858
	}
M
Mel Gorman 已提交
859 860
	if (locked_pgdat)
		spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
L
Linus Torvalds 已提交
861

862
	mem_cgroup_uncharge_list(&pages_to_free);
863
	free_unref_page_list(&pages_to_free);
L
Linus Torvalds 已提交
864
}
M
Miklos Szeredi 已提交
865
EXPORT_SYMBOL(release_pages);
L
Linus Torvalds 已提交
866 867 868 869 870 871 872 873 874 875 876 877 878

/*
 * The pages which we're about to release may be in the deferred lru-addition
 * queues.  That would prevent them from really being freed right now.  That's
 * OK from a correctness point of view but is inefficient - those pages may be
 * cache-warm and we want to give them back to the page allocator ASAP.
 *
 * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 * and __pagevec_lru_add_active() call release_pages() directly to avoid
 * mutual recursion.
 */
void __pagevec_release(struct pagevec *pvec)
{
879
	if (!pvec->percpu_pvec_drained) {
880
		lru_add_drain();
881
		pvec->percpu_pvec_drained = true;
882
	}
883
	release_pages(pvec->pages, pagevec_count(pvec));
L
Linus Torvalds 已提交
884 885
	pagevec_reinit(pvec);
}
886 887
EXPORT_SYMBOL(__pagevec_release);

888
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
889
/* used by __split_huge_page_refcount() */
890
void lru_add_page_tail(struct page *page, struct page *page_tail,
891
		       struct lruvec *lruvec, struct list_head *list)
892 893 894
{
	const int file = 0;

895 896 897
	VM_BUG_ON_PAGE(!PageHead(page), page);
	VM_BUG_ON_PAGE(PageCompound(page_tail), page);
	VM_BUG_ON_PAGE(PageLRU(page_tail), page);
898
	lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock);
899

900 901
	if (!list)
		SetPageLRU(page_tail);
902

903 904
	if (likely(PageLRU(page)))
		list_add_tail(&page_tail->lru, &page->lru);
905 906 907 908 909
	else if (list) {
		/* page reclaim is reclaiming a huge page */
		get_page(page_tail);
		list_add_tail(&page_tail->lru, list);
	} else {
910 911 912 913
		/*
		 * Head page has not yet been counted, as an hpage,
		 * so we must account for each subpage individually.
		 *
914 915
		 * Put page_tail on the list at the correct position
		 * so they all end up in order.
916
		 */
917 918
		add_page_to_lru_list_tail(page_tail, lruvec,
					  page_lru(page_tail));
919
	}
920 921

	if (!PageUnevictable(page))
922
		update_page_reclaim_stat(lruvec, file, PageActive(page_tail));
923
}
924
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
925

926 927
static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
				 void *arg)
S
Shaohua Li 已提交
928
{
929 930
	enum lru_list lru;
	int was_unevictable = TestClearPageUnevictable(page);
S
Shaohua Li 已提交
931

932
	VM_BUG_ON_PAGE(PageLRU(page), page);
S
Shaohua Li 已提交
933

934 935
	/*
	 * Page becomes evictable in two ways:
936
	 * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()].
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
	 * 2) Before acquiring LRU lock to put the page to correct LRU and then
	 *   a) do PageLRU check with lock [check_move_unevictable_pages]
	 *   b) do PageLRU check before lock [clear_page_mlock]
	 *
	 * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need
	 * following strict ordering:
	 *
	 * #0: __pagevec_lru_add_fn		#1: clear_page_mlock
	 *
	 * SetPageLRU()				TestClearPageMlocked()
	 * smp_mb() // explicit ordering	// above provides strict
	 *					// ordering
	 * PageMlocked()			PageLRU()
	 *
	 *
	 * if '#1' does not observe setting of PG_lru by '#0' and fails
	 * isolation, the explicit barrier will make sure that page_evictable
	 * check will put the page in correct LRU. Without smp_mb(), SetPageLRU
	 * can be reordered after PageMlocked check and can make '#1' to fail
	 * the isolation of the page whose Mlocked bit is cleared (#0 is also
	 * looking at the same page) and the evictable page will be stranded
	 * in an unevictable LRU.
	 */
960 961
	SetPageLRU(page);
	smp_mb__after_atomic();
962 963 964

	if (page_evictable(page)) {
		lru = page_lru(page);
H
Huang Ying 已提交
965
		update_page_reclaim_stat(lruvec, page_is_file_lru(page),
966 967 968 969 970 971 972 973 974 975 976
					 PageActive(page));
		if (was_unevictable)
			count_vm_event(UNEVICTABLE_PGRESCUED);
	} else {
		lru = LRU_UNEVICTABLE;
		ClearPageActive(page);
		SetPageUnevictable(page);
		if (!was_unevictable)
			count_vm_event(UNEVICTABLE_PGCULLED);
	}

977
	add_page_to_lru_list(page, lruvec, lru);
978
	trace_mm_lru_insertion(page, lru);
S
Shaohua Li 已提交
979 980
}

L
Linus Torvalds 已提交
981 982 983 984
/*
 * Add the passed pages to the LRU, then drop the caller's refcount
 * on them.  Reinitialises the caller's pagevec.
 */
985
void __pagevec_lru_add(struct pagevec *pvec)
L
Linus Torvalds 已提交
986
{
987
	pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
L
Linus Torvalds 已提交
988 989
}

990 991 992 993 994
/**
 * pagevec_lookup_entries - gang pagecache lookup
 * @pvec:	Where the resulting entries are placed
 * @mapping:	The address_space to search
 * @start:	The starting entry index
995
 * @nr_entries:	The maximum number of pages
996 997 998
 * @indices:	The cache indices corresponding to the entries in @pvec
 *
 * pagevec_lookup_entries() will search for and return a group of up
999
 * to @nr_pages pages and shadow entries in the mapping.  All
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
 * entries are placed in @pvec.  pagevec_lookup_entries() takes a
 * reference against actual pages in @pvec.
 *
 * The search returns a group of mapping-contiguous entries with
 * ascending indexes.  There may be holes in the indices due to
 * not-present entries.
 *
 * pagevec_lookup_entries() returns the number of entries which were
 * found.
 */
unsigned pagevec_lookup_entries(struct pagevec *pvec,
				struct address_space *mapping,
1012
				pgoff_t start, unsigned nr_entries,
1013 1014
				pgoff_t *indices)
{
1015
	pvec->nr = find_get_entries(mapping, start, nr_entries,
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
				    pvec->pages, indices);
	return pagevec_count(pvec);
}

/**
 * pagevec_remove_exceptionals - pagevec exceptionals pruning
 * @pvec:	The pagevec to prune
 *
 * pagevec_lookup_entries() fills both pages and exceptional radix
 * tree entries into the pagevec.  This function prunes all
 * exceptionals from @pvec without leaving holes, so that it can be
 * passed on to page-only pagevec operations.
 */
void pagevec_remove_exceptionals(struct pagevec *pvec)
{
	int i, j;

	for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
		struct page *page = pvec->pages[i];
1035
		if (!xa_is_value(page))
1036 1037 1038 1039 1040
			pvec->pages[j++] = page;
	}
	pvec->nr = j;
}

L
Linus Torvalds 已提交
1041
/**
J
Jan Kara 已提交
1042
 * pagevec_lookup_range - gang pagecache lookup
L
Linus Torvalds 已提交
1043 1044 1045
 * @pvec:	Where the resulting pages are placed
 * @mapping:	The address_space to search
 * @start:	The starting page index
J
Jan Kara 已提交
1046
 * @end:	The final page index
L
Linus Torvalds 已提交
1047
 *
1048
 * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE
J
Jan Kara 已提交
1049 1050
 * pages in the mapping starting from index @start and upto index @end
 * (inclusive).  The pages are placed in @pvec.  pagevec_lookup() takes a
L
Linus Torvalds 已提交
1051 1052 1053
 * reference against the pages in @pvec.
 *
 * The search returns a group of mapping-contiguous pages with ascending
1054 1055
 * indexes.  There may be holes in the indices due to not-present pages. We
 * also update @start to index the next page for the traversal.
L
Linus Torvalds 已提交
1056
 *
J
Jan Kara 已提交
1057
 * pagevec_lookup_range() returns the number of pages which were found. If this
1058
 * number is smaller than PAGEVEC_SIZE, the end of specified range has been
J
Jan Kara 已提交
1059
 * reached.
L
Linus Torvalds 已提交
1060
 */
J
Jan Kara 已提交
1061
unsigned pagevec_lookup_range(struct pagevec *pvec,
1062
		struct address_space *mapping, pgoff_t *start, pgoff_t end)
L
Linus Torvalds 已提交
1063
{
1064
	pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE,
J
Jan Kara 已提交
1065
					pvec->pages);
L
Linus Torvalds 已提交
1066 1067
	return pagevec_count(pvec);
}
J
Jan Kara 已提交
1068
EXPORT_SYMBOL(pagevec_lookup_range);
1069

1070 1071
unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
		struct address_space *mapping, pgoff_t *index, pgoff_t end,
M
Matthew Wilcox 已提交
1072
		xa_mark_t tag)
L
Linus Torvalds 已提交
1073
{
1074
	pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
1075
					PAGEVEC_SIZE, pvec->pages);
L
Linus Torvalds 已提交
1076 1077
	return pagevec_count(pvec);
}
1078
EXPORT_SYMBOL(pagevec_lookup_range_tag);
L
Linus Torvalds 已提交
1079

1080 1081
unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,
		struct address_space *mapping, pgoff_t *index, pgoff_t end,
M
Matthew Wilcox 已提交
1082
		xa_mark_t tag, unsigned max_pages)
1083 1084 1085 1086 1087 1088
{
	pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
		min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages);
	return pagevec_count(pvec);
}
EXPORT_SYMBOL(pagevec_lookup_range_nr_tag);
L
Linus Torvalds 已提交
1089 1090 1091 1092 1093
/*
 * Perform any setup for the swap system
 */
void __init swap_setup(void)
{
1094
	unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT);
P
Peter Zijlstra 已提交
1095

L
Linus Torvalds 已提交
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	/* Use a smaller cluster for small-memory machines */
	if (megs < 16)
		page_cluster = 2;
	else
		page_cluster = 3;
	/*
	 * Right now other parts of the system means that we
	 * _really_ don't want to cluster much more
	 */
}
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128

#ifdef CONFIG_DEV_PAGEMAP_OPS
void put_devmap_managed_page(struct page *page)
{
	int count;

	if (WARN_ON_ONCE(!page_is_devmap_managed(page)))
		return;

	count = page_ref_dec_return(page);

	/*
	 * devmap page refcounts are 1-based, rather than 0-based: if
	 * refcount is 1, then the page is free and the refcount is
	 * stable because nobody holds a reference on the page.
	 */
	if (count == 1)
		free_devmap_managed_page(page);
	else if (!count)
		__put_page(page);
}
EXPORT_SYMBOL(put_devmap_managed_page);
#endif