migrate.c 67.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Memory Migration functionality - linux/mm/migrate.c
4 5 6 7 8 9 10 11 12
 *
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 *
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
C
Christoph Lameter 已提交
13
 * Christoph Lameter
14 15 16
 */

#include <linux/migrate.h>
17
#include <linux/export.h>
18
#include <linux/swap.h>
19
#include <linux/swapops.h>
20
#include <linux/pagemap.h>
21
#include <linux/buffer_head.h>
22
#include <linux/mm_inline.h>
23
#include <linux/nsproxy.h>
24
#include <linux/pagevec.h>
25
#include <linux/ksm.h>
26 27 28 29
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
30
#include <linux/writeback.h>
31 32
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
33
#include <linux/security.h>
34
#include <linux/backing-dev.h>
35
#include <linux/compaction.h>
36
#include <linux/syscalls.h>
37
#include <linux/compat.h>
38
#include <linux/hugetlb.h>
39
#include <linux/hugetlb_cgroup.h>
40
#include <linux/gfp.h>
41
#include <linux/pfn_t.h>
42
#include <linux/memremap.h>
43
#include <linux/userfaultfd_k.h>
44
#include <linux/balloon_compaction.h>
45
#include <linux/page_idle.h>
46
#include <linux/page_owner.h>
47
#include <linux/sched/mm.h>
48
#include <linux/ptrace.h>
49
#include <linux/oom.h>
50
#include <linux/memory.h>
51
#include <linux/random.h>
52

53 54
#include <asm/tlbflush.h>

55 56 57
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>

58 59
#include "internal.h"

60
int isolate_movable_page(struct page *page, isolate_mode_t mode)
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
{
	struct address_space *mapping;

	/*
	 * Avoid burning cycles with pages that are yet under __free_pages(),
	 * or just got freed under us.
	 *
	 * In case we 'win' a race for a movable page being freed under us and
	 * raise its refcount preventing __free_pages() from doing its job
	 * the put_page() at the end of this block will take care of
	 * release this page, thus avoiding a nasty leakage.
	 */
	if (unlikely(!get_page_unless_zero(page)))
		goto out;

	/*
	 * Check PageMovable before holding a PG_lock because page's owner
	 * assumes anybody doesn't touch PG_lock of newly allocated page
79
	 * so unconditionally grabbing the lock ruins page's owner side.
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
	 */
	if (unlikely(!__PageMovable(page)))
		goto out_putpage;
	/*
	 * As movable pages are not isolated from LRU lists, concurrent
	 * compaction threads can race against page migration functions
	 * as well as race against the releasing a page.
	 *
	 * In order to avoid having an already isolated movable page
	 * being (wrongly) re-isolated while it is under migration,
	 * or to avoid attempting to isolate pages being released,
	 * lets be sure we have the page lock
	 * before proceeding with the movable page isolation steps.
	 */
	if (unlikely(!trylock_page(page)))
		goto out_putpage;

	if (!PageMovable(page) || PageIsolated(page))
		goto out_no_isolated;

	mapping = page_mapping(page);
	VM_BUG_ON_PAGE(!mapping, page);

	if (!mapping->a_ops->isolate_page(page, mode))
		goto out_no_isolated;

	/* Driver shouldn't use PG_isolated bit of page->flags */
	WARN_ON_ONCE(PageIsolated(page));
	__SetPageIsolated(page);
	unlock_page(page);

111
	return 0;
112 113 114 115 116 117

out_no_isolated:
	unlock_page(page);
out_putpage:
	put_page(page);
out:
118
	return -EBUSY;
119 120
}

121
static void putback_movable_page(struct page *page)
122 123 124 125 126 127 128 129
{
	struct address_space *mapping;

	mapping = page_mapping(page);
	mapping->a_ops->putback_page(page);
	__ClearPageIsolated(page);
}

130 131 132 133
/*
 * Put previously isolated pages back onto the appropriate lists
 * from where they were once taken off for compaction/migration.
 *
134 135 136
 * This function shall be used whenever the isolated pageset has been
 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 * and isolate_huge_page().
137 138 139 140 141 142
 */
void putback_movable_pages(struct list_head *l)
{
	struct page *page;
	struct page *page2;

143
	list_for_each_entry_safe(page, page2, l, lru) {
144 145 146 147
		if (unlikely(PageHuge(page))) {
			putback_active_hugepage(page);
			continue;
		}
148
		list_del(&page->lru);
149 150 151 152 153
		/*
		 * We isolated non-lru movable page so here we can use
		 * __PageMovable because LRU page's mapping cannot have
		 * PAGE_MAPPING_MOVABLE.
		 */
154
		if (unlikely(__PageMovable(page))) {
155 156 157 158 159 160 161 162 163
			VM_BUG_ON_PAGE(!PageIsolated(page), page);
			lock_page(page);
			if (PageMovable(page))
				putback_movable_page(page);
			else
				__ClearPageIsolated(page);
			unlock_page(page);
			put_page(page);
		} else {
164
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
165
					page_is_file_lru(page), -thp_nr_pages(page));
166
			putback_lru_page(page);
167
		}
168 169 170
	}
}

171 172 173
/*
 * Restore a potential migration pte to a working pte entry
 */
174
static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
175
				 unsigned long addr, void *old)
176
{
177 178
	struct folio *folio = page_folio(page);
	DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
179

180 181
	VM_BUG_ON_PAGE(PageTail(page), page);
	while (page_vma_mapped_walk(&pvmw)) {
182 183 184 185 186 187 188 189 190
		pte_t pte;
		swp_entry_t entry;
		struct page *new;
		unsigned long idx = 0;

		/* pgoff is invalid for ksm pages, but they are never large */
		if (folio_test_large(folio) && !folio_test_hugetlb(folio))
			idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
		new = folio_page(folio, idx);
191

192 193 194
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
		/* PMD-mapped THP migration entry */
		if (!pvmw.pte) {
195 196
			VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
					!folio_test_pmd_mappable(folio), folio);
197 198 199 200 201
			remove_migration_pmd(&pvmw, new);
			continue;
		}
#endif

202
		folio_get(folio);
203 204 205
		pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
		if (pte_swp_soft_dirty(*pvmw.pte))
			pte = pte_mksoft_dirty(pte);
206

207 208 209 210
		/*
		 * Recheck VMA as permissions can change since migration started
		 */
		entry = pte_to_swp_entry(*pvmw.pte);
211
		if (is_writable_migration_entry(entry))
212
			pte = maybe_mkwrite(pte, vma);
213 214
		else if (pte_swp_uffd_wp(*pvmw.pte))
			pte = pte_mkuffd_wp(pte);
215

216
		if (unlikely(is_device_private_page(new))) {
217 218 219 220 221 222
			if (pte_write(pte))
				entry = make_writable_device_private_entry(
							page_to_pfn(new));
			else
				entry = make_readable_device_private_entry(
							page_to_pfn(new));
223
			pte = swp_entry_to_pte(entry);
224 225
			if (pte_swp_soft_dirty(*pvmw.pte))
				pte = pte_swp_mksoft_dirty(pte);
226 227
			if (pte_swp_uffd_wp(*pvmw.pte))
				pte = pte_swp_mkuffd_wp(pte);
228
		}
229

230
#ifdef CONFIG_HUGETLB_PAGE
231
		if (folio_test_hugetlb(folio)) {
232 233
			unsigned int shift = huge_page_shift(hstate_vma(vma));

234
			pte = pte_mkhuge(pte);
235
			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
236
			if (folio_test_anon(folio))
237 238 239
				hugepage_add_anon_rmap(new, vma, pvmw.address);
			else
				page_dup_rmap(new, true);
240
			set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
241 242 243
		} else
#endif
		{
244
			if (folio_test_anon(folio))
245 246
				page_add_anon_rmap(new, vma, pvmw.address, false);
			else
247
				page_add_file_rmap(new, vma, false);
248
			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
249
		}
250 251
		if (vma->vm_flags & VM_LOCKED)
			mlock_page_drain(smp_processor_id());
252

253 254 255
		/* No need to invalidate - it was non-present before */
		update_mmu_cache(vma, pvmw.address, pvmw.pte);
	}
256

257
	return true;
258 259
}

260 261 262 263
/*
 * Get rid of all migration entries and replace them by
 * references to the indicated page.
 */
264
void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked)
265
{
266 267
	struct rmap_walk_control rwc = {
		.rmap_one = remove_migration_pte,
268
		.arg = src,
269 270
	};

271
	if (locked)
272
		rmap_walk_locked(&dst->page, &rwc);
273
	else
274
		rmap_walk(&dst->page, &rwc);
275 276
}

277 278 279 280 281
/*
 * Something used the pte of a page under migration. We need to
 * get to the page and wait until migration is finished.
 * When we return from this function the fault will be retried.
 */
282
void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
283
				spinlock_t *ptl)
284
{
285
	pte_t pte;
286 287
	swp_entry_t entry;

288
	spin_lock(ptl);
289 290 291 292 293 294 295 296
	pte = *ptep;
	if (!is_swap_pte(pte))
		goto out;

	entry = pte_to_swp_entry(pte);
	if (!is_migration_entry(entry))
		goto out;

297
	migration_entry_wait_on_locked(entry, ptep, ptl);
298 299 300 301 302
	return;
out:
	pte_unmap_unlock(ptep, ptl);
}

303 304 305 306 307 308 309 310
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
				unsigned long address)
{
	spinlock_t *ptl = pte_lockptr(mm, pmd);
	pte_t *ptep = pte_offset_map(pmd, address);
	__migration_entry_wait(mm, ptep, ptl);
}

311 312
void migration_entry_wait_huge(struct vm_area_struct *vma,
		struct mm_struct *mm, pte_t *pte)
313
{
314
	spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
315 316 317
	__migration_entry_wait(mm, pte, ptl);
}

318 319 320 321 322 323 324 325
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
{
	spinlock_t *ptl;

	ptl = pmd_lock(mm, pmd);
	if (!is_pmd_migration_entry(*pmd))
		goto unlock;
326
	migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl);
327 328 329 330 331 332
	return;
unlock:
	spin_unlock(ptl);
}
#endif

333
static int expected_page_refs(struct address_space *mapping, struct page *page)
334 335 336
{
	int expected_count = 1;

337
	if (mapping)
338
		expected_count += compound_nr(page) + page_has_private(page);
339 340 341
	return expected_count;
}

342
/*
343
 * Replace the page in the mapping.
344 345 346 347
 *
 * The number of remaining references must be:
 * 1 for anonymous pages without a mapping
 * 2 for pages with a mapping
348
 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
349
 */
350 351
int folio_migrate_mapping(struct address_space *mapping,
		struct folio *newfolio, struct folio *folio, int extra_count)
352
{
353
	XA_STATE(xas, &mapping->i_pages, folio_index(folio));
354 355
	struct zone *oldzone, *newzone;
	int dirty;
356 357
	int expected_count = expected_page_refs(mapping, &folio->page) + extra_count;
	long nr = folio_nr_pages(folio);
358

359
	if (!mapping) {
360
		/* Anonymous page without mapping */
361
		if (folio_ref_count(folio) != expected_count)
362
			return -EAGAIN;
363 364

		/* No turning back from here */
365 366 367 368
		newfolio->index = folio->index;
		newfolio->mapping = folio->mapping;
		if (folio_test_swapbacked(folio))
			__folio_set_swapbacked(newfolio);
369

370
		return MIGRATEPAGE_SUCCESS;
371 372
	}

373 374
	oldzone = folio_zone(folio);
	newzone = folio_zone(newfolio);
375

376
	xas_lock_irq(&xas);
377
	if (!folio_ref_freeze(folio, expected_count)) {
378
		xas_unlock_irq(&xas);
379 380 381
		return -EAGAIN;
	}

382
	/*
383
	 * Now we know that no one else is looking at the folio:
384
	 * no turning back from here.
385
	 */
386 387 388 389 390 391 392 393
	newfolio->index = folio->index;
	newfolio->mapping = folio->mapping;
	folio_ref_add(newfolio, nr); /* add cache reference */
	if (folio_test_swapbacked(folio)) {
		__folio_set_swapbacked(newfolio);
		if (folio_test_swapcache(folio)) {
			folio_set_swapcache(newfolio);
			newfolio->private = folio_get_private(folio);
394 395
		}
	} else {
396
		VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
397 398
	}

399
	/* Move dirty while page refs frozen and newpage not yet exposed */
400
	dirty = folio_test_dirty(folio);
401
	if (dirty) {
402 403
		folio_clear_dirty(folio);
		folio_set_dirty(newfolio);
404 405
	}

406
	xas_store(&xas, newfolio);
407 408

	/*
409 410
	 * Drop cache reference from old page by unfreezing
	 * to one less reference.
411 412
	 * We know this isn't the last reference.
	 */
413
	folio_ref_unfreeze(folio, expected_count - nr);
414

415
	xas_unlock(&xas);
416 417
	/* Leave irq disabled to prevent preemption while updating stats */

418 419 420 421 422 423 424
	/*
	 * If moved to a different zone then also account
	 * the page for that zone. Other VM counters will be
	 * taken care of when we establish references to the
	 * new page and drop references to the old page.
	 *
	 * Note that anonymous pages are accounted for
425
	 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
426 427
	 * are mapped to swap space.
	 */
428
	if (newzone != oldzone) {
429 430 431
		struct lruvec *old_lruvec, *new_lruvec;
		struct mem_cgroup *memcg;

432
		memcg = folio_memcg(folio);
433 434 435
		old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
		new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);

436 437
		__mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
		__mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
438
		if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) {
439 440
			__mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
			__mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
441
		}
442
#ifdef CONFIG_SWAP
443
		if (folio_test_swapcache(folio)) {
444 445 446 447
			__mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
			__mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
		}
#endif
448
		if (dirty && mapping_can_writeback(mapping)) {
449 450 451 452
			__mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
			__mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
			__mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
			__mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
453
		}
454
	}
455
	local_irq_enable();
456

457
	return MIGRATEPAGE_SUCCESS;
458
}
459
EXPORT_SYMBOL(folio_migrate_mapping);
460

461 462
/*
 * The expected number of remaining references is the same as that
463
 * of folio_migrate_mapping().
464 465 466 467
 */
int migrate_huge_page_move_mapping(struct address_space *mapping,
				   struct page *newpage, struct page *page)
{
468
	XA_STATE(xas, &mapping->i_pages, page_index(page));
469 470
	int expected_count;

471
	xas_lock_irq(&xas);
472
	expected_count = 2 + page_has_private(page);
473 474
	if (page_count(page) != expected_count || xas_load(&xas) != page) {
		xas_unlock_irq(&xas);
475 476 477
		return -EAGAIN;
	}

478
	if (!page_ref_freeze(page, expected_count)) {
479
		xas_unlock_irq(&xas);
480 481 482
		return -EAGAIN;
	}

483 484
	newpage->index = page->index;
	newpage->mapping = page->mapping;
485

486 487
	get_page(newpage);

488
	xas_store(&xas, newpage);
489

490
	page_ref_unfreeze(page, expected_count - 1);
491

492
	xas_unlock_irq(&xas);
493

494
	return MIGRATEPAGE_SUCCESS;
495 496
}

497
/*
498
 * Copy the flags and some other ancillary information
499
 */
500
void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
501
{
502 503
	int cpupid;

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
	if (folio_test_error(folio))
		folio_set_error(newfolio);
	if (folio_test_referenced(folio))
		folio_set_referenced(newfolio);
	if (folio_test_uptodate(folio))
		folio_mark_uptodate(newfolio);
	if (folio_test_clear_active(folio)) {
		VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio);
		folio_set_active(newfolio);
	} else if (folio_test_clear_unevictable(folio))
		folio_set_unevictable(newfolio);
	if (folio_test_workingset(folio))
		folio_set_workingset(newfolio);
	if (folio_test_checked(folio))
		folio_set_checked(newfolio);
	if (folio_test_mappedtodisk(folio))
		folio_set_mappedtodisk(newfolio);
521

522
	/* Move dirty on pages not done by folio_migrate_mapping() */
523 524
	if (folio_test_dirty(folio))
		folio_set_dirty(newfolio);
525

526 527 528 529
	if (folio_test_young(folio))
		folio_set_young(newfolio);
	if (folio_test_idle(folio))
		folio_set_idle(newfolio);
530

531 532 533 534
	/*
	 * Copy NUMA information to the new page, to prevent over-eager
	 * future migrations of this same page.
	 */
535 536
	cpupid = page_cpupid_xchg_last(&folio->page, -1);
	page_cpupid_xchg_last(&newfolio->page, cpupid);
537

538
	folio_migrate_ksm(newfolio, folio);
539 540 541 542
	/*
	 * Please do not reorder this without considering how mm/ksm.c's
	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
	 */
543 544 545
	if (folio_test_swapcache(folio))
		folio_clear_swapcache(folio);
	folio_clear_private(folio);
546 547

	/* page->private contains hugetlb specific flags */
548 549
	if (!folio_test_hugetlb(folio))
		folio->private = NULL;
550 551 552 553 554

	/*
	 * If any waiters have accumulated on the new page then
	 * wake them up.
	 */
555 556
	if (folio_test_writeback(newfolio))
		folio_end_writeback(newfolio);
557

558 559 560 561 562
	/*
	 * PG_readahead shares the same bit with PG_reclaim.  The above
	 * end_page_writeback() may clear PG_readahead mistakenly, so set the
	 * bit after that.
	 */
563 564
	if (folio_test_readahead(folio))
		folio_set_readahead(newfolio);
565

566
	folio_copy_owner(newfolio, folio);
567

568
	if (!folio_test_hugetlb(folio))
569
		mem_cgroup_migrate(folio, newfolio);
570
}
571
EXPORT_SYMBOL(folio_migrate_flags);
572

573
void folio_migrate_copy(struct folio *newfolio, struct folio *folio)
574
{
575 576
	folio_copy(newfolio, folio);
	folio_migrate_flags(newfolio, folio);
577
}
578
EXPORT_SYMBOL(folio_migrate_copy);
579

580 581 582 583
/************************************************************
 *                    Migration functions
 ***********************************************************/

584
/*
585
 * Common logic to directly migrate a single LRU page suitable for
586
 * pages that do not use PagePrivate/PagePrivate2.
587 588 589
 *
 * Pages are locked upon entry and exit.
 */
590
int migrate_page(struct address_space *mapping,
591 592
		struct page *newpage, struct page *page,
		enum migrate_mode mode)
593
{
594 595
	struct folio *newfolio = page_folio(newpage);
	struct folio *folio = page_folio(page);
596 597
	int rc;

598
	BUG_ON(folio_test_writeback(folio));	/* Writeback must be complete */
599

600
	rc = folio_migrate_mapping(mapping, newfolio, folio, 0);
601

602
	if (rc != MIGRATEPAGE_SUCCESS)
603 604
		return rc;

605
	if (mode != MIGRATE_SYNC_NO_COPY)
606
		folio_migrate_copy(newfolio, folio);
607
	else
608
		folio_migrate_flags(newfolio, folio);
609
	return MIGRATEPAGE_SUCCESS;
610 611 612
}
EXPORT_SYMBOL(migrate_page);

613
#ifdef CONFIG_BLOCK
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
							enum migrate_mode mode)
{
	struct buffer_head *bh = head;

	/* Simple case, sync compaction */
	if (mode != MIGRATE_ASYNC) {
		do {
			lock_buffer(bh);
			bh = bh->b_this_page;

		} while (bh != head);

		return true;
	}

	/* async case, we cannot block on lock_buffer so use trylock_buffer */
	do {
		if (!trylock_buffer(bh)) {
			/*
			 * We failed to lock the buffer and cannot stall in
			 * async migration. Release the taken locks
			 */
			struct buffer_head *failed_bh = bh;
			bh = head;
			while (bh != failed_bh) {
				unlock_buffer(bh);
				bh = bh->b_this_page;
			}
			return false;
		}

		bh = bh->b_this_page;
	} while (bh != head);
	return true;
}

652 653 654
static int __buffer_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode,
		bool check_refs)
655 656 657
{
	struct buffer_head *bh, *head;
	int rc;
658
	int expected_count;
659 660

	if (!page_has_buffers(page))
661
		return migrate_page(mapping, newpage, page, mode);
662

663
	/* Check whether page does not have extra refs before we do more work */
664
	expected_count = expected_page_refs(mapping, page);
665 666
	if (page_count(page) != expected_count)
		return -EAGAIN;
667

668 669 670
	head = page_buffers(page);
	if (!buffer_migrate_lock_buffers(head, mode))
		return -EAGAIN;
671

672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
	if (check_refs) {
		bool busy;
		bool invalidated = false;

recheck_buffers:
		busy = false;
		spin_lock(&mapping->private_lock);
		bh = head;
		do {
			if (atomic_read(&bh->b_count)) {
				busy = true;
				break;
			}
			bh = bh->b_this_page;
		} while (bh != head);
		if (busy) {
			if (invalidated) {
				rc = -EAGAIN;
				goto unlock_buffers;
			}
692
			spin_unlock(&mapping->private_lock);
693 694 695 696 697 698
			invalidate_bh_lrus();
			invalidated = true;
			goto recheck_buffers;
		}
	}

699
	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
700
	if (rc != MIGRATEPAGE_SUCCESS)
701
		goto unlock_buffers;
702

703
	attach_page_private(newpage, detach_page_private(page));
704 705 706 707 708 709 710 711

	bh = head;
	do {
		set_bh_page(bh, newpage, bh_offset(bh));
		bh = bh->b_this_page;

	} while (bh != head);

712 713 714 715
	if (mode != MIGRATE_SYNC_NO_COPY)
		migrate_page_copy(newpage, page);
	else
		migrate_page_states(newpage, page);
716

717 718
	rc = MIGRATEPAGE_SUCCESS;
unlock_buffers:
719 720
	if (check_refs)
		spin_unlock(&mapping->private_lock);
721 722 723 724 725 726 727
	bh = head;
	do {
		unlock_buffer(bh);
		bh = bh->b_this_page;

	} while (bh != head);

728
	return rc;
729
}
730 731 732 733 734 735 736 737 738 739 740

/*
 * Migration function for pages with buffers. This function can only be used
 * if the underlying filesystem guarantees that no other references to "page"
 * exist. For example attached buffer heads are accessed only under page lock.
 */
int buffer_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	return __buffer_migrate_page(mapping, newpage, page, mode, false);
}
741
EXPORT_SYMBOL(buffer_migrate_page);
742 743 744 745 746 747 748 749 750 751 752 753

/*
 * Same as above except that this variant is more careful and checks that there
 * are also no buffer head references. This function is the right one for
 * mappings where buffer heads are directly looked up and referenced (such as
 * block device mappings).
 */
int buffer_migrate_page_norefs(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	return __buffer_migrate_page(mapping, newpage, page, mode, true);
}
754
#endif
755

756 757 758 759
/*
 * Writeback a page to clean the dirty state
 */
static int writeout(struct address_space *mapping, struct page *page)
760
{
761
	struct folio *folio = page_folio(page);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = 1,
		.range_start = 0,
		.range_end = LLONG_MAX,
		.for_reclaim = 1
	};
	int rc;

	if (!mapping->a_ops->writepage)
		/* No write method for the address space */
		return -EINVAL;

	if (!clear_page_dirty_for_io(page))
		/* Someone else already triggered a write */
		return -EAGAIN;

779
	/*
780 781 782 783 784 785
	 * A dirty page may imply that the underlying filesystem has
	 * the page on some queue. So the page must be clean for
	 * migration. Writeout may mean we loose the lock and the
	 * page state is no longer what we checked for earlier.
	 * At this point we know that the migration attempt cannot
	 * be successful.
786
	 */
787
	remove_migration_ptes(folio, folio, false);
788

789
	rc = mapping->a_ops->writepage(page, &wbc);
790

791 792 793 794
	if (rc != AOP_WRITEPAGE_ACTIVATE)
		/* unlocked. Relock */
		lock_page(page);

795
	return (rc < 0) ? -EIO : -EAGAIN;
796 797 798 799 800 801
}

/*
 * Default handling if a filesystem does not provide a migration function.
 */
static int fallback_migrate_page(struct address_space *mapping,
802
	struct page *newpage, struct page *page, enum migrate_mode mode)
803
{
804
	if (PageDirty(page)) {
805
		/* Only writeback pages in full synchronous migration */
806 807 808 809 810
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
811
			return -EBUSY;
812
		}
813
		return writeout(mapping, page);
814
	}
815 816 817 818 819

	/*
	 * Buffers may be managed in a filesystem specific way.
	 * We must have no buffers or drop them.
	 */
820
	if (page_has_private(page) &&
821
	    !try_to_release_page(page, GFP_KERNEL))
822
		return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
823

824
	return migrate_page(mapping, newpage, page, mode);
825 826
}

827 828 829 830 831 832
/*
 * Move a page to a newly allocated page
 * The page is locked and all ptes have been successfully removed.
 *
 * The new page will have replaced the old page if this function
 * is successful.
833 834 835
 *
 * Return value:
 *   < 0 - error code
836
 *  MIGRATEPAGE_SUCCESS - success
837
 */
838
static int move_to_new_page(struct page *newpage, struct page *page,
839
				enum migrate_mode mode)
840 841
{
	struct address_space *mapping;
842 843
	int rc = -EAGAIN;
	bool is_lru = !__PageMovable(page);
844

845 846
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
847 848

	mapping = page_mapping(page);
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866

	if (likely(is_lru)) {
		if (!mapping)
			rc = migrate_page(mapping, newpage, page, mode);
		else if (mapping->a_ops->migratepage)
			/*
			 * Most pages have a mapping and most filesystems
			 * provide a migratepage callback. Anonymous pages
			 * are part of swap space which also has its own
			 * migratepage callback. This is the most common path
			 * for page migration.
			 */
			rc = mapping->a_ops->migratepage(mapping, newpage,
							page, mode);
		else
			rc = fallback_migrate_page(mapping, newpage,
							page, mode);
	} else {
867
		/*
868 869
		 * In case of non-lru page, it could be released after
		 * isolation step. In that case, we shouldn't try migration.
870
		 */
871 872 873 874 875 876 877 878 879 880 881 882
		VM_BUG_ON_PAGE(!PageIsolated(page), page);
		if (!PageMovable(page)) {
			rc = MIGRATEPAGE_SUCCESS;
			__ClearPageIsolated(page);
			goto out;
		}

		rc = mapping->a_ops->migratepage(mapping, newpage,
						page, mode);
		WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
			!PageIsolated(page));
	}
883

884 885 886 887 888
	/*
	 * When successful, old pagecache page->mapping must be cleared before
	 * page is freed; but stats require that PageAnon be left as PageAnon.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
889 890 891 892 893 894 895 896 897 898 899
		if (__PageMovable(page)) {
			VM_BUG_ON_PAGE(!PageIsolated(page), page);

			/*
			 * We clear PG_movable under page_lock so any compactor
			 * cannot try to migrate this page.
			 */
			__ClearPageIsolated(page);
		}

		/*
900
		 * Anonymous and movable page->mapping will be cleared by
901 902 903 904
		 * free_pages_prepare so don't reset it here for keeping
		 * the type to work PageAnon, for example.
		 */
		if (!PageMappingFlags(page))
905
			page->mapping = NULL;
906

907
		if (likely(!is_zone_device_page(newpage)))
908 909
			flush_dcache_page(newpage);

910
	}
911
out:
912 913 914
	return rc;
}

915
static int __unmap_and_move(struct page *page, struct page *newpage,
916
				int force, enum migrate_mode mode)
917
{
918
	struct folio *folio = page_folio(page);
919
	struct folio *dst = page_folio(newpage);
920
	int rc = -EAGAIN;
921
	bool page_was_mapped = false;
922
	struct anon_vma *anon_vma = NULL;
923
	bool is_lru = !__PageMovable(page);
924

N
Nick Piggin 已提交
925
	if (!trylock_page(page)) {
926
		if (!force || mode == MIGRATE_ASYNC)
927
			goto out;
928 929 930 931 932 933 934

		/*
		 * It's not safe for direct compaction to call lock_page.
		 * For example, during page readahead pages are added locked
		 * to the LRU. Later, when the IO completes the pages are
		 * marked uptodate and unlocked. However, the queueing
		 * could be merging multiple pages for one bio (e.g.
935
		 * mpage_readahead). If an allocation happens for the
936 937 938 939 940 941 942
		 * second or third page, the process can end up locking
		 * the same page twice and deadlocking. Rather than
		 * trying to be clever about what pages can be locked,
		 * avoid the use of lock_page for direct compaction
		 * altogether.
		 */
		if (current->flags & PF_MEMALLOC)
943
			goto out;
944

945 946 947 948
		lock_page(page);
	}

	if (PageWriteback(page)) {
949
		/*
950
		 * Only in the case of a full synchronous migration is it
951 952 953
		 * necessary to wait for PageWriteback. In the async case,
		 * the retry loop is too short and in the sync-light case,
		 * the overhead of stalling is too much
954
		 */
955 956 957 958 959
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
960
			rc = -EBUSY;
961
			goto out_unlock;
962 963
		}
		if (!force)
964
			goto out_unlock;
965 966
		wait_on_page_writeback(page);
	}
967

968
	/*
969
	 * By try_to_migrate(), page->mapcount goes down to 0 here. In this case,
970
	 * we cannot notice that anon_vma is freed while we migrates a page.
971
	 * This get_anon_vma() delays freeing anon_vma pointer until the end
972
	 * of migration. File cache pages are no problem because of page_lock()
973 974
	 * File Caches may use write_page() or lock_page() in migration, then,
	 * just care Anon page here.
975 976 977 978 979 980
	 *
	 * Only page_get_anon_vma() understands the subtleties of
	 * getting a hold on an anon_vma from outside one of its mms.
	 * But if we cannot get anon_vma, then we won't need it anyway,
	 * because that implies that the anon page is no longer mapped
	 * (and cannot be remapped so long as we hold the page lock).
981
	 */
982
	if (PageAnon(page) && !PageKsm(page))
983
		anon_vma = page_get_anon_vma(page);
984

985 986 987 988 989 990 991 992 993 994 995
	/*
	 * Block others from accessing the new page when we get around to
	 * establishing additional references. We are usually the only one
	 * holding a reference to newpage at this point. We used to have a BUG
	 * here if trylock_page(newpage) fails, but would like to allow for
	 * cases where there might be a race with the previous use of newpage.
	 * This is much like races on refcount of oldpage: just don't BUG().
	 */
	if (unlikely(!trylock_page(newpage)))
		goto out_unlock;

996 997 998 999 1000
	if (unlikely(!is_lru)) {
		rc = move_to_new_page(newpage, page, mode);
		goto out_unlock_both;
	}

1001
	/*
1002 1003 1004 1005 1006
	 * Corner case handling:
	 * 1. When a new swap-cache page is read into, it is added to the LRU
	 * and treated as swapcache but it has no rmap yet.
	 * Calling try_to_unmap() against a page->mapping==NULL page will
	 * trigger a BUG.  So handle it here.
1007
	 * 2. An orphaned page (see truncate_cleanup_page) might have
1008 1009 1010 1011
	 * fs-private metadata. The page can be picked up due to memory
	 * offlining.  Everywhere else except page reclaim, the page is
	 * invisible to the vm, so the page can not be migrated.  So try to
	 * free the metadata, so the page can be freed.
1012
	 */
1013
	if (!page->mapping) {
1014
		VM_BUG_ON_PAGE(PageAnon(page), page);
1015
		if (page_has_private(page)) {
1016
			try_to_free_buffers(page);
1017
			goto out_unlock_both;
1018
		}
1019 1020
	} else if (page_mapped(page)) {
		/* Establish migration ptes */
1021 1022
		VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
				page);
1023
		try_to_migrate(folio, 0);
1024
		page_was_mapped = true;
1025
	}
1026

1027
	if (!page_mapped(page))
1028
		rc = move_to_new_page(newpage, page, mode);
1029

1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
	/*
	 * When successful, push newpage to LRU immediately: so that if it
	 * turns out to be an mlocked page, remove_migration_ptes() will
	 * automatically build up the correct newpage->mlock_count for it.
	 *
	 * We would like to do something similar for the old page, when
	 * unsuccessful, and other cases when a page has been temporarily
	 * isolated from the unevictable LRU: but this case is the easiest.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
		lru_cache_add(newpage);
		if (page_was_mapped)
			lru_add_drain();
	}

1045
	if (page_was_mapped)
1046 1047
		remove_migration_ptes(folio,
			rc == MIGRATEPAGE_SUCCESS ? dst : folio, false);
1048

1049 1050 1051
out_unlock_both:
	unlock_page(newpage);
out_unlock:
1052
	/* Drop an anon_vma reference if we took one */
1053
	if (anon_vma)
1054
		put_anon_vma(anon_vma);
1055
	unlock_page(page);
1056
out:
1057
	/*
1058
	 * If migration is successful, decrease refcount of the newpage,
1059
	 * which will not free the page because new page owner increased
1060
	 * refcounter.
1061
	 */
1062 1063
	if (rc == MIGRATEPAGE_SUCCESS)
		put_page(newpage);
1064

1065 1066
	return rc;
}
1067

1068 1069 1070 1071
/*
 * Obtain the lock on page, remove all ptes and migrate the page
 * to the newly allocated page in newpage.
 */
1072
static int unmap_and_move(new_page_t get_new_page,
1073 1074
				   free_page_t put_new_page,
				   unsigned long private, struct page *page,
1075
				   int force, enum migrate_mode mode,
1076 1077
				   enum migrate_reason reason,
				   struct list_head *ret)
1078
{
1079
	int rc = MIGRATEPAGE_SUCCESS;
1080
	struct page *newpage = NULL;
1081

1082
	if (!thp_migration_supported() && PageTransHuge(page))
1083
		return -ENOSYS;
1084

1085 1086
	if (page_count(page) == 1) {
		/* page was freed from under us. So we are done. */
1087 1088
		ClearPageActive(page);
		ClearPageUnevictable(page);
1089 1090 1091 1092 1093 1094
		if (unlikely(__PageMovable(page))) {
			lock_page(page);
			if (!PageMovable(page))
				__ClearPageIsolated(page);
			unlock_page(page);
		}
1095 1096 1097
		goto out;
	}

1098 1099 1100 1101
	newpage = get_new_page(page, private);
	if (!newpage)
		return -ENOMEM;

1102
	rc = __unmap_and_move(page, newpage, force, mode);
1103
	if (rc == MIGRATEPAGE_SUCCESS)
1104
		set_page_owner_migrate_reason(newpage, reason);
1105

1106
out:
1107
	if (rc != -EAGAIN) {
1108 1109 1110
		/*
		 * A page that has been migrated has all references
		 * removed and will be freed. A page that has not been
1111
		 * migrated will have kept its references and be restored.
1112 1113
		 */
		list_del(&page->lru);
1114
	}
1115

1116 1117 1118 1119 1120 1121
	/*
	 * If migration is successful, releases reference grabbed during
	 * isolation. Otherwise, restore the page to right list unless
	 * we want to retry.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
1122 1123 1124 1125 1126 1127
		/*
		 * Compaction can migrate also non-LRU pages which are
		 * not accounted to NR_ISOLATED_*. They can be recognized
		 * as __PageMovable
		 */
		if (likely(!__PageMovable(page)))
1128
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1129
					page_is_file_lru(page), -thp_nr_pages(page));
1130

1131
		if (reason != MR_MEMORY_FAILURE)
1132
			/*
1133
			 * We release the page in page_handle_poison.
1134
			 */
1135
			put_page(page);
1136
	} else {
1137 1138
		if (rc != -EAGAIN)
			list_add_tail(&page->lru, ret);
1139

1140 1141 1142 1143
		if (put_new_page)
			put_new_page(newpage, private);
		else
			put_page(newpage);
1144
	}
1145

1146 1147 1148
	return rc;
}

1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
/*
 * Counterpart of unmap_and_move_page() for hugepage migration.
 *
 * This function doesn't wait the completion of hugepage I/O
 * because there is no race between I/O and migration for hugepage.
 * Note that currently hugepage I/O occurs only in direct I/O
 * where no lock is held and PG_writeback is irrelevant,
 * and writeback status of all subpages are counted in the reference
 * count of the head page (i.e. if all subpages of a 2MB hugepage are
 * under direct I/O, the reference of the head page is 512 and a bit more.)
 * This means that when we try to migrate hugepage whose subpages are
 * doing direct I/O, some references remain after try_to_unmap() and
 * hugepage migration fails without data corruption.
 *
 * There is also no race when direct I/O is issued on the page under migration,
 * because then pte is replaced with migration swap entry and direct I/O code
 * will wait in the page fault for migration to complete.
 */
static int unmap_and_move_huge_page(new_page_t get_new_page,
1168 1169
				free_page_t put_new_page, unsigned long private,
				struct page *hpage, int force,
1170 1171
				enum migrate_mode mode, int reason,
				struct list_head *ret)
1172
{
1173
	struct folio *dst, *src = page_folio(hpage);
1174
	int rc = -EAGAIN;
1175
	int page_was_mapped = 0;
1176
	struct page *new_hpage;
1177
	struct anon_vma *anon_vma = NULL;
1178
	struct address_space *mapping = NULL;
1179

1180
	/*
1181
	 * Migratability of hugepages depends on architectures and their size.
1182 1183 1184 1185 1186
	 * This check is necessary because some callers of hugepage migration
	 * like soft offline and memory hotremove don't walk through page
	 * tables or check whether the hugepage is pmd-based or not before
	 * kicking migration.
	 */
1187
	if (!hugepage_migration_supported(page_hstate(hpage))) {
1188
		list_move_tail(&hpage->lru, ret);
1189
		return -ENOSYS;
1190
	}
1191

1192 1193 1194 1195 1196 1197
	if (page_count(hpage) == 1) {
		/* page was freed from under us. So we are done. */
		putback_active_hugepage(hpage);
		return MIGRATEPAGE_SUCCESS;
	}

1198
	new_hpage = get_new_page(hpage, private);
1199 1200
	if (!new_hpage)
		return -ENOMEM;
1201
	dst = page_folio(new_hpage);
1202 1203

	if (!trylock_page(hpage)) {
1204
		if (!force)
1205
			goto out;
1206 1207 1208 1209 1210 1211 1212
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
			goto out;
		}
1213 1214 1215
		lock_page(hpage);
	}

1216 1217 1218 1219 1220
	/*
	 * Check for pages which are in the process of being freed.  Without
	 * page_mapping() set, hugetlbfs specific move page routine will not
	 * be called and we could leak usage counts for subpools.
	 */
1221
	if (hugetlb_page_subpool(hpage) && !page_mapping(hpage)) {
1222 1223 1224 1225
		rc = -EBUSY;
		goto out_unlock;
	}

1226 1227
	if (PageAnon(hpage))
		anon_vma = page_get_anon_vma(hpage);
1228

1229 1230 1231
	if (unlikely(!trylock_page(new_hpage)))
		goto put_anon;

1232
	if (page_mapped(hpage)) {
1233
		bool mapping_locked = false;
1234
		enum ttu_flags ttu = 0;
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249

		if (!PageAnon(hpage)) {
			/*
			 * In shared mappings, try_to_unmap could potentially
			 * call huge_pmd_unshare.  Because of this, take
			 * semaphore in write mode here and set TTU_RMAP_LOCKED
			 * to let lower levels know we have taken the lock.
			 */
			mapping = hugetlb_page_mapping_lock_write(hpage);
			if (unlikely(!mapping))
				goto unlock_put_anon;

			mapping_locked = true;
			ttu |= TTU_RMAP_LOCKED;
		}
1250

1251
		try_to_migrate(src, ttu);
1252
		page_was_mapped = 1;
1253 1254 1255

		if (mapping_locked)
			i_mmap_unlock_write(mapping);
1256
	}
1257 1258

	if (!page_mapped(hpage))
1259
		rc = move_to_new_page(new_hpage, hpage, mode);
1260

1261
	if (page_was_mapped)
1262 1263
		remove_migration_ptes(src,
			rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
1264

1265
unlock_put_anon:
1266 1267 1268
	unlock_page(new_hpage);

put_anon:
H
Hugh Dickins 已提交
1269
	if (anon_vma)
1270
		put_anon_vma(anon_vma);
1271

1272
	if (rc == MIGRATEPAGE_SUCCESS) {
1273
		move_hugetlb_state(hpage, new_hpage, reason);
1274 1275
		put_new_page = NULL;
	}
1276

1277
out_unlock:
1278
	unlock_page(hpage);
1279
out:
1280
	if (rc == MIGRATEPAGE_SUCCESS)
1281
		putback_active_hugepage(hpage);
1282
	else if (rc != -EAGAIN)
1283
		list_move_tail(&hpage->lru, ret);
1284 1285 1286 1287 1288 1289

	/*
	 * If migration was not successful and there's a freeing callback, use
	 * it.  Otherwise, put_page() will drop the reference grabbed during
	 * isolation.
	 */
1290
	if (put_new_page)
1291 1292
		put_new_page(new_hpage, private);
	else
1293
		putback_active_hugepage(new_hpage);
1294

1295 1296 1297
	return rc;
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
static inline int try_split_thp(struct page *page, struct page **page2,
				struct list_head *from)
{
	int rc = 0;

	lock_page(page);
	rc = split_huge_page_to_list(page, from);
	unlock_page(page);
	if (!rc)
		list_safe_reset_next(page, *page2, lru);

	return rc;
}

1312
/*
1313 1314
 * migrate_pages - migrate the pages specified in a list, to the free pages
 *		   supplied as the target for the page migration
1315
 *
1316 1317 1318
 * @from:		The list of pages to be migrated.
 * @get_new_page:	The function used to allocate free pages to be used
 *			as the target of the page migration.
1319 1320
 * @put_new_page:	The function used to free target pages if migration
 *			fails, or NULL if no special handling is necessary.
1321 1322 1323 1324
 * @private:		Private data to be passed on to get_new_page()
 * @mode:		The migration mode that specifies the constraints for
 *			page migration, if any.
 * @reason:		The reason for page migration.
1325
 * @ret_succeeded:	Set to the number of normal pages migrated successfully if
1326
 *			the caller passes a non-NULL pointer.
1327
 *
1328 1329
 * The function returns after 10 attempts or if no pages are movable any more
 * because the list has become empty or no retryable pages exist any more.
1330 1331
 * It is caller's responsibility to call putback_movable_pages() to return pages
 * to the LRU or free list only if ret != 0.
1332
 *
1333 1334 1335
 * Returns the number of {normal page, THP, hugetlb} that were not migrated, or
 * an error code. The number of THP splits will be considered as the number of
 * non-migrated THP, no matter how many subpages of the THP are migrated successfully.
1336
 */
1337
int migrate_pages(struct list_head *from, new_page_t get_new_page,
1338
		free_page_t put_new_page, unsigned long private,
1339
		enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
1340
{
1341
	int retry = 1;
1342
	int thp_retry = 1;
1343
	int nr_failed = 0;
1344
	int nr_failed_pages = 0;
1345
	int nr_succeeded = 0;
1346 1347 1348
	int nr_thp_succeeded = 0;
	int nr_thp_failed = 0;
	int nr_thp_split = 0;
1349
	int pass = 0;
1350
	bool is_thp = false;
1351 1352 1353
	struct page *page;
	struct page *page2;
	int swapwrite = current->flags & PF_SWAPWRITE;
1354
	int rc, nr_subpages;
1355
	LIST_HEAD(ret_pages);
1356
	LIST_HEAD(thp_split_pages);
1357
	bool nosplit = (reason == MR_NUMA_MISPLACED);
1358
	bool no_subpage_counting = false;
1359

1360 1361
	trace_mm_migrate_pages_start(mode, reason);

1362 1363 1364
	if (!swapwrite)
		current->flags |= PF_SWAPWRITE;

1365
thp_subpage_migration:
1366
	for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
1367
		retry = 0;
1368
		thp_retry = 0;
1369

1370
		list_for_each_entry_safe(page, page2, from, lru) {
1371
retry:
1372 1373 1374 1375 1376
			/*
			 * THP statistics is based on the source huge page.
			 * Capture required information that might get lost
			 * during migration.
			 */
1377
			is_thp = PageTransHuge(page) && !PageHuge(page);
1378
			nr_subpages = compound_nr(page);
1379
			cond_resched();
1380

1381 1382
			if (PageHuge(page))
				rc = unmap_and_move_huge_page(get_new_page,
1383
						put_new_page, private, page,
1384 1385
						pass > 2, mode, reason,
						&ret_pages);
1386
			else
1387
				rc = unmap_and_move(get_new_page, put_new_page,
1388
						private, page, pass > 2, mode,
1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
						reason, &ret_pages);
			/*
			 * The rules are:
			 *	Success: non hugetlb page will be freed, hugetlb
			 *		 page will be put back
			 *	-EAGAIN: stay on the from list
			 *	-ENOMEM: stay on the from list
			 *	Other errno: put on ret_pages list then splice to
			 *		     from list
			 */
1399
			switch(rc) {
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
			/*
			 * THP migration might be unsupported or the
			 * allocation could've failed so we should
			 * retry on the same page with the THP split
			 * to base pages.
			 *
			 * Head page is retried immediately and tail
			 * pages are added to the tail of the list so
			 * we encounter them after the rest of the list
			 * is processed.
			 */
			case -ENOSYS:
				/* THP migration is unsupported */
				if (is_thp) {
1414 1415
					nr_thp_failed++;
					if (!try_split_thp(page, &page2, &thp_split_pages)) {
1416 1417 1418 1419
						nr_thp_split++;
						goto retry;
					}

1420
					nr_failed_pages += nr_subpages;
1421 1422 1423 1424
					break;
				}

				/* Hugetlb migration is unsupported */
1425 1426
				if (!no_subpage_counting)
					nr_failed++;
1427
				nr_failed_pages += nr_subpages;
1428
				break;
1429
			case -ENOMEM:
1430
				/*
1431 1432
				 * When memory is low, don't bother to try to migrate
				 * other pages, just exit.
1433
				 * THP NUMA faulting doesn't split THP to retry.
1434
				 */
1435
				if (is_thp && !nosplit) {
1436 1437
					nr_thp_failed++;
					if (!try_split_thp(page, &page2, &thp_split_pages)) {
1438
						nr_thp_split++;
1439 1440
						goto retry;
					}
1441

1442
					nr_failed_pages += nr_subpages;
1443 1444
					goto out;
				}
1445 1446 1447

				if (!no_subpage_counting)
					nr_failed++;
1448
				nr_failed_pages += nr_subpages;
1449
				goto out;
1450
			case -EAGAIN:
1451 1452 1453 1454
				if (is_thp) {
					thp_retry++;
					break;
				}
1455
				retry++;
1456
				break;
1457
			case MIGRATEPAGE_SUCCESS:
1458
				nr_succeeded += nr_subpages;
1459 1460 1461 1462
				if (is_thp) {
					nr_thp_succeeded++;
					break;
				}
1463 1464
				break;
			default:
1465
				/*
1466
				 * Permanent failure (-EBUSY, etc.):
1467 1468 1469 1470
				 * unlike -EAGAIN case, the failed page is
				 * removed from migration page list and not
				 * retried in the next outer loop.
				 */
1471 1472
				if (is_thp) {
					nr_thp_failed++;
1473
					nr_failed_pages += nr_subpages;
1474 1475
					break;
				}
1476 1477 1478

				if (!no_subpage_counting)
					nr_failed++;
1479
				nr_failed_pages += nr_subpages;
1480
				break;
1481
			}
1482 1483
		}
	}
1484
	nr_failed += retry;
1485
	nr_thp_failed += thp_retry;
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	/*
	 * Try to migrate subpages of fail-to-migrate THPs, no nr_failed
	 * counting in this round, since all subpages of a THP is counted
	 * as 1 failure in the first round.
	 */
	if (!list_empty(&thp_split_pages)) {
		/*
		 * Move non-migrated pages (after 10 retries) to ret_pages
		 * to avoid migrating them again.
		 */
		list_splice_init(from, &ret_pages);
		list_splice_init(&thp_split_pages, from);
		no_subpage_counting = true;
		retry = 1;
		goto thp_subpage_migration;
	}

	rc = nr_failed + nr_thp_failed;
1504
out:
1505 1506 1507 1508 1509 1510
	/*
	 * Put the permanent failure page back to migration list, they
	 * will be put back to the right list by the caller.
	 */
	list_splice(&ret_pages, from);

1511
	count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1512
	count_vm_events(PGMIGRATE_FAIL, nr_failed_pages);
1513 1514 1515
	count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
	count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
	count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
1516
	trace_mm_migrate_pages(nr_succeeded, nr_failed_pages, nr_thp_succeeded,
1517
			       nr_thp_failed, nr_thp_split, mode, reason);
1518

1519 1520 1521
	if (!swapwrite)
		current->flags &= ~PF_SWAPWRITE;

1522 1523 1524
	if (ret_succeeded)
		*ret_succeeded = nr_succeeded;

1525
	return rc;
1526
}
1527

1528
struct page *alloc_migration_target(struct page *page, unsigned long private)
1529
{
1530 1531
	struct migration_target_control *mtc;
	gfp_t gfp_mask;
1532 1533
	unsigned int order = 0;
	struct page *new_page = NULL;
1534 1535 1536 1537 1538 1539 1540 1541
	int nid;
	int zidx;

	mtc = (struct migration_target_control *)private;
	gfp_mask = mtc->gfp_mask;
	nid = mtc->nid;
	if (nid == NUMA_NO_NODE)
		nid = page_to_nid(page);
1542

1543 1544 1545
	if (PageHuge(page)) {
		struct hstate *h = page_hstate(compound_head(page));

1546 1547
		gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
		return alloc_huge_page_nodemask(h, nid, mtc->nmask, gfp_mask);
1548
	}
1549 1550

	if (PageTransHuge(page)) {
1551 1552 1553 1554 1555
		/*
		 * clear __GFP_RECLAIM to make the migration callback
		 * consistent with regular THP allocations.
		 */
		gfp_mask &= ~__GFP_RECLAIM;
1556 1557 1558
		gfp_mask |= GFP_TRANSHUGE;
		order = HPAGE_PMD_ORDER;
	}
1559 1560
	zidx = zone_idx(page_zone(page));
	if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
1561 1562
		gfp_mask |= __GFP_HIGHMEM;

1563
	new_page = __alloc_pages(gfp_mask, order, nid, mtc->nmask);
1564 1565 1566 1567 1568 1569 1570

	if (new_page && PageTransHuge(new_page))
		prep_transhuge_page(new_page);

	return new_page;
}

1571 1572
#ifdef CONFIG_NUMA

1573
static int store_status(int __user *status, int start, int value, int nr)
1574
{
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
	while (nr-- > 0) {
		if (put_user(value, status + start))
			return -EFAULT;
		start++;
	}

	return 0;
}

static int do_move_pages_to_node(struct mm_struct *mm,
		struct list_head *pagelist, int node)
{
	int err;
1588 1589 1590 1591
	struct migration_target_control mtc = {
		.nid = node,
		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
	};
1592

1593
	err = migrate_pages(pagelist, alloc_migration_target, NULL,
1594
		(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1595 1596 1597
	if (err)
		putback_movable_pages(pagelist);
	return err;
1598 1599 1600
}

/*
1601 1602
 * Resolves the given address to a struct page, isolates it from the LRU and
 * puts it to the given pagelist.
1603 1604 1605 1606 1607
 * Returns:
 *     errno - if the page cannot be found/isolated
 *     0 - when it doesn't have to be migrated because it is already on the
 *         target node
 *     1 - when it has been queued
1608
 */
1609 1610
static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
		int node, struct list_head *pagelist, bool migrate_all)
1611
{
1612 1613 1614
	struct vm_area_struct *vma;
	struct page *page;
	unsigned int follflags;
1615 1616
	int err;

1617
	mmap_read_lock(mm);
1618 1619 1620 1621
	err = -EFAULT;
	vma = find_vma(mm, addr);
	if (!vma || addr < vma->vm_start || !vma_migratable(vma))
		goto out;
1622

1623 1624 1625
	/* FOLL_DUMP to ignore special (like zero) pages */
	follflags = FOLL_GET | FOLL_DUMP;
	page = follow_page(vma, addr, follflags);
1626

1627 1628 1629
	err = PTR_ERR(page);
	if (IS_ERR(page))
		goto out;
1630

1631 1632 1633
	err = -ENOENT;
	if (!page)
		goto out;
1634

1635 1636 1637
	err = 0;
	if (page_to_nid(page) == node)
		goto out_putpage;
1638

1639 1640 1641
	err = -EACCES;
	if (page_mapcount(page) > 1 && !migrate_all)
		goto out_putpage;
1642

1643 1644 1645
	if (PageHuge(page)) {
		if (PageHead(page)) {
			isolate_huge_page(page, pagelist);
1646
			err = 1;
1647
		}
1648 1649
	} else {
		struct page *head;
1650

1651 1652
		head = compound_head(page);
		err = isolate_lru_page(head);
1653
		if (err)
1654
			goto out_putpage;
1655

1656
		err = 1;
1657 1658
		list_add_tail(&head->lru, pagelist);
		mod_node_page_state(page_pgdat(head),
1659
			NR_ISOLATED_ANON + page_is_file_lru(head),
1660
			thp_nr_pages(head));
1661 1662 1663 1664 1665 1666 1667 1668 1669
	}
out_putpage:
	/*
	 * Either remove the duplicate refcount from
	 * isolate_lru_page() or drop the page ref if it was
	 * not isolated.
	 */
	put_page(page);
out:
1670
	mmap_read_unlock(mm);
1671 1672 1673
	return err;
}

1674 1675 1676 1677 1678 1679
static int move_pages_and_store_status(struct mm_struct *mm, int node,
		struct list_head *pagelist, int __user *status,
		int start, int i, unsigned long nr_pages)
{
	int err;

1680 1681 1682
	if (list_empty(pagelist))
		return 0;

1683 1684 1685 1686 1687 1688
	err = do_move_pages_to_node(mm, pagelist, node);
	if (err) {
		/*
		 * Positive err means the number of failed
		 * pages to migrate.  Since we are going to
		 * abort and return the number of non-migrated
1689
		 * pages, so need to include the rest of the
1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
		 * nr_pages that have not been attempted as
		 * well.
		 */
		if (err > 0)
			err += nr_pages - i - 1;
		return err;
	}
	return store_status(status, start, node, i - start);
}

1700 1701 1702 1703
/*
 * Migrate an array of page address onto an array of nodes and fill
 * the corresponding array of status.
 */
1704
static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1705 1706 1707 1708 1709
			 unsigned long nr_pages,
			 const void __user * __user *pages,
			 const int __user *nodes,
			 int __user *status, int flags)
{
1710 1711 1712 1713
	int current_node = NUMA_NO_NODE;
	LIST_HEAD(pagelist);
	int start, i;
	int err = 0, err1;
1714

1715
	lru_cache_disable();
1716

1717 1718 1719 1720
	for (i = start = 0; i < nr_pages; i++) {
		const void __user *p;
		unsigned long addr;
		int node;
1721

1722 1723 1724 1725 1726
		err = -EFAULT;
		if (get_user(p, pages + i))
			goto out_flush;
		if (get_user(node, nodes + i))
			goto out_flush;
1727
		addr = (unsigned long)untagged_addr(p);
1728 1729 1730 1731 1732 1733

		err = -ENODEV;
		if (node < 0 || node >= MAX_NUMNODES)
			goto out_flush;
		if (!node_state(node, N_MEMORY))
			goto out_flush;
1734

1735 1736 1737 1738 1739 1740 1741 1742
		err = -EACCES;
		if (!node_isset(node, task_nodes))
			goto out_flush;

		if (current_node == NUMA_NO_NODE) {
			current_node = node;
			start = i;
		} else if (node != current_node) {
1743 1744
			err = move_pages_and_store_status(mm, current_node,
					&pagelist, status, start, i, nr_pages);
1745 1746 1747 1748
			if (err)
				goto out;
			start = i;
			current_node = node;
1749 1750
		}

1751 1752 1753 1754 1755 1756
		/*
		 * Errors in the page lookup or isolation are not fatal and we simply
		 * report them via status
		 */
		err = add_page_for_migration(mm, addr, current_node,
				&pagelist, flags & MPOL_MF_MOVE_ALL);
1757

1758
		if (err > 0) {
1759 1760 1761
			/* The page is successfully queued for migration */
			continue;
		}
1762

1763 1764 1765 1766 1767
		/*
		 * If the page is already on the target node (!err), store the
		 * node, otherwise, store the err.
		 */
		err = store_status(status, i, err ? : current_node, 1);
1768 1769
		if (err)
			goto out_flush;
1770

1771 1772
		err = move_pages_and_store_status(mm, current_node, &pagelist,
				status, start, i, nr_pages);
1773 1774
		if (err)
			goto out;
1775
		current_node = NUMA_NO_NODE;
1776
	}
1777 1778
out_flush:
	/* Make sure we do not overwrite the existing error */
1779 1780
	err1 = move_pages_and_store_status(mm, current_node, &pagelist,
				status, start, i, nr_pages);
1781
	if (err >= 0)
1782
		err = err1;
1783
out:
1784
	lru_cache_enable();
1785 1786 1787
	return err;
}

1788
/*
1789
 * Determine the nodes of an array of pages and store it in an array of status.
1790
 */
1791 1792
static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
				const void __user **pages, int *status)
1793
{
1794 1795
	unsigned long i;

1796
	mmap_read_lock(mm);
1797

1798
	for (i = 0; i < nr_pages; i++) {
1799
		unsigned long addr = (unsigned long)(*pages);
1800 1801
		struct vm_area_struct *vma;
		struct page *page;
1802
		int err = -EFAULT;
1803

1804 1805
		vma = vma_lookup(mm, addr);
		if (!vma)
1806 1807
			goto set_status;

1808 1809
		/* FOLL_DUMP to ignore special (like zero) pages */
		page = follow_page(vma, addr, FOLL_DUMP);
1810 1811 1812 1813 1814

		err = PTR_ERR(page);
		if (IS_ERR(page))
			goto set_status;

1815
		err = page ? page_to_nid(page) : -ENOENT;
1816
set_status:
1817 1818 1819 1820 1821 1822
		*status = err;

		pages++;
		status++;
	}

1823
	mmap_read_unlock(mm);
1824 1825
}

1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842
static int get_compat_pages_array(const void __user *chunk_pages[],
				  const void __user * __user *pages,
				  unsigned long chunk_nr)
{
	compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages;
	compat_uptr_t p;
	int i;

	for (i = 0; i < chunk_nr; i++) {
		if (get_user(p, pages32 + i))
			return -EFAULT;
		chunk_pages[i] = compat_ptr(p);
	}

	return 0;
}

1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
/*
 * Determine the nodes of a user array of pages and store it in
 * a user array of status.
 */
static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
			 const void __user * __user *pages,
			 int __user *status)
{
#define DO_PAGES_STAT_CHUNK_NR 16
	const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
	int chunk_status[DO_PAGES_STAT_CHUNK_NR];

1855 1856
	while (nr_pages) {
		unsigned long chunk_nr;
1857

1858 1859 1860 1861
		chunk_nr = nr_pages;
		if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
			chunk_nr = DO_PAGES_STAT_CHUNK_NR;

1862 1863 1864 1865 1866 1867 1868 1869 1870
		if (in_compat_syscall()) {
			if (get_compat_pages_array(chunk_pages, pages,
						   chunk_nr))
				break;
		} else {
			if (copy_from_user(chunk_pages, pages,
				      chunk_nr * sizeof(*chunk_pages)))
				break;
		}
1871 1872 1873

		do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);

1874 1875
		if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
			break;
1876

1877 1878 1879 1880 1881
		pages += chunk_nr;
		status += chunk_nr;
		nr_pages -= chunk_nr;
	}
	return nr_pages ? -EFAULT : 0;
1882 1883
}

1884
static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
1885 1886 1887 1888
{
	struct task_struct *task;
	struct mm_struct *mm;

1889 1890 1891 1892 1893 1894 1895 1896 1897
	/*
	 * There is no need to check if current process has the right to modify
	 * the specified process when they are same.
	 */
	if (!pid) {
		mmget(current->mm);
		*mem_nodes = cpuset_mems_allowed(current);
		return current->mm;
	}
1898 1899

	/* Find the mm_struct */
1900
	rcu_read_lock();
1901
	task = find_task_by_vpid(pid);
1902
	if (!task) {
1903
		rcu_read_unlock();
1904
		return ERR_PTR(-ESRCH);
1905
	}
1906
	get_task_struct(task);
1907 1908 1909

	/*
	 * Check if this process has the right to modify the specified
1910
	 * process. Use the regular "ptrace_may_access()" checks.
1911
	 */
1912
	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1913
		rcu_read_unlock();
1914
		mm = ERR_PTR(-EPERM);
1915
		goto out;
1916
	}
1917
	rcu_read_unlock();
1918

1919 1920
	mm = ERR_PTR(security_task_movememory(task));
	if (IS_ERR(mm))
1921
		goto out;
1922
	*mem_nodes = cpuset_mems_allowed(task);
1923
	mm = get_task_mm(task);
1924
out:
1925
	put_task_struct(task);
1926
	if (!mm)
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
		mm = ERR_PTR(-EINVAL);
	return mm;
}

/*
 * Move a list of pages in the address space of the currently executing
 * process.
 */
static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
			     const void __user * __user *pages,
			     const int __user *nodes,
			     int __user *status, int flags)
{
	struct mm_struct *mm;
	int err;
	nodemask_t task_nodes;

	/* Check flags */
	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1946 1947
		return -EINVAL;

1948 1949 1950 1951 1952 1953 1954
	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
		return -EPERM;

	mm = find_mm_struct(pid, &task_nodes);
	if (IS_ERR(mm))
		return PTR_ERR(mm);

1955 1956 1957 1958 1959
	if (nodes)
		err = do_pages_move(mm, task_nodes, nr_pages, pages,
				    nodes, status, flags);
	else
		err = do_pages_stat(mm, nr_pages, pages, status);
1960 1961 1962 1963 1964

	mmput(mm);
	return err;
}

1965 1966 1967 1968 1969 1970 1971 1972
SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
		const void __user * __user *, pages,
		const int __user *, nodes,
		int __user *, status, int, flags)
{
	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
}

1973 1974 1975 1976 1977 1978
#ifdef CONFIG_NUMA_BALANCING
/*
 * Returns true if this is a safe migration target node for misplaced NUMA
 * pages. Currently it only checks the watermarks which crude
 */
static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1979
				   unsigned long nr_migrate_pages)
1980 1981
{
	int z;
1982

1983 1984 1985 1986 1987 1988 1989 1990 1991 1992
	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
		struct zone *zone = pgdat->node_zones + z;

		if (!populated_zone(zone))
			continue;

		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
		if (!zone_watermark_ok(zone, 0,
				       high_wmark_pages(zone) +
				       nr_migrate_pages,
1993
				       ZONE_MOVABLE, 0))
1994 1995 1996 1997 1998 1999 2000
			continue;
		return true;
	}
	return false;
}

static struct page *alloc_misplaced_dst_page(struct page *page,
2001
					   unsigned long data)
2002 2003 2004 2005
{
	int nid = (int) data;
	struct page *newpage;

2006
	newpage = __alloc_pages_node(nid,
2007 2008 2009
					 (GFP_HIGHUSER_MOVABLE |
					  __GFP_THISNODE | __GFP_NOMEMALLOC |
					  __GFP_NORETRY | __GFP_NOWARN) &
2010
					 ~__GFP_RECLAIM, 0);
2011

2012 2013 2014
	return newpage;
}

2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031
static struct page *alloc_misplaced_dst_page_thp(struct page *page,
						 unsigned long data)
{
	int nid = (int) data;
	struct page *newpage;

	newpage = alloc_pages_node(nid, (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
				   HPAGE_PMD_ORDER);
	if (!newpage)
		goto out;

	prep_transhuge_page(newpage);

out:
	return newpage;
}

2032
static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
2033
{
2034
	int page_lru;
2035
	int nr_pages = thp_nr_pages(page);
2036

2037
	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
2038

2039 2040 2041 2042
	/* Do not migrate THP mapped by multiple processes */
	if (PageTransHuge(page) && total_mapcount(page) > 1)
		return 0;

2043
	/* Avoid migrating to a node that is nearly full */
2044
	if (!migrate_balanced_pgdat(pgdat, nr_pages))
2045
		return 0;
2046

2047 2048
	if (isolate_lru_page(page))
		return 0;
2049

2050
	page_lru = page_is_file_lru(page);
2051
	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
2052
			    nr_pages);
2053

2054
	/*
2055 2056 2057
	 * Isolating the page has taken another reference, so the
	 * caller's reference can be safely dropped without the page
	 * disappearing underneath us during migration.
2058 2059
	 */
	put_page(page);
2060
	return 1;
2061 2062 2063 2064 2065 2066 2067
}

/*
 * Attempt to migrate a misplaced page to the specified destination
 * node. Caller is expected to have an elevated reference count on
 * the page that will be dropped by this function before returning.
 */
2068 2069
int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
			   int node)
2070 2071
{
	pg_data_t *pgdat = NODE_DATA(node);
2072
	int isolated;
2073 2074
	int nr_remaining;
	LIST_HEAD(migratepages);
2075 2076
	new_page_t *new;
	bool compound;
2077
	int nr_pages = thp_nr_pages(page);
2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089

	/*
	 * PTE mapped THP or HugeTLB page can't reach here so the page could
	 * be either base page or THP.  And it must be head page if it is
	 * THP.
	 */
	compound = PageTransHuge(page);

	if (compound)
		new = alloc_misplaced_dst_page_thp;
	else
		new = alloc_misplaced_dst_page;
2090 2091

	/*
2092 2093
	 * Don't migrate file pages that are mapped in multiple processes
	 * with execute permissions as they are probably shared libraries.
2094
	 */
2095 2096
	if (page_mapcount(page) != 1 && page_is_file_lru(page) &&
	    (vma->vm_flags & VM_EXEC))
2097 2098
		goto out;

2099 2100 2101 2102
	/*
	 * Also do not migrate dirty pages as not all filesystems can move
	 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
	 */
2103
	if (page_is_file_lru(page) && PageDirty(page))
2104 2105
		goto out;

2106 2107 2108 2109 2110
	isolated = numamigrate_isolate_page(pgdat, page);
	if (!isolated)
		goto out;

	list_add(&page->lru, &migratepages);
2111
	nr_remaining = migrate_pages(&migratepages, *new, NULL, node,
2112
				     MIGRATE_ASYNC, MR_NUMA_MISPLACED, NULL);
2113
	if (nr_remaining) {
2114 2115
		if (!list_empty(&migratepages)) {
			list_del(&page->lru);
2116 2117
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
					page_is_file_lru(page), -nr_pages);
2118 2119
			putback_lru_page(page);
		}
2120 2121
		isolated = 0;
	} else
2122
		count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_pages);
2123 2124
	BUG_ON(!list_empty(&migratepages));
	return isolated;
2125 2126 2127 2128

out:
	put_page(page);
	return 0;
2129
}
2130
#endif /* CONFIG_NUMA_BALANCING */
2131
#endif /* CONFIG_NUMA */
2132

2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264
/*
 * node_demotion[] example:
 *
 * Consider a system with two sockets.  Each socket has
 * three classes of memory attached: fast, medium and slow.
 * Each memory class is placed in its own NUMA node.  The
 * CPUs are placed in the node with the "fast" memory.  The
 * 6 NUMA nodes (0-5) might be split among the sockets like
 * this:
 *
 *	Socket A: 0, 1, 2
 *	Socket B: 3, 4, 5
 *
 * When Node 0 fills up, its memory should be migrated to
 * Node 1.  When Node 1 fills up, it should be migrated to
 * Node 2.  The migration path start on the nodes with the
 * processors (since allocations default to this node) and
 * fast memory, progress through medium and end with the
 * slow memory:
 *
 *	0 -> 1 -> 2 -> stop
 *	3 -> 4 -> 5 -> stop
 *
 * This is represented in the node_demotion[] like this:
 *
 *	{  nr=1, nodes[0]=1 }, // Node 0 migrates to 1
 *	{  nr=1, nodes[0]=2 }, // Node 1 migrates to 2
 *	{  nr=0, nodes[0]=-1 }, // Node 2 does not migrate
 *	{  nr=1, nodes[0]=4 }, // Node 3 migrates to 4
 *	{  nr=1, nodes[0]=5 }, // Node 4 migrates to 5
 *	{  nr=0, nodes[0]=-1 }, // Node 5 does not migrate
 *
 * Moreover some systems may have multiple slow memory nodes.
 * Suppose a system has one socket with 3 memory nodes, node 0
 * is fast memory type, and node 1/2 both are slow memory
 * type, and the distance between fast memory node and slow
 * memory node is same. So the migration path should be:
 *
 *	0 -> 1/2 -> stop
 *
 * This is represented in the node_demotion[] like this:
 *	{ nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2
 *	{ nr=0, nodes[0]=-1, }, // Node 1 dose not migrate
 *	{ nr=0, nodes[0]=-1, }, // Node 2 does not migrate
 */

/*
 * Writes to this array occur without locking.  Cycles are
 * not allowed: Node X demotes to Y which demotes to X...
 *
 * If multiple reads are performed, a single rcu_read_lock()
 * must be held over all reads to ensure that no cycles are
 * observed.
 */
#define DEFAULT_DEMOTION_TARGET_NODES 15

#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES
#define DEMOTION_TARGET_NODES	(MAX_NUMNODES - 1)
#else
#define DEMOTION_TARGET_NODES	DEFAULT_DEMOTION_TARGET_NODES
#endif

struct demotion_nodes {
	unsigned short nr;
	short nodes[DEMOTION_TARGET_NODES];
};

static struct demotion_nodes *node_demotion __read_mostly;

/**
 * next_demotion_node() - Get the next node in the demotion path
 * @node: The starting node to lookup the next node
 *
 * Return: node id for next memory node in the demotion path hierarchy
 * from @node; NUMA_NO_NODE if @node is terminal.  This does not keep
 * @node online or guarantee that it *continues* to be the next demotion
 * target.
 */
int next_demotion_node(int node)
{
	struct demotion_nodes *nd;
	unsigned short target_nr, index;
	int target;

	if (!node_demotion)
		return NUMA_NO_NODE;

	nd = &node_demotion[node];

	/*
	 * node_demotion[] is updated without excluding this
	 * function from running.  RCU doesn't provide any
	 * compiler barriers, so the READ_ONCE() is required
	 * to avoid compiler reordering or read merging.
	 *
	 * Make sure to use RCU over entire code blocks if
	 * node_demotion[] reads need to be consistent.
	 */
	rcu_read_lock();
	target_nr = READ_ONCE(nd->nr);

	switch (target_nr) {
	case 0:
		target = NUMA_NO_NODE;
		goto out;
	case 1:
		index = 0;
		break;
	default:
		/*
		 * If there are multiple target nodes, just select one
		 * target node randomly.
		 *
		 * In addition, we can also use round-robin to select
		 * target node, but we should introduce another variable
		 * for node_demotion[] to record last selected target node,
		 * that may cause cache ping-pong due to the changing of
		 * last target node. Or introducing per-cpu data to avoid
		 * caching issue, which seems more complicated. So selecting
		 * target node randomly seems better until now.
		 */
		index = get_random_int() % target_nr;
		break;
	}

	target = READ_ONCE(nd->nodes[index]);

out:
	rcu_read_unlock();
	return target;
}

2265
#if defined(CONFIG_HOTPLUG_CPU)
2266 2267 2268
/* Disable reclaim-based migration. */
static void __disable_all_migrate_targets(void)
{
2269
	int node, i;
2270

2271 2272
	if (!node_demotion)
		return;
2273

2274 2275 2276 2277 2278
	for_each_online_node(node) {
		node_demotion[node].nr = 0;
		for (i = 0; i < DEMOTION_TARGET_NODES; i++)
			node_demotion[node].nodes[i] = NUMA_NO_NODE;
	}
2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
}

static void disable_all_migrate_targets(void)
{
	__disable_all_migrate_targets();

	/*
	 * Ensure that the "disable" is visible across the system.
	 * Readers will see either a combination of before+disable
	 * state or disable+after.  They will never see before and
	 * after state together.
	 *
	 * The before+after state together might have cycles and
	 * could cause readers to do things like loop until this
	 * function finishes.  This ensures they can only see a
	 * single "bad" read and would, for instance, only loop
	 * once.
	 */
	synchronize_rcu();
}

/*
 * Find an automatic demotion target for 'node'.
 * Failing here is OK.  It might just indicate
 * being at the end of a chain.
 */
2305 2306
static int establish_migrate_target(int node, nodemask_t *used,
				    int best_distance)
2307
{
2308 2309
	int migration_target, index, val;
	struct demotion_nodes *nd;
2310

2311
	if (!node_demotion)
2312 2313
		return NUMA_NO_NODE;

2314 2315
	nd = &node_demotion[node];

2316 2317 2318 2319
	migration_target = find_next_best_node(node, used);
	if (migration_target == NUMA_NO_NODE)
		return NUMA_NO_NODE;

2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
	/*
	 * If the node has been set a migration target node before,
	 * which means it's the best distance between them. Still
	 * check if this node can be demoted to other target nodes
	 * if they have a same best distance.
	 */
	if (best_distance != -1) {
		val = node_distance(node, migration_target);
		if (val > best_distance)
			return NUMA_NO_NODE;
	}

	index = nd->nr;
	if (WARN_ONCE(index >= DEMOTION_TARGET_NODES,
		      "Exceeds maximum demotion target nodes\n"))
		return NUMA_NO_NODE;

	nd->nodes[index] = migration_target;
	nd->nr++;
2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353

	return migration_target;
}

/*
 * When memory fills up on a node, memory contents can be
 * automatically migrated to another node instead of
 * discarded at reclaim.
 *
 * Establish a "migration path" which will start at nodes
 * with CPUs and will follow the priorities used to build the
 * page allocator zonelists.
 *
 * The difference here is that cycles must be avoided.  If
 * node0 migrates to node1, then neither node1, nor anything
2354 2355 2356
 * node1 migrates to can migrate to node0. Also one node can
 * be migrated to multiple nodes if the target nodes all have
 * a same best-distance against the source node.
2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
 *
 * This function can run simultaneously with readers of
 * node_demotion[].  However, it can not run simultaneously
 * with itself.  Exclusion is provided by memory hotplug events
 * being single-threaded.
 */
static void __set_migration_target_nodes(void)
{
	nodemask_t next_pass	= NODE_MASK_NONE;
	nodemask_t this_pass	= NODE_MASK_NONE;
	nodemask_t used_targets = NODE_MASK_NONE;
2368
	int node, best_distance;
2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397

	/*
	 * Avoid any oddities like cycles that could occur
	 * from changes in the topology.  This will leave
	 * a momentary gap when migration is disabled.
	 */
	disable_all_migrate_targets();

	/*
	 * Allocations go close to CPUs, first.  Assume that
	 * the migration path starts at the nodes with CPUs.
	 */
	next_pass = node_states[N_CPU];
again:
	this_pass = next_pass;
	next_pass = NODE_MASK_NONE;
	/*
	 * To avoid cycles in the migration "graph", ensure
	 * that migration sources are not future targets by
	 * setting them in 'used_targets'.  Do this only
	 * once per pass so that multiple source nodes can
	 * share a target node.
	 *
	 * 'used_targets' will become unavailable in future
	 * passes.  This limits some opportunities for
	 * multiple source nodes to share a destination.
	 */
	nodes_or(used_targets, used_targets, this_pass);

2398 2399
	for_each_node_mask(node, this_pass) {
		best_distance = -1;
2400 2401

		/*
2402 2403 2404
		 * Try to set up the migration path for the node, and the target
		 * migration nodes can be multiple, so doing a loop to find all
		 * the target nodes if they all have a best node distance.
2405
		 */
2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
		do {
			int target_node =
				establish_migrate_target(node, &used_targets,
							 best_distance);

			if (target_node == NUMA_NO_NODE)
				break;

			if (best_distance == -1)
				best_distance = node_distance(node, target_node);

			/*
			 * Visit targets from this pass in the next pass.
			 * Eventually, every node will have been part of
			 * a pass, and will become set in 'used_targets'.
			 */
			node_set(target_node, next_pass);
		} while (1);
2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
	}
	/*
	 * 'next_pass' contains nodes which became migration
	 * targets in this pass.  Make additional passes until
	 * no more migrations targets are available.
	 */
	if (!nodes_empty(next_pass))
		goto again;
}

/*
 * For callers that do not hold get_online_mems() already.
 */
static void set_migration_target_nodes(void)
{
	get_online_mems();
	__set_migration_target_nodes();
	put_online_mems();
}
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455

/*
 * This leaves migrate-on-reclaim transiently disabled between
 * the MEM_GOING_OFFLINE and MEM_OFFLINE events.  This runs
 * whether reclaim-based migration is enabled or not, which
 * ensures that the user can turn reclaim-based migration at
 * any time without needing to recalculate migration targets.
 *
 * These callbacks already hold get_online_mems().  That is why
 * __set_migration_target_nodes() can be used as opposed to
 * set_migration_target_nodes().
 */
static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
2456
						 unsigned long action, void *_arg)
2457
{
2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
	struct memory_notify *arg = _arg;

	/*
	 * Only update the node migration order when a node is
	 * changing status, like online->offline.  This avoids
	 * the overhead of synchronize_rcu() in most cases.
	 */
	if (arg->status_change_nid < 0)
		return notifier_from_errno(0);

2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
	switch (action) {
	case MEM_GOING_OFFLINE:
		/*
		 * Make sure there are not transient states where
		 * an offline node is a migration target.  This
		 * will leave migration disabled until the offline
		 * completes and the MEM_OFFLINE case below runs.
		 */
		disable_all_migrate_targets();
		break;
	case MEM_OFFLINE:
	case MEM_ONLINE:
		/*
		 * Recalculate the target nodes once the node
		 * reaches its final state (online or offline).
		 */
		__set_migration_target_nodes();
		break;
	case MEM_CANCEL_OFFLINE:
		/*
		 * MEM_GOING_OFFLINE disabled all the migration
		 * targets.  Reenable them.
		 */
		__set_migration_target_nodes();
		break;
	case MEM_GOING_ONLINE:
	case MEM_CANCEL_ONLINE:
		break;
	}

	return notifier_from_errno(0);
}

2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
/*
 * React to hotplug events that might affect the migration targets
 * like events that online or offline NUMA nodes.
 *
 * The ordering is also currently dependent on which nodes have
 * CPUs.  That means we need CPU on/offline notification too.
 */
static int migration_online_cpu(unsigned int cpu)
{
	set_migration_target_nodes();
	return 0;
}

static int migration_offline_cpu(unsigned int cpu)
{
	set_migration_target_nodes();
	return 0;
}

2520 2521 2522 2523
static int __init migrate_on_reclaim_init(void)
{
	int ret;

2524 2525 2526 2527 2528
	node_demotion = kmalloc_array(nr_node_ids,
				      sizeof(struct demotion_nodes),
				      GFP_KERNEL);
	WARN_ON(!node_demotion);

2529 2530
	ret = cpuhp_setup_state_nocalls(CPUHP_MM_DEMOTION_DEAD, "mm/demotion:offline",
					NULL, migration_offline_cpu);
2531 2532 2533 2534 2535 2536 2537
	/*
	 * In the unlikely case that this fails, the automatic
	 * migration targets may become suboptimal for nodes
	 * where N_CPU changes.  With such a small impact in a
	 * rare case, do not bother trying to do anything special.
	 */
	WARN_ON(ret < 0);
2538 2539 2540
	ret = cpuhp_setup_state(CPUHP_AP_MM_DEMOTION_ONLINE, "mm/demotion:online",
				migration_online_cpu, NULL);
	WARN_ON(ret < 0);
2541 2542 2543 2544 2545

	hotplug_memory_notifier(migrate_on_reclaim_callback, 100);
	return 0;
}
late_initcall(migrate_on_reclaim_init);
2546
#endif /* CONFIG_HOTPLUG_CPU */
2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607

bool numa_demotion_enabled = false;

#ifdef CONFIG_SYSFS
static ssize_t numa_demotion_enabled_show(struct kobject *kobj,
					  struct kobj_attribute *attr, char *buf)
{
	return sysfs_emit(buf, "%s\n",
			  numa_demotion_enabled ? "true" : "false");
}

static ssize_t numa_demotion_enabled_store(struct kobject *kobj,
					   struct kobj_attribute *attr,
					   const char *buf, size_t count)
{
	if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1))
		numa_demotion_enabled = true;
	else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1))
		numa_demotion_enabled = false;
	else
		return -EINVAL;

	return count;
}

static struct kobj_attribute numa_demotion_enabled_attr =
	__ATTR(demotion_enabled, 0644, numa_demotion_enabled_show,
	       numa_demotion_enabled_store);

static struct attribute *numa_attrs[] = {
	&numa_demotion_enabled_attr.attr,
	NULL,
};

static const struct attribute_group numa_attr_group = {
	.attrs = numa_attrs,
};

static int __init numa_init_sysfs(void)
{
	int err;
	struct kobject *numa_kobj;

	numa_kobj = kobject_create_and_add("numa", mm_kobj);
	if (!numa_kobj) {
		pr_err("failed to create numa kobject\n");
		return -ENOMEM;
	}
	err = sysfs_create_group(numa_kobj, &numa_attr_group);
	if (err) {
		pr_err("failed to register numa group\n");
		goto delete_obj;
	}
	return 0;

delete_obj:
	kobject_put(numa_kobj);
	return err;
}
subsys_initcall(numa_init_sysfs);
#endif
反馈
建议
客服 返回
顶部