migrate.c 67.0 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Memory Migration functionality - linux/mm/migrate.c
4 5 6 7 8 9 10 11 12
 *
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 *
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
C
Christoph Lameter 已提交
13
 * Christoph Lameter
14 15 16
 */

#include <linux/migrate.h>
17
#include <linux/export.h>
18
#include <linux/swap.h>
19
#include <linux/swapops.h>
20
#include <linux/pagemap.h>
21
#include <linux/buffer_head.h>
22
#include <linux/mm_inline.h>
23
#include <linux/nsproxy.h>
24
#include <linux/pagevec.h>
25
#include <linux/ksm.h>
26 27 28 29
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
30
#include <linux/writeback.h>
31 32
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
33
#include <linux/security.h>
34
#include <linux/backing-dev.h>
35
#include <linux/compaction.h>
36
#include <linux/syscalls.h>
37
#include <linux/compat.h>
38
#include <linux/hugetlb.h>
39
#include <linux/hugetlb_cgroup.h>
40
#include <linux/gfp.h>
41
#include <linux/pfn_t.h>
42
#include <linux/memremap.h>
43
#include <linux/userfaultfd_k.h>
44
#include <linux/balloon_compaction.h>
45
#include <linux/page_idle.h>
46
#include <linux/page_owner.h>
47
#include <linux/sched/mm.h>
48
#include <linux/ptrace.h>
49
#include <linux/oom.h>
50
#include <linux/memory.h>
51
#include <linux/random.h>
52

53 54
#include <asm/tlbflush.h>

55 56 57
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>

58 59
#include "internal.h"

60
int isolate_movable_page(struct page *page, isolate_mode_t mode)
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
{
	struct address_space *mapping;

	/*
	 * Avoid burning cycles with pages that are yet under __free_pages(),
	 * or just got freed under us.
	 *
	 * In case we 'win' a race for a movable page being freed under us and
	 * raise its refcount preventing __free_pages() from doing its job
	 * the put_page() at the end of this block will take care of
	 * release this page, thus avoiding a nasty leakage.
	 */
	if (unlikely(!get_page_unless_zero(page)))
		goto out;

	/*
	 * Check PageMovable before holding a PG_lock because page's owner
	 * assumes anybody doesn't touch PG_lock of newly allocated page
79
	 * so unconditionally grabbing the lock ruins page's owner side.
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
	 */
	if (unlikely(!__PageMovable(page)))
		goto out_putpage;
	/*
	 * As movable pages are not isolated from LRU lists, concurrent
	 * compaction threads can race against page migration functions
	 * as well as race against the releasing a page.
	 *
	 * In order to avoid having an already isolated movable page
	 * being (wrongly) re-isolated while it is under migration,
	 * or to avoid attempting to isolate pages being released,
	 * lets be sure we have the page lock
	 * before proceeding with the movable page isolation steps.
	 */
	if (unlikely(!trylock_page(page)))
		goto out_putpage;

	if (!PageMovable(page) || PageIsolated(page))
		goto out_no_isolated;

	mapping = page_mapping(page);
	VM_BUG_ON_PAGE(!mapping, page);

	if (!mapping->a_ops->isolate_page(page, mode))
		goto out_no_isolated;

	/* Driver shouldn't use PG_isolated bit of page->flags */
	WARN_ON_ONCE(PageIsolated(page));
	__SetPageIsolated(page);
	unlock_page(page);

111
	return 0;
112 113 114 115 116 117

out_no_isolated:
	unlock_page(page);
out_putpage:
	put_page(page);
out:
118
	return -EBUSY;
119 120
}

121
static void putback_movable_page(struct page *page)
122 123 124 125 126 127 128 129
{
	struct address_space *mapping;

	mapping = page_mapping(page);
	mapping->a_ops->putback_page(page);
	__ClearPageIsolated(page);
}

130 131 132 133
/*
 * Put previously isolated pages back onto the appropriate lists
 * from where they were once taken off for compaction/migration.
 *
134 135 136
 * This function shall be used whenever the isolated pageset has been
 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 * and isolate_huge_page().
137 138 139 140 141 142
 */
void putback_movable_pages(struct list_head *l)
{
	struct page *page;
	struct page *page2;

143
	list_for_each_entry_safe(page, page2, l, lru) {
144 145 146 147
		if (unlikely(PageHuge(page))) {
			putback_active_hugepage(page);
			continue;
		}
148
		list_del(&page->lru);
149 150 151 152 153
		/*
		 * We isolated non-lru movable page so here we can use
		 * __PageMovable because LRU page's mapping cannot have
		 * PAGE_MAPPING_MOVABLE.
		 */
154
		if (unlikely(__PageMovable(page))) {
155 156 157 158 159 160 161 162 163
			VM_BUG_ON_PAGE(!PageIsolated(page), page);
			lock_page(page);
			if (PageMovable(page))
				putback_movable_page(page);
			else
				__ClearPageIsolated(page);
			unlock_page(page);
			put_page(page);
		} else {
164
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
165
					page_is_file_lru(page), -thp_nr_pages(page));
166
			putback_lru_page(page);
167
		}
168 169 170
	}
}

171 172 173
/*
 * Restore a potential migration pte to a working pte entry
 */
174
static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
175
				 unsigned long addr, void *old)
176
{
177 178
	DEFINE_PAGE_VMA_WALK(pvmw, (struct page *)old, vma, addr,
				PVMW_SYNC | PVMW_MIGRATION);
179 180
	struct page *new;
	pte_t pte;
181 182
	swp_entry_t entry;

183 184
	VM_BUG_ON_PAGE(PageTail(page), page);
	while (page_vma_mapped_walk(&pvmw)) {
185 186 187
		if (PageKsm(page))
			new = page;
		else
188
			new = page - pvmw.pgoff +
189
				linear_page_index(vma, pvmw.address);
190

191 192 193 194 195 196 197 198 199
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
		/* PMD-mapped THP migration entry */
		if (!pvmw.pte) {
			VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
			remove_migration_pmd(&pvmw, new);
			continue;
		}
#endif

200 201 202 203
		get_page(new);
		pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
		if (pte_swp_soft_dirty(*pvmw.pte))
			pte = pte_mksoft_dirty(pte);
204

205 206 207 208
		/*
		 * Recheck VMA as permissions can change since migration started
		 */
		entry = pte_to_swp_entry(*pvmw.pte);
209
		if (is_writable_migration_entry(entry))
210
			pte = maybe_mkwrite(pte, vma);
211 212
		else if (pte_swp_uffd_wp(*pvmw.pte))
			pte = pte_mkuffd_wp(pte);
213

214
		if (unlikely(is_device_private_page(new))) {
215 216 217 218 219 220
			if (pte_write(pte))
				entry = make_writable_device_private_entry(
							page_to_pfn(new));
			else
				entry = make_readable_device_private_entry(
							page_to_pfn(new));
221
			pte = swp_entry_to_pte(entry);
222 223
			if (pte_swp_soft_dirty(*pvmw.pte))
				pte = pte_swp_mksoft_dirty(pte);
224 225
			if (pte_swp_uffd_wp(*pvmw.pte))
				pte = pte_swp_mkuffd_wp(pte);
226
		}
227

228
#ifdef CONFIG_HUGETLB_PAGE
229
		if (PageHuge(new)) {
230 231
			unsigned int shift = huge_page_shift(hstate_vma(vma));

232
			pte = pte_mkhuge(pte);
233
			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
234 235 236 237
			if (PageAnon(new))
				hugepage_add_anon_rmap(new, vma, pvmw.address);
			else
				page_dup_rmap(new, true);
238
			set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
239 240 241 242 243 244
		} else
#endif
		{
			if (PageAnon(new))
				page_add_anon_rmap(new, vma, pvmw.address, false);
			else
245
				page_add_file_rmap(new, vma, false);
246
			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
247
		}
248 249
		if (vma->vm_flags & VM_LOCKED)
			mlock_page_drain(smp_processor_id());
250

251 252 253
		/* No need to invalidate - it was non-present before */
		update_mmu_cache(vma, pvmw.address, pvmw.pte);
	}
254

255
	return true;
256 257
}

258 259 260 261
/*
 * Get rid of all migration entries and replace them by
 * references to the indicated page.
 */
262
void remove_migration_ptes(struct page *old, struct page *new, bool locked)
263
{
264 265 266 267 268
	struct rmap_walk_control rwc = {
		.rmap_one = remove_migration_pte,
		.arg = old,
	};

269 270 271 272
	if (locked)
		rmap_walk_locked(new, &rwc);
	else
		rmap_walk(new, &rwc);
273 274
}

275 276 277 278 279
/*
 * Something used the pte of a page under migration. We need to
 * get to the page and wait until migration is finished.
 * When we return from this function the fault will be retried.
 */
280
void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
281
				spinlock_t *ptl)
282
{
283
	pte_t pte;
284 285
	swp_entry_t entry;

286
	spin_lock(ptl);
287 288 289 290 291 292 293 294
	pte = *ptep;
	if (!is_swap_pte(pte))
		goto out;

	entry = pte_to_swp_entry(pte);
	if (!is_migration_entry(entry))
		goto out;

295
	migration_entry_wait_on_locked(entry, ptep, ptl);
296 297 298 299 300
	return;
out:
	pte_unmap_unlock(ptep, ptl);
}

301 302 303 304 305 306 307 308
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
				unsigned long address)
{
	spinlock_t *ptl = pte_lockptr(mm, pmd);
	pte_t *ptep = pte_offset_map(pmd, address);
	__migration_entry_wait(mm, ptep, ptl);
}

309 310
void migration_entry_wait_huge(struct vm_area_struct *vma,
		struct mm_struct *mm, pte_t *pte)
311
{
312
	spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
313 314 315
	__migration_entry_wait(mm, pte, ptl);
}

316 317 318 319 320 321 322 323
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
{
	spinlock_t *ptl;

	ptl = pmd_lock(mm, pmd);
	if (!is_pmd_migration_entry(*pmd))
		goto unlock;
324
	migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl);
325 326 327 328 329 330
	return;
unlock:
	spin_unlock(ptl);
}
#endif

331
static int expected_page_refs(struct address_space *mapping, struct page *page)
332 333 334
{
	int expected_count = 1;

335
	if (mapping)
336
		expected_count += compound_nr(page) + page_has_private(page);
337 338 339
	return expected_count;
}

340
/*
341
 * Replace the page in the mapping.
342 343 344 345
 *
 * The number of remaining references must be:
 * 1 for anonymous pages without a mapping
 * 2 for pages with a mapping
346
 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
347
 */
348 349
int folio_migrate_mapping(struct address_space *mapping,
		struct folio *newfolio, struct folio *folio, int extra_count)
350
{
351
	XA_STATE(xas, &mapping->i_pages, folio_index(folio));
352 353
	struct zone *oldzone, *newzone;
	int dirty;
354 355
	int expected_count = expected_page_refs(mapping, &folio->page) + extra_count;
	long nr = folio_nr_pages(folio);
356

357
	if (!mapping) {
358
		/* Anonymous page without mapping */
359
		if (folio_ref_count(folio) != expected_count)
360
			return -EAGAIN;
361 362

		/* No turning back from here */
363 364 365 366
		newfolio->index = folio->index;
		newfolio->mapping = folio->mapping;
		if (folio_test_swapbacked(folio))
			__folio_set_swapbacked(newfolio);
367

368
		return MIGRATEPAGE_SUCCESS;
369 370
	}

371 372
	oldzone = folio_zone(folio);
	newzone = folio_zone(newfolio);
373

374
	xas_lock_irq(&xas);
375
	if (!folio_ref_freeze(folio, expected_count)) {
376
		xas_unlock_irq(&xas);
377 378 379
		return -EAGAIN;
	}

380
	/*
381
	 * Now we know that no one else is looking at the folio:
382
	 * no turning back from here.
383
	 */
384 385 386 387 388 389 390 391
	newfolio->index = folio->index;
	newfolio->mapping = folio->mapping;
	folio_ref_add(newfolio, nr); /* add cache reference */
	if (folio_test_swapbacked(folio)) {
		__folio_set_swapbacked(newfolio);
		if (folio_test_swapcache(folio)) {
			folio_set_swapcache(newfolio);
			newfolio->private = folio_get_private(folio);
392 393
		}
	} else {
394
		VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
395 396
	}

397
	/* Move dirty while page refs frozen and newpage not yet exposed */
398
	dirty = folio_test_dirty(folio);
399
	if (dirty) {
400 401
		folio_clear_dirty(folio);
		folio_set_dirty(newfolio);
402 403
	}

404
	xas_store(&xas, newfolio);
405 406

	/*
407 408
	 * Drop cache reference from old page by unfreezing
	 * to one less reference.
409 410
	 * We know this isn't the last reference.
	 */
411
	folio_ref_unfreeze(folio, expected_count - nr);
412

413
	xas_unlock(&xas);
414 415
	/* Leave irq disabled to prevent preemption while updating stats */

416 417 418 419 420 421 422
	/*
	 * If moved to a different zone then also account
	 * the page for that zone. Other VM counters will be
	 * taken care of when we establish references to the
	 * new page and drop references to the old page.
	 *
	 * Note that anonymous pages are accounted for
423
	 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
424 425
	 * are mapped to swap space.
	 */
426
	if (newzone != oldzone) {
427 428 429
		struct lruvec *old_lruvec, *new_lruvec;
		struct mem_cgroup *memcg;

430
		memcg = folio_memcg(folio);
431 432 433
		old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
		new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);

434 435
		__mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
		__mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
436
		if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) {
437 438
			__mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
			__mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
439
		}
440
#ifdef CONFIG_SWAP
441
		if (folio_test_swapcache(folio)) {
442 443 444 445
			__mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
			__mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
		}
#endif
446
		if (dirty && mapping_can_writeback(mapping)) {
447 448 449 450
			__mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
			__mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
			__mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
			__mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
451
		}
452
	}
453
	local_irq_enable();
454

455
	return MIGRATEPAGE_SUCCESS;
456
}
457
EXPORT_SYMBOL(folio_migrate_mapping);
458

459 460
/*
 * The expected number of remaining references is the same as that
461
 * of folio_migrate_mapping().
462 463 464 465
 */
int migrate_huge_page_move_mapping(struct address_space *mapping,
				   struct page *newpage, struct page *page)
{
466
	XA_STATE(xas, &mapping->i_pages, page_index(page));
467 468
	int expected_count;

469
	xas_lock_irq(&xas);
470
	expected_count = 2 + page_has_private(page);
471 472
	if (page_count(page) != expected_count || xas_load(&xas) != page) {
		xas_unlock_irq(&xas);
473 474 475
		return -EAGAIN;
	}

476
	if (!page_ref_freeze(page, expected_count)) {
477
		xas_unlock_irq(&xas);
478 479 480
		return -EAGAIN;
	}

481 482
	newpage->index = page->index;
	newpage->mapping = page->mapping;
483

484 485
	get_page(newpage);

486
	xas_store(&xas, newpage);
487

488
	page_ref_unfreeze(page, expected_count - 1);
489

490
	xas_unlock_irq(&xas);
491

492
	return MIGRATEPAGE_SUCCESS;
493 494
}

495
/*
496
 * Copy the flags and some other ancillary information
497
 */
498
void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
499
{
500 501
	int cpupid;

502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
	if (folio_test_error(folio))
		folio_set_error(newfolio);
	if (folio_test_referenced(folio))
		folio_set_referenced(newfolio);
	if (folio_test_uptodate(folio))
		folio_mark_uptodate(newfolio);
	if (folio_test_clear_active(folio)) {
		VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio);
		folio_set_active(newfolio);
	} else if (folio_test_clear_unevictable(folio))
		folio_set_unevictable(newfolio);
	if (folio_test_workingset(folio))
		folio_set_workingset(newfolio);
	if (folio_test_checked(folio))
		folio_set_checked(newfolio);
	if (folio_test_mappedtodisk(folio))
		folio_set_mappedtodisk(newfolio);
519

520
	/* Move dirty on pages not done by folio_migrate_mapping() */
521 522
	if (folio_test_dirty(folio))
		folio_set_dirty(newfolio);
523

524 525 526 527
	if (folio_test_young(folio))
		folio_set_young(newfolio);
	if (folio_test_idle(folio))
		folio_set_idle(newfolio);
528

529 530 531 532
	/*
	 * Copy NUMA information to the new page, to prevent over-eager
	 * future migrations of this same page.
	 */
533 534
	cpupid = page_cpupid_xchg_last(&folio->page, -1);
	page_cpupid_xchg_last(&newfolio->page, cpupid);
535

536
	folio_migrate_ksm(newfolio, folio);
537 538 539 540
	/*
	 * Please do not reorder this without considering how mm/ksm.c's
	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
	 */
541 542 543
	if (folio_test_swapcache(folio))
		folio_clear_swapcache(folio);
	folio_clear_private(folio);
544 545

	/* page->private contains hugetlb specific flags */
546 547
	if (!folio_test_hugetlb(folio))
		folio->private = NULL;
548 549 550 551 552

	/*
	 * If any waiters have accumulated on the new page then
	 * wake them up.
	 */
553 554
	if (folio_test_writeback(newfolio))
		folio_end_writeback(newfolio);
555

556 557 558 559 560
	/*
	 * PG_readahead shares the same bit with PG_reclaim.  The above
	 * end_page_writeback() may clear PG_readahead mistakenly, so set the
	 * bit after that.
	 */
561 562
	if (folio_test_readahead(folio))
		folio_set_readahead(newfolio);
563

564
	folio_copy_owner(newfolio, folio);
565

566
	if (!folio_test_hugetlb(folio))
567
		mem_cgroup_migrate(folio, newfolio);
568
}
569
EXPORT_SYMBOL(folio_migrate_flags);
570

571
void folio_migrate_copy(struct folio *newfolio, struct folio *folio)
572
{
573 574
	folio_copy(newfolio, folio);
	folio_migrate_flags(newfolio, folio);
575
}
576
EXPORT_SYMBOL(folio_migrate_copy);
577

578 579 580 581
/************************************************************
 *                    Migration functions
 ***********************************************************/

582
/*
583
 * Common logic to directly migrate a single LRU page suitable for
584
 * pages that do not use PagePrivate/PagePrivate2.
585 586 587
 *
 * Pages are locked upon entry and exit.
 */
588
int migrate_page(struct address_space *mapping,
589 590
		struct page *newpage, struct page *page,
		enum migrate_mode mode)
591
{
592 593
	struct folio *newfolio = page_folio(newpage);
	struct folio *folio = page_folio(page);
594 595
	int rc;

596
	BUG_ON(folio_test_writeback(folio));	/* Writeback must be complete */
597

598
	rc = folio_migrate_mapping(mapping, newfolio, folio, 0);
599

600
	if (rc != MIGRATEPAGE_SUCCESS)
601 602
		return rc;

603
	if (mode != MIGRATE_SYNC_NO_COPY)
604
		folio_migrate_copy(newfolio, folio);
605
	else
606
		folio_migrate_flags(newfolio, folio);
607
	return MIGRATEPAGE_SUCCESS;
608 609 610
}
EXPORT_SYMBOL(migrate_page);

611
#ifdef CONFIG_BLOCK
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
							enum migrate_mode mode)
{
	struct buffer_head *bh = head;

	/* Simple case, sync compaction */
	if (mode != MIGRATE_ASYNC) {
		do {
			lock_buffer(bh);
			bh = bh->b_this_page;

		} while (bh != head);

		return true;
	}

	/* async case, we cannot block on lock_buffer so use trylock_buffer */
	do {
		if (!trylock_buffer(bh)) {
			/*
			 * We failed to lock the buffer and cannot stall in
			 * async migration. Release the taken locks
			 */
			struct buffer_head *failed_bh = bh;
			bh = head;
			while (bh != failed_bh) {
				unlock_buffer(bh);
				bh = bh->b_this_page;
			}
			return false;
		}

		bh = bh->b_this_page;
	} while (bh != head);
	return true;
}

650 651 652
static int __buffer_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode,
		bool check_refs)
653 654 655
{
	struct buffer_head *bh, *head;
	int rc;
656
	int expected_count;
657 658

	if (!page_has_buffers(page))
659
		return migrate_page(mapping, newpage, page, mode);
660

661
	/* Check whether page does not have extra refs before we do more work */
662
	expected_count = expected_page_refs(mapping, page);
663 664
	if (page_count(page) != expected_count)
		return -EAGAIN;
665

666 667 668
	head = page_buffers(page);
	if (!buffer_migrate_lock_buffers(head, mode))
		return -EAGAIN;
669

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
	if (check_refs) {
		bool busy;
		bool invalidated = false;

recheck_buffers:
		busy = false;
		spin_lock(&mapping->private_lock);
		bh = head;
		do {
			if (atomic_read(&bh->b_count)) {
				busy = true;
				break;
			}
			bh = bh->b_this_page;
		} while (bh != head);
		if (busy) {
			if (invalidated) {
				rc = -EAGAIN;
				goto unlock_buffers;
			}
690
			spin_unlock(&mapping->private_lock);
691 692 693 694 695 696
			invalidate_bh_lrus();
			invalidated = true;
			goto recheck_buffers;
		}
	}

697
	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
698
	if (rc != MIGRATEPAGE_SUCCESS)
699
		goto unlock_buffers;
700

701
	attach_page_private(newpage, detach_page_private(page));
702 703 704 705 706 707 708 709

	bh = head;
	do {
		set_bh_page(bh, newpage, bh_offset(bh));
		bh = bh->b_this_page;

	} while (bh != head);

710 711 712 713
	if (mode != MIGRATE_SYNC_NO_COPY)
		migrate_page_copy(newpage, page);
	else
		migrate_page_states(newpage, page);
714

715 716
	rc = MIGRATEPAGE_SUCCESS;
unlock_buffers:
717 718
	if (check_refs)
		spin_unlock(&mapping->private_lock);
719 720 721 722 723 724 725
	bh = head;
	do {
		unlock_buffer(bh);
		bh = bh->b_this_page;

	} while (bh != head);

726
	return rc;
727
}
728 729 730 731 732 733 734 735 736 737 738

/*
 * Migration function for pages with buffers. This function can only be used
 * if the underlying filesystem guarantees that no other references to "page"
 * exist. For example attached buffer heads are accessed only under page lock.
 */
int buffer_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	return __buffer_migrate_page(mapping, newpage, page, mode, false);
}
739
EXPORT_SYMBOL(buffer_migrate_page);
740 741 742 743 744 745 746 747 748 749 750 751

/*
 * Same as above except that this variant is more careful and checks that there
 * are also no buffer head references. This function is the right one for
 * mappings where buffer heads are directly looked up and referenced (such as
 * block device mappings).
 */
int buffer_migrate_page_norefs(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	return __buffer_migrate_page(mapping, newpage, page, mode, true);
}
752
#endif
753

754 755 756 757
/*
 * Writeback a page to clean the dirty state
 */
static int writeout(struct address_space *mapping, struct page *page)
758
{
759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = 1,
		.range_start = 0,
		.range_end = LLONG_MAX,
		.for_reclaim = 1
	};
	int rc;

	if (!mapping->a_ops->writepage)
		/* No write method for the address space */
		return -EINVAL;

	if (!clear_page_dirty_for_io(page))
		/* Someone else already triggered a write */
		return -EAGAIN;

776
	/*
777 778 779 780 781 782
	 * A dirty page may imply that the underlying filesystem has
	 * the page on some queue. So the page must be clean for
	 * migration. Writeout may mean we loose the lock and the
	 * page state is no longer what we checked for earlier.
	 * At this point we know that the migration attempt cannot
	 * be successful.
783
	 */
784
	remove_migration_ptes(page, page, false);
785

786
	rc = mapping->a_ops->writepage(page, &wbc);
787

788 789 790 791
	if (rc != AOP_WRITEPAGE_ACTIVATE)
		/* unlocked. Relock */
		lock_page(page);

792
	return (rc < 0) ? -EIO : -EAGAIN;
793 794 795 796 797 798
}

/*
 * Default handling if a filesystem does not provide a migration function.
 */
static int fallback_migrate_page(struct address_space *mapping,
799
	struct page *newpage, struct page *page, enum migrate_mode mode)
800
{
801
	if (PageDirty(page)) {
802
		/* Only writeback pages in full synchronous migration */
803 804 805 806 807
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
808
			return -EBUSY;
809
		}
810
		return writeout(mapping, page);
811
	}
812 813 814 815 816

	/*
	 * Buffers may be managed in a filesystem specific way.
	 * We must have no buffers or drop them.
	 */
817
	if (page_has_private(page) &&
818
	    !try_to_release_page(page, GFP_KERNEL))
819
		return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
820

821
	return migrate_page(mapping, newpage, page, mode);
822 823
}

824 825 826 827 828 829
/*
 * Move a page to a newly allocated page
 * The page is locked and all ptes have been successfully removed.
 *
 * The new page will have replaced the old page if this function
 * is successful.
830 831 832
 *
 * Return value:
 *   < 0 - error code
833
 *  MIGRATEPAGE_SUCCESS - success
834
 */
835
static int move_to_new_page(struct page *newpage, struct page *page,
836
				enum migrate_mode mode)
837 838
{
	struct address_space *mapping;
839 840
	int rc = -EAGAIN;
	bool is_lru = !__PageMovable(page);
841

842 843
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
844 845

	mapping = page_mapping(page);
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863

	if (likely(is_lru)) {
		if (!mapping)
			rc = migrate_page(mapping, newpage, page, mode);
		else if (mapping->a_ops->migratepage)
			/*
			 * Most pages have a mapping and most filesystems
			 * provide a migratepage callback. Anonymous pages
			 * are part of swap space which also has its own
			 * migratepage callback. This is the most common path
			 * for page migration.
			 */
			rc = mapping->a_ops->migratepage(mapping, newpage,
							page, mode);
		else
			rc = fallback_migrate_page(mapping, newpage,
							page, mode);
	} else {
864
		/*
865 866
		 * In case of non-lru page, it could be released after
		 * isolation step. In that case, we shouldn't try migration.
867
		 */
868 869 870 871 872 873 874 875 876 877 878 879
		VM_BUG_ON_PAGE(!PageIsolated(page), page);
		if (!PageMovable(page)) {
			rc = MIGRATEPAGE_SUCCESS;
			__ClearPageIsolated(page);
			goto out;
		}

		rc = mapping->a_ops->migratepage(mapping, newpage,
						page, mode);
		WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
			!PageIsolated(page));
	}
880

881 882 883 884 885
	/*
	 * When successful, old pagecache page->mapping must be cleared before
	 * page is freed; but stats require that PageAnon be left as PageAnon.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
886 887 888 889 890 891 892 893 894 895 896
		if (__PageMovable(page)) {
			VM_BUG_ON_PAGE(!PageIsolated(page), page);

			/*
			 * We clear PG_movable under page_lock so any compactor
			 * cannot try to migrate this page.
			 */
			__ClearPageIsolated(page);
		}

		/*
897
		 * Anonymous and movable page->mapping will be cleared by
898 899 900 901
		 * free_pages_prepare so don't reset it here for keeping
		 * the type to work PageAnon, for example.
		 */
		if (!PageMappingFlags(page))
902
			page->mapping = NULL;
903

904
		if (likely(!is_zone_device_page(newpage)))
905 906
			flush_dcache_page(newpage);

907
	}
908
out:
909 910 911
	return rc;
}

912
static int __unmap_and_move(struct page *page, struct page *newpage,
913
				int force, enum migrate_mode mode)
914
{
915
	struct folio *folio = page_folio(page);
916
	int rc = -EAGAIN;
917
	bool page_was_mapped = false;
918
	struct anon_vma *anon_vma = NULL;
919
	bool is_lru = !__PageMovable(page);
920

N
Nick Piggin 已提交
921
	if (!trylock_page(page)) {
922
		if (!force || mode == MIGRATE_ASYNC)
923
			goto out;
924 925 926 927 928 929 930

		/*
		 * It's not safe for direct compaction to call lock_page.
		 * For example, during page readahead pages are added locked
		 * to the LRU. Later, when the IO completes the pages are
		 * marked uptodate and unlocked. However, the queueing
		 * could be merging multiple pages for one bio (e.g.
931
		 * mpage_readahead). If an allocation happens for the
932 933 934 935 936 937 938
		 * second or third page, the process can end up locking
		 * the same page twice and deadlocking. Rather than
		 * trying to be clever about what pages can be locked,
		 * avoid the use of lock_page for direct compaction
		 * altogether.
		 */
		if (current->flags & PF_MEMALLOC)
939
			goto out;
940

941 942 943 944
		lock_page(page);
	}

	if (PageWriteback(page)) {
945
		/*
946
		 * Only in the case of a full synchronous migration is it
947 948 949
		 * necessary to wait for PageWriteback. In the async case,
		 * the retry loop is too short and in the sync-light case,
		 * the overhead of stalling is too much
950
		 */
951 952 953 954 955
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
956
			rc = -EBUSY;
957
			goto out_unlock;
958 959
		}
		if (!force)
960
			goto out_unlock;
961 962
		wait_on_page_writeback(page);
	}
963

964
	/*
965
	 * By try_to_migrate(), page->mapcount goes down to 0 here. In this case,
966
	 * we cannot notice that anon_vma is freed while we migrates a page.
967
	 * This get_anon_vma() delays freeing anon_vma pointer until the end
968
	 * of migration. File cache pages are no problem because of page_lock()
969 970
	 * File Caches may use write_page() or lock_page() in migration, then,
	 * just care Anon page here.
971 972 973 974 975 976
	 *
	 * Only page_get_anon_vma() understands the subtleties of
	 * getting a hold on an anon_vma from outside one of its mms.
	 * But if we cannot get anon_vma, then we won't need it anyway,
	 * because that implies that the anon page is no longer mapped
	 * (and cannot be remapped so long as we hold the page lock).
977
	 */
978
	if (PageAnon(page) && !PageKsm(page))
979
		anon_vma = page_get_anon_vma(page);
980

981 982 983 984 985 986 987 988 989 990 991
	/*
	 * Block others from accessing the new page when we get around to
	 * establishing additional references. We are usually the only one
	 * holding a reference to newpage at this point. We used to have a BUG
	 * here if trylock_page(newpage) fails, but would like to allow for
	 * cases where there might be a race with the previous use of newpage.
	 * This is much like races on refcount of oldpage: just don't BUG().
	 */
	if (unlikely(!trylock_page(newpage)))
		goto out_unlock;

992 993 994 995 996
	if (unlikely(!is_lru)) {
		rc = move_to_new_page(newpage, page, mode);
		goto out_unlock_both;
	}

997
	/*
998 999 1000 1001 1002
	 * Corner case handling:
	 * 1. When a new swap-cache page is read into, it is added to the LRU
	 * and treated as swapcache but it has no rmap yet.
	 * Calling try_to_unmap() against a page->mapping==NULL page will
	 * trigger a BUG.  So handle it here.
1003
	 * 2. An orphaned page (see truncate_cleanup_page) might have
1004 1005 1006 1007
	 * fs-private metadata. The page can be picked up due to memory
	 * offlining.  Everywhere else except page reclaim, the page is
	 * invisible to the vm, so the page can not be migrated.  So try to
	 * free the metadata, so the page can be freed.
1008
	 */
1009
	if (!page->mapping) {
1010
		VM_BUG_ON_PAGE(PageAnon(page), page);
1011
		if (page_has_private(page)) {
1012
			try_to_free_buffers(page);
1013
			goto out_unlock_both;
1014
		}
1015 1016
	} else if (page_mapped(page)) {
		/* Establish migration ptes */
1017 1018
		VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
				page);
1019
		try_to_migrate(folio, 0);
1020
		page_was_mapped = true;
1021
	}
1022

1023
	if (!page_mapped(page))
1024
		rc = move_to_new_page(newpage, page, mode);
1025

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	/*
	 * When successful, push newpage to LRU immediately: so that if it
	 * turns out to be an mlocked page, remove_migration_ptes() will
	 * automatically build up the correct newpage->mlock_count for it.
	 *
	 * We would like to do something similar for the old page, when
	 * unsuccessful, and other cases when a page has been temporarily
	 * isolated from the unevictable LRU: but this case is the easiest.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
		lru_cache_add(newpage);
		if (page_was_mapped)
			lru_add_drain();
	}

1041 1042
	if (page_was_mapped)
		remove_migration_ptes(page,
1043
			rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1044

1045 1046 1047
out_unlock_both:
	unlock_page(newpage);
out_unlock:
1048
	/* Drop an anon_vma reference if we took one */
1049
	if (anon_vma)
1050
		put_anon_vma(anon_vma);
1051
	unlock_page(page);
1052
out:
1053
	/*
1054
	 * If migration is successful, decrease refcount of the newpage,
1055
	 * which will not free the page because new page owner increased
1056
	 * refcounter.
1057
	 */
1058 1059
	if (rc == MIGRATEPAGE_SUCCESS)
		put_page(newpage);
1060

1061 1062
	return rc;
}
1063

1064 1065 1066 1067
/*
 * Obtain the lock on page, remove all ptes and migrate the page
 * to the newly allocated page in newpage.
 */
1068
static int unmap_and_move(new_page_t get_new_page,
1069 1070
				   free_page_t put_new_page,
				   unsigned long private, struct page *page,
1071
				   int force, enum migrate_mode mode,
1072 1073
				   enum migrate_reason reason,
				   struct list_head *ret)
1074
{
1075
	int rc = MIGRATEPAGE_SUCCESS;
1076
	struct page *newpage = NULL;
1077

1078
	if (!thp_migration_supported() && PageTransHuge(page))
1079
		return -ENOSYS;
1080

1081 1082
	if (page_count(page) == 1) {
		/* page was freed from under us. So we are done. */
1083 1084
		ClearPageActive(page);
		ClearPageUnevictable(page);
1085 1086 1087 1088 1089 1090
		if (unlikely(__PageMovable(page))) {
			lock_page(page);
			if (!PageMovable(page))
				__ClearPageIsolated(page);
			unlock_page(page);
		}
1091 1092 1093
		goto out;
	}

1094 1095 1096 1097
	newpage = get_new_page(page, private);
	if (!newpage)
		return -ENOMEM;

1098
	rc = __unmap_and_move(page, newpage, force, mode);
1099
	if (rc == MIGRATEPAGE_SUCCESS)
1100
		set_page_owner_migrate_reason(newpage, reason);
1101

1102
out:
1103
	if (rc != -EAGAIN) {
1104 1105 1106
		/*
		 * A page that has been migrated has all references
		 * removed and will be freed. A page that has not been
1107
		 * migrated will have kept its references and be restored.
1108 1109
		 */
		list_del(&page->lru);
1110
	}
1111

1112 1113 1114 1115 1116 1117
	/*
	 * If migration is successful, releases reference grabbed during
	 * isolation. Otherwise, restore the page to right list unless
	 * we want to retry.
	 */
	if (rc == MIGRATEPAGE_SUCCESS) {
1118 1119 1120 1121 1122 1123
		/*
		 * Compaction can migrate also non-LRU pages which are
		 * not accounted to NR_ISOLATED_*. They can be recognized
		 * as __PageMovable
		 */
		if (likely(!__PageMovable(page)))
1124
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1125
					page_is_file_lru(page), -thp_nr_pages(page));
1126

1127
		if (reason != MR_MEMORY_FAILURE)
1128
			/*
1129
			 * We release the page in page_handle_poison.
1130
			 */
1131
			put_page(page);
1132
	} else {
1133 1134
		if (rc != -EAGAIN)
			list_add_tail(&page->lru, ret);
1135

1136 1137 1138 1139
		if (put_new_page)
			put_new_page(newpage, private);
		else
			put_page(newpage);
1140
	}
1141

1142 1143 1144
	return rc;
}

1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
/*
 * Counterpart of unmap_and_move_page() for hugepage migration.
 *
 * This function doesn't wait the completion of hugepage I/O
 * because there is no race between I/O and migration for hugepage.
 * Note that currently hugepage I/O occurs only in direct I/O
 * where no lock is held and PG_writeback is irrelevant,
 * and writeback status of all subpages are counted in the reference
 * count of the head page (i.e. if all subpages of a 2MB hugepage are
 * under direct I/O, the reference of the head page is 512 and a bit more.)
 * This means that when we try to migrate hugepage whose subpages are
 * doing direct I/O, some references remain after try_to_unmap() and
 * hugepage migration fails without data corruption.
 *
 * There is also no race when direct I/O is issued on the page under migration,
 * because then pte is replaced with migration swap entry and direct I/O code
 * will wait in the page fault for migration to complete.
 */
static int unmap_and_move_huge_page(new_page_t get_new_page,
1164 1165
				free_page_t put_new_page, unsigned long private,
				struct page *hpage, int force,
1166 1167
				enum migrate_mode mode, int reason,
				struct list_head *ret)
1168
{
1169
	struct folio *src = page_folio(hpage);
1170
	int rc = -EAGAIN;
1171
	int page_was_mapped = 0;
1172
	struct page *new_hpage;
1173
	struct anon_vma *anon_vma = NULL;
1174
	struct address_space *mapping = NULL;
1175

1176
	/*
1177
	 * Migratability of hugepages depends on architectures and their size.
1178 1179 1180 1181 1182
	 * This check is necessary because some callers of hugepage migration
	 * like soft offline and memory hotremove don't walk through page
	 * tables or check whether the hugepage is pmd-based or not before
	 * kicking migration.
	 */
1183
	if (!hugepage_migration_supported(page_hstate(hpage))) {
1184
		list_move_tail(&hpage->lru, ret);
1185
		return -ENOSYS;
1186
	}
1187

1188 1189 1190 1191 1192 1193
	if (page_count(hpage) == 1) {
		/* page was freed from under us. So we are done. */
		putback_active_hugepage(hpage);
		return MIGRATEPAGE_SUCCESS;
	}

1194
	new_hpage = get_new_page(hpage, private);
1195 1196 1197 1198
	if (!new_hpage)
		return -ENOMEM;

	if (!trylock_page(hpage)) {
1199
		if (!force)
1200
			goto out;
1201 1202 1203 1204 1205 1206 1207
		switch (mode) {
		case MIGRATE_SYNC:
		case MIGRATE_SYNC_NO_COPY:
			break;
		default:
			goto out;
		}
1208 1209 1210
		lock_page(hpage);
	}

1211 1212 1213 1214 1215
	/*
	 * Check for pages which are in the process of being freed.  Without
	 * page_mapping() set, hugetlbfs specific move page routine will not
	 * be called and we could leak usage counts for subpools.
	 */
1216
	if (hugetlb_page_subpool(hpage) && !page_mapping(hpage)) {
1217 1218 1219 1220
		rc = -EBUSY;
		goto out_unlock;
	}

1221 1222
	if (PageAnon(hpage))
		anon_vma = page_get_anon_vma(hpage);
1223

1224 1225 1226
	if (unlikely(!trylock_page(new_hpage)))
		goto put_anon;

1227
	if (page_mapped(hpage)) {
1228
		bool mapping_locked = false;
1229
		enum ttu_flags ttu = 0;
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244

		if (!PageAnon(hpage)) {
			/*
			 * In shared mappings, try_to_unmap could potentially
			 * call huge_pmd_unshare.  Because of this, take
			 * semaphore in write mode here and set TTU_RMAP_LOCKED
			 * to let lower levels know we have taken the lock.
			 */
			mapping = hugetlb_page_mapping_lock_write(hpage);
			if (unlikely(!mapping))
				goto unlock_put_anon;

			mapping_locked = true;
			ttu |= TTU_RMAP_LOCKED;
		}
1245

1246
		try_to_migrate(src, ttu);
1247
		page_was_mapped = 1;
1248 1249 1250

		if (mapping_locked)
			i_mmap_unlock_write(mapping);
1251
	}
1252 1253

	if (!page_mapped(hpage))
1254
		rc = move_to_new_page(new_hpage, hpage, mode);
1255

1256
	if (page_was_mapped)
1257
		remove_migration_ptes(hpage,
1258
			rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1259

1260
unlock_put_anon:
1261 1262 1263
	unlock_page(new_hpage);

put_anon:
H
Hugh Dickins 已提交
1264
	if (anon_vma)
1265
		put_anon_vma(anon_vma);
1266

1267
	if (rc == MIGRATEPAGE_SUCCESS) {
1268
		move_hugetlb_state(hpage, new_hpage, reason);
1269 1270
		put_new_page = NULL;
	}
1271

1272
out_unlock:
1273
	unlock_page(hpage);
1274
out:
1275
	if (rc == MIGRATEPAGE_SUCCESS)
1276
		putback_active_hugepage(hpage);
1277
	else if (rc != -EAGAIN)
1278
		list_move_tail(&hpage->lru, ret);
1279 1280 1281 1282 1283 1284

	/*
	 * If migration was not successful and there's a freeing callback, use
	 * it.  Otherwise, put_page() will drop the reference grabbed during
	 * isolation.
	 */
1285
	if (put_new_page)
1286 1287
		put_new_page(new_hpage, private);
	else
1288
		putback_active_hugepage(new_hpage);
1289

1290 1291 1292
	return rc;
}

1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
static inline int try_split_thp(struct page *page, struct page **page2,
				struct list_head *from)
{
	int rc = 0;

	lock_page(page);
	rc = split_huge_page_to_list(page, from);
	unlock_page(page);
	if (!rc)
		list_safe_reset_next(page, *page2, lru);

	return rc;
}

1307
/*
1308 1309
 * migrate_pages - migrate the pages specified in a list, to the free pages
 *		   supplied as the target for the page migration
1310
 *
1311 1312 1313
 * @from:		The list of pages to be migrated.
 * @get_new_page:	The function used to allocate free pages to be used
 *			as the target of the page migration.
1314 1315
 * @put_new_page:	The function used to free target pages if migration
 *			fails, or NULL if no special handling is necessary.
1316 1317 1318 1319
 * @private:		Private data to be passed on to get_new_page()
 * @mode:		The migration mode that specifies the constraints for
 *			page migration, if any.
 * @reason:		The reason for page migration.
1320
 * @ret_succeeded:	Set to the number of normal pages migrated successfully if
1321
 *			the caller passes a non-NULL pointer.
1322
 *
1323 1324
 * The function returns after 10 attempts or if no pages are movable any more
 * because the list has become empty or no retryable pages exist any more.
1325 1326
 * It is caller's responsibility to call putback_movable_pages() to return pages
 * to the LRU or free list only if ret != 0.
1327
 *
1328 1329 1330
 * Returns the number of {normal page, THP, hugetlb} that were not migrated, or
 * an error code. The number of THP splits will be considered as the number of
 * non-migrated THP, no matter how many subpages of the THP are migrated successfully.
1331
 */
1332
int migrate_pages(struct list_head *from, new_page_t get_new_page,
1333
		free_page_t put_new_page, unsigned long private,
1334
		enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
1335
{
1336
	int retry = 1;
1337
	int thp_retry = 1;
1338
	int nr_failed = 0;
1339
	int nr_failed_pages = 0;
1340
	int nr_succeeded = 0;
1341 1342 1343
	int nr_thp_succeeded = 0;
	int nr_thp_failed = 0;
	int nr_thp_split = 0;
1344
	int pass = 0;
1345
	bool is_thp = false;
1346 1347 1348
	struct page *page;
	struct page *page2;
	int swapwrite = current->flags & PF_SWAPWRITE;
1349
	int rc, nr_subpages;
1350
	LIST_HEAD(ret_pages);
1351
	LIST_HEAD(thp_split_pages);
1352
	bool nosplit = (reason == MR_NUMA_MISPLACED);
1353
	bool no_subpage_counting = false;
1354

1355 1356
	trace_mm_migrate_pages_start(mode, reason);

1357 1358 1359
	if (!swapwrite)
		current->flags |= PF_SWAPWRITE;

1360
thp_subpage_migration:
1361
	for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
1362
		retry = 0;
1363
		thp_retry = 0;
1364

1365
		list_for_each_entry_safe(page, page2, from, lru) {
1366
retry:
1367 1368 1369 1370 1371
			/*
			 * THP statistics is based on the source huge page.
			 * Capture required information that might get lost
			 * during migration.
			 */
1372
			is_thp = PageTransHuge(page) && !PageHuge(page);
1373
			nr_subpages = compound_nr(page);
1374
			cond_resched();
1375

1376 1377
			if (PageHuge(page))
				rc = unmap_and_move_huge_page(get_new_page,
1378
						put_new_page, private, page,
1379 1380
						pass > 2, mode, reason,
						&ret_pages);
1381
			else
1382
				rc = unmap_and_move(get_new_page, put_new_page,
1383
						private, page, pass > 2, mode,
1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
						reason, &ret_pages);
			/*
			 * The rules are:
			 *	Success: non hugetlb page will be freed, hugetlb
			 *		 page will be put back
			 *	-EAGAIN: stay on the from list
			 *	-ENOMEM: stay on the from list
			 *	Other errno: put on ret_pages list then splice to
			 *		     from list
			 */
1394
			switch(rc) {
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
			/*
			 * THP migration might be unsupported or the
			 * allocation could've failed so we should
			 * retry on the same page with the THP split
			 * to base pages.
			 *
			 * Head page is retried immediately and tail
			 * pages are added to the tail of the list so
			 * we encounter them after the rest of the list
			 * is processed.
			 */
			case -ENOSYS:
				/* THP migration is unsupported */
				if (is_thp) {
1409 1410
					nr_thp_failed++;
					if (!try_split_thp(page, &page2, &thp_split_pages)) {
1411 1412 1413 1414
						nr_thp_split++;
						goto retry;
					}

1415
					nr_failed_pages += nr_subpages;
1416 1417 1418 1419
					break;
				}

				/* Hugetlb migration is unsupported */
1420 1421
				if (!no_subpage_counting)
					nr_failed++;
1422
				nr_failed_pages += nr_subpages;
1423
				break;
1424
			case -ENOMEM:
1425
				/*
1426 1427
				 * When memory is low, don't bother to try to migrate
				 * other pages, just exit.
1428
				 * THP NUMA faulting doesn't split THP to retry.
1429
				 */
1430
				if (is_thp && !nosplit) {
1431 1432
					nr_thp_failed++;
					if (!try_split_thp(page, &page2, &thp_split_pages)) {
1433
						nr_thp_split++;
1434 1435
						goto retry;
					}
1436

1437
					nr_failed_pages += nr_subpages;
1438 1439
					goto out;
				}
1440 1441 1442

				if (!no_subpage_counting)
					nr_failed++;
1443
				nr_failed_pages += nr_subpages;
1444
				goto out;
1445
			case -EAGAIN:
1446 1447 1448 1449
				if (is_thp) {
					thp_retry++;
					break;
				}
1450
				retry++;
1451
				break;
1452
			case MIGRATEPAGE_SUCCESS:
1453
				nr_succeeded += nr_subpages;
1454 1455 1456 1457
				if (is_thp) {
					nr_thp_succeeded++;
					break;
				}
1458 1459
				break;
			default:
1460
				/*
1461
				 * Permanent failure (-EBUSY, etc.):
1462 1463 1464 1465
				 * unlike -EAGAIN case, the failed page is
				 * removed from migration page list and not
				 * retried in the next outer loop.
				 */
1466 1467
				if (is_thp) {
					nr_thp_failed++;
1468
					nr_failed_pages += nr_subpages;
1469 1470
					break;
				}
1471 1472 1473

				if (!no_subpage_counting)
					nr_failed++;
1474
				nr_failed_pages += nr_subpages;
1475
				break;
1476
			}
1477 1478
		}
	}
1479
	nr_failed += retry;
1480
	nr_thp_failed += thp_retry;
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
	/*
	 * Try to migrate subpages of fail-to-migrate THPs, no nr_failed
	 * counting in this round, since all subpages of a THP is counted
	 * as 1 failure in the first round.
	 */
	if (!list_empty(&thp_split_pages)) {
		/*
		 * Move non-migrated pages (after 10 retries) to ret_pages
		 * to avoid migrating them again.
		 */
		list_splice_init(from, &ret_pages);
		list_splice_init(&thp_split_pages, from);
		no_subpage_counting = true;
		retry = 1;
		goto thp_subpage_migration;
	}

	rc = nr_failed + nr_thp_failed;
1499
out:
1500 1501 1502 1503 1504 1505
	/*
	 * Put the permanent failure page back to migration list, they
	 * will be put back to the right list by the caller.
	 */
	list_splice(&ret_pages, from);

1506
	count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1507
	count_vm_events(PGMIGRATE_FAIL, nr_failed_pages);
1508 1509 1510
	count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
	count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
	count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
1511
	trace_mm_migrate_pages(nr_succeeded, nr_failed_pages, nr_thp_succeeded,
1512
			       nr_thp_failed, nr_thp_split, mode, reason);
1513

1514 1515 1516
	if (!swapwrite)
		current->flags &= ~PF_SWAPWRITE;

1517 1518 1519
	if (ret_succeeded)
		*ret_succeeded = nr_succeeded;

1520
	return rc;
1521
}
1522

1523
struct page *alloc_migration_target(struct page *page, unsigned long private)
1524
{
1525 1526
	struct migration_target_control *mtc;
	gfp_t gfp_mask;
1527 1528
	unsigned int order = 0;
	struct page *new_page = NULL;
1529 1530 1531 1532 1533 1534 1535 1536
	int nid;
	int zidx;

	mtc = (struct migration_target_control *)private;
	gfp_mask = mtc->gfp_mask;
	nid = mtc->nid;
	if (nid == NUMA_NO_NODE)
		nid = page_to_nid(page);
1537

1538 1539 1540
	if (PageHuge(page)) {
		struct hstate *h = page_hstate(compound_head(page));

1541 1542
		gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
		return alloc_huge_page_nodemask(h, nid, mtc->nmask, gfp_mask);
1543
	}
1544 1545

	if (PageTransHuge(page)) {
1546 1547 1548 1549 1550
		/*
		 * clear __GFP_RECLAIM to make the migration callback
		 * consistent with regular THP allocations.
		 */
		gfp_mask &= ~__GFP_RECLAIM;
1551 1552 1553
		gfp_mask |= GFP_TRANSHUGE;
		order = HPAGE_PMD_ORDER;
	}
1554 1555
	zidx = zone_idx(page_zone(page));
	if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
1556 1557
		gfp_mask |= __GFP_HIGHMEM;

1558
	new_page = __alloc_pages(gfp_mask, order, nid, mtc->nmask);
1559 1560 1561 1562 1563 1564 1565

	if (new_page && PageTransHuge(new_page))
		prep_transhuge_page(new_page);

	return new_page;
}

1566 1567
#ifdef CONFIG_NUMA

1568
static int store_status(int __user *status, int start, int value, int nr)
1569
{
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
	while (nr-- > 0) {
		if (put_user(value, status + start))
			return -EFAULT;
		start++;
	}

	return 0;
}

static int do_move_pages_to_node(struct mm_struct *mm,
		struct list_head *pagelist, int node)
{
	int err;
1583 1584 1585 1586
	struct migration_target_control mtc = {
		.nid = node,
		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
	};
1587

1588
	err = migrate_pages(pagelist, alloc_migration_target, NULL,
1589
		(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1590 1591 1592
	if (err)
		putback_movable_pages(pagelist);
	return err;
1593 1594 1595
}

/*
1596 1597
 * Resolves the given address to a struct page, isolates it from the LRU and
 * puts it to the given pagelist.
1598 1599 1600 1601 1602
 * Returns:
 *     errno - if the page cannot be found/isolated
 *     0 - when it doesn't have to be migrated because it is already on the
 *         target node
 *     1 - when it has been queued
1603
 */
1604 1605
static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
		int node, struct list_head *pagelist, bool migrate_all)
1606
{
1607 1608 1609
	struct vm_area_struct *vma;
	struct page *page;
	unsigned int follflags;
1610 1611
	int err;

1612
	mmap_read_lock(mm);
1613 1614 1615 1616
	err = -EFAULT;
	vma = find_vma(mm, addr);
	if (!vma || addr < vma->vm_start || !vma_migratable(vma))
		goto out;
1617

1618 1619 1620
	/* FOLL_DUMP to ignore special (like zero) pages */
	follflags = FOLL_GET | FOLL_DUMP;
	page = follow_page(vma, addr, follflags);
1621

1622 1623 1624
	err = PTR_ERR(page);
	if (IS_ERR(page))
		goto out;
1625

1626 1627 1628
	err = -ENOENT;
	if (!page)
		goto out;
1629

1630 1631 1632
	err = 0;
	if (page_to_nid(page) == node)
		goto out_putpage;
1633

1634 1635 1636
	err = -EACCES;
	if (page_mapcount(page) > 1 && !migrate_all)
		goto out_putpage;
1637

1638 1639 1640
	if (PageHuge(page)) {
		if (PageHead(page)) {
			isolate_huge_page(page, pagelist);
1641
			err = 1;
1642
		}
1643 1644
	} else {
		struct page *head;
1645

1646 1647
		head = compound_head(page);
		err = isolate_lru_page(head);
1648
		if (err)
1649
			goto out_putpage;
1650

1651
		err = 1;
1652 1653
		list_add_tail(&head->lru, pagelist);
		mod_node_page_state(page_pgdat(head),
1654
			NR_ISOLATED_ANON + page_is_file_lru(head),
1655
			thp_nr_pages(head));
1656 1657 1658 1659 1660 1661 1662 1663 1664
	}
out_putpage:
	/*
	 * Either remove the duplicate refcount from
	 * isolate_lru_page() or drop the page ref if it was
	 * not isolated.
	 */
	put_page(page);
out:
1665
	mmap_read_unlock(mm);
1666 1667 1668
	return err;
}

1669 1670 1671 1672 1673 1674
static int move_pages_and_store_status(struct mm_struct *mm, int node,
		struct list_head *pagelist, int __user *status,
		int start, int i, unsigned long nr_pages)
{
	int err;

1675 1676 1677
	if (list_empty(pagelist))
		return 0;

1678 1679 1680 1681 1682 1683
	err = do_move_pages_to_node(mm, pagelist, node);
	if (err) {
		/*
		 * Positive err means the number of failed
		 * pages to migrate.  Since we are going to
		 * abort and return the number of non-migrated
1684
		 * pages, so need to include the rest of the
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694
		 * nr_pages that have not been attempted as
		 * well.
		 */
		if (err > 0)
			err += nr_pages - i - 1;
		return err;
	}
	return store_status(status, start, node, i - start);
}

1695 1696 1697 1698
/*
 * Migrate an array of page address onto an array of nodes and fill
 * the corresponding array of status.
 */
1699
static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1700 1701 1702 1703 1704
			 unsigned long nr_pages,
			 const void __user * __user *pages,
			 const int __user *nodes,
			 int __user *status, int flags)
{
1705 1706 1707 1708
	int current_node = NUMA_NO_NODE;
	LIST_HEAD(pagelist);
	int start, i;
	int err = 0, err1;
1709

1710
	lru_cache_disable();
1711

1712 1713 1714 1715
	for (i = start = 0; i < nr_pages; i++) {
		const void __user *p;
		unsigned long addr;
		int node;
1716

1717 1718 1719 1720 1721
		err = -EFAULT;
		if (get_user(p, pages + i))
			goto out_flush;
		if (get_user(node, nodes + i))
			goto out_flush;
1722
		addr = (unsigned long)untagged_addr(p);
1723 1724 1725 1726 1727 1728

		err = -ENODEV;
		if (node < 0 || node >= MAX_NUMNODES)
			goto out_flush;
		if (!node_state(node, N_MEMORY))
			goto out_flush;
1729

1730 1731 1732 1733 1734 1735 1736 1737
		err = -EACCES;
		if (!node_isset(node, task_nodes))
			goto out_flush;

		if (current_node == NUMA_NO_NODE) {
			current_node = node;
			start = i;
		} else if (node != current_node) {
1738 1739
			err = move_pages_and_store_status(mm, current_node,
					&pagelist, status, start, i, nr_pages);
1740 1741 1742 1743
			if (err)
				goto out;
			start = i;
			current_node = node;
1744 1745
		}

1746 1747 1748 1749 1750 1751
		/*
		 * Errors in the page lookup or isolation are not fatal and we simply
		 * report them via status
		 */
		err = add_page_for_migration(mm, addr, current_node,
				&pagelist, flags & MPOL_MF_MOVE_ALL);
1752

1753
		if (err > 0) {
1754 1755 1756
			/* The page is successfully queued for migration */
			continue;
		}
1757

1758 1759 1760 1761 1762
		/*
		 * If the page is already on the target node (!err), store the
		 * node, otherwise, store the err.
		 */
		err = store_status(status, i, err ? : current_node, 1);
1763 1764
		if (err)
			goto out_flush;
1765

1766 1767
		err = move_pages_and_store_status(mm, current_node, &pagelist,
				status, start, i, nr_pages);
1768 1769
		if (err)
			goto out;
1770
		current_node = NUMA_NO_NODE;
1771
	}
1772 1773
out_flush:
	/* Make sure we do not overwrite the existing error */
1774 1775
	err1 = move_pages_and_store_status(mm, current_node, &pagelist,
				status, start, i, nr_pages);
1776
	if (err >= 0)
1777
		err = err1;
1778
out:
1779
	lru_cache_enable();
1780 1781 1782
	return err;
}

1783
/*
1784
 * Determine the nodes of an array of pages and store it in an array of status.
1785
 */
1786 1787
static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
				const void __user **pages, int *status)
1788
{
1789 1790
	unsigned long i;

1791
	mmap_read_lock(mm);
1792

1793
	for (i = 0; i < nr_pages; i++) {
1794
		unsigned long addr = (unsigned long)(*pages);
1795 1796
		struct vm_area_struct *vma;
		struct page *page;
1797
		int err = -EFAULT;
1798

1799 1800
		vma = vma_lookup(mm, addr);
		if (!vma)
1801 1802
			goto set_status;

1803 1804
		/* FOLL_DUMP to ignore special (like zero) pages */
		page = follow_page(vma, addr, FOLL_DUMP);
1805 1806 1807 1808 1809

		err = PTR_ERR(page);
		if (IS_ERR(page))
			goto set_status;

1810
		err = page ? page_to_nid(page) : -ENOENT;
1811
set_status:
1812 1813 1814 1815 1816 1817
		*status = err;

		pages++;
		status++;
	}

1818
	mmap_read_unlock(mm);
1819 1820
}

1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
static int get_compat_pages_array(const void __user *chunk_pages[],
				  const void __user * __user *pages,
				  unsigned long chunk_nr)
{
	compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages;
	compat_uptr_t p;
	int i;

	for (i = 0; i < chunk_nr; i++) {
		if (get_user(p, pages32 + i))
			return -EFAULT;
		chunk_pages[i] = compat_ptr(p);
	}

	return 0;
}

1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
/*
 * Determine the nodes of a user array of pages and store it in
 * a user array of status.
 */
static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
			 const void __user * __user *pages,
			 int __user *status)
{
#define DO_PAGES_STAT_CHUNK_NR 16
	const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
	int chunk_status[DO_PAGES_STAT_CHUNK_NR];

1850 1851
	while (nr_pages) {
		unsigned long chunk_nr;
1852

1853 1854 1855 1856
		chunk_nr = nr_pages;
		if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
			chunk_nr = DO_PAGES_STAT_CHUNK_NR;

1857 1858 1859 1860 1861 1862 1863 1864 1865
		if (in_compat_syscall()) {
			if (get_compat_pages_array(chunk_pages, pages,
						   chunk_nr))
				break;
		} else {
			if (copy_from_user(chunk_pages, pages,
				      chunk_nr * sizeof(*chunk_pages)))
				break;
		}
1866 1867 1868

		do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);

1869 1870
		if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
			break;
1871

1872 1873 1874 1875 1876
		pages += chunk_nr;
		status += chunk_nr;
		nr_pages -= chunk_nr;
	}
	return nr_pages ? -EFAULT : 0;
1877 1878
}

1879
static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
1880 1881 1882 1883
{
	struct task_struct *task;
	struct mm_struct *mm;

1884 1885 1886 1887 1888 1889 1890 1891 1892
	/*
	 * There is no need to check if current process has the right to modify
	 * the specified process when they are same.
	 */
	if (!pid) {
		mmget(current->mm);
		*mem_nodes = cpuset_mems_allowed(current);
		return current->mm;
	}
1893 1894

	/* Find the mm_struct */
1895
	rcu_read_lock();
1896
	task = find_task_by_vpid(pid);
1897
	if (!task) {
1898
		rcu_read_unlock();
1899
		return ERR_PTR(-ESRCH);
1900
	}
1901
	get_task_struct(task);
1902 1903 1904

	/*
	 * Check if this process has the right to modify the specified
1905
	 * process. Use the regular "ptrace_may_access()" checks.
1906
	 */
1907
	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1908
		rcu_read_unlock();
1909
		mm = ERR_PTR(-EPERM);
1910
		goto out;
1911
	}
1912
	rcu_read_unlock();
1913

1914 1915
	mm = ERR_PTR(security_task_movememory(task));
	if (IS_ERR(mm))
1916
		goto out;
1917
	*mem_nodes = cpuset_mems_allowed(task);
1918
	mm = get_task_mm(task);
1919
out:
1920
	put_task_struct(task);
1921
	if (!mm)
1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
		mm = ERR_PTR(-EINVAL);
	return mm;
}

/*
 * Move a list of pages in the address space of the currently executing
 * process.
 */
static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
			     const void __user * __user *pages,
			     const int __user *nodes,
			     int __user *status, int flags)
{
	struct mm_struct *mm;
	int err;
	nodemask_t task_nodes;

	/* Check flags */
	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1941 1942
		return -EINVAL;

1943 1944 1945 1946 1947 1948 1949
	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
		return -EPERM;

	mm = find_mm_struct(pid, &task_nodes);
	if (IS_ERR(mm))
		return PTR_ERR(mm);

1950 1951 1952 1953 1954
	if (nodes)
		err = do_pages_move(mm, task_nodes, nr_pages, pages,
				    nodes, status, flags);
	else
		err = do_pages_stat(mm, nr_pages, pages, status);
1955 1956 1957 1958 1959

	mmput(mm);
	return err;
}

1960 1961 1962 1963 1964 1965 1966 1967
SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
		const void __user * __user *, pages,
		const int __user *, nodes,
		int __user *, status, int, flags)
{
	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
}

1968 1969 1970 1971 1972 1973
#ifdef CONFIG_NUMA_BALANCING
/*
 * Returns true if this is a safe migration target node for misplaced NUMA
 * pages. Currently it only checks the watermarks which crude
 */
static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1974
				   unsigned long nr_migrate_pages)
1975 1976
{
	int z;
1977

1978 1979 1980 1981 1982 1983 1984 1985 1986 1987
	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
		struct zone *zone = pgdat->node_zones + z;

		if (!populated_zone(zone))
			continue;

		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
		if (!zone_watermark_ok(zone, 0,
				       high_wmark_pages(zone) +
				       nr_migrate_pages,
1988
				       ZONE_MOVABLE, 0))
1989 1990 1991 1992 1993 1994 1995
			continue;
		return true;
	}
	return false;
}

static struct page *alloc_misplaced_dst_page(struct page *page,
1996
					   unsigned long data)
1997 1998 1999 2000
{
	int nid = (int) data;
	struct page *newpage;

2001
	newpage = __alloc_pages_node(nid,
2002 2003 2004
					 (GFP_HIGHUSER_MOVABLE |
					  __GFP_THISNODE | __GFP_NOMEMALLOC |
					  __GFP_NORETRY | __GFP_NOWARN) &
2005
					 ~__GFP_RECLAIM, 0);
2006

2007 2008 2009
	return newpage;
}

2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
static struct page *alloc_misplaced_dst_page_thp(struct page *page,
						 unsigned long data)
{
	int nid = (int) data;
	struct page *newpage;

	newpage = alloc_pages_node(nid, (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
				   HPAGE_PMD_ORDER);
	if (!newpage)
		goto out;

	prep_transhuge_page(newpage);

out:
	return newpage;
}

2027
static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
2028
{
2029
	int page_lru;
2030
	int nr_pages = thp_nr_pages(page);
2031

2032
	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
2033

2034 2035 2036 2037
	/* Do not migrate THP mapped by multiple processes */
	if (PageTransHuge(page) && total_mapcount(page) > 1)
		return 0;

2038
	/* Avoid migrating to a node that is nearly full */
2039
	if (!migrate_balanced_pgdat(pgdat, nr_pages))
2040
		return 0;
2041

2042 2043
	if (isolate_lru_page(page))
		return 0;
2044

2045
	page_lru = page_is_file_lru(page);
2046
	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
2047
			    nr_pages);
2048

2049
	/*
2050 2051 2052
	 * Isolating the page has taken another reference, so the
	 * caller's reference can be safely dropped without the page
	 * disappearing underneath us during migration.
2053 2054
	 */
	put_page(page);
2055
	return 1;
2056 2057 2058 2059 2060 2061 2062
}

/*
 * Attempt to migrate a misplaced page to the specified destination
 * node. Caller is expected to have an elevated reference count on
 * the page that will be dropped by this function before returning.
 */
2063 2064
int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
			   int node)
2065 2066
{
	pg_data_t *pgdat = NODE_DATA(node);
2067
	int isolated;
2068 2069
	int nr_remaining;
	LIST_HEAD(migratepages);
2070 2071
	new_page_t *new;
	bool compound;
2072
	int nr_pages = thp_nr_pages(page);
2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084

	/*
	 * PTE mapped THP or HugeTLB page can't reach here so the page could
	 * be either base page or THP.  And it must be head page if it is
	 * THP.
	 */
	compound = PageTransHuge(page);

	if (compound)
		new = alloc_misplaced_dst_page_thp;
	else
		new = alloc_misplaced_dst_page;
2085 2086

	/*
2087 2088
	 * Don't migrate file pages that are mapped in multiple processes
	 * with execute permissions as they are probably shared libraries.
2089
	 */
2090 2091
	if (page_mapcount(page) != 1 && page_is_file_lru(page) &&
	    (vma->vm_flags & VM_EXEC))
2092 2093
		goto out;

2094 2095 2096 2097
	/*
	 * Also do not migrate dirty pages as not all filesystems can move
	 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
	 */
2098
	if (page_is_file_lru(page) && PageDirty(page))
2099 2100
		goto out;

2101 2102 2103 2104 2105
	isolated = numamigrate_isolate_page(pgdat, page);
	if (!isolated)
		goto out;

	list_add(&page->lru, &migratepages);
2106
	nr_remaining = migrate_pages(&migratepages, *new, NULL, node,
2107
				     MIGRATE_ASYNC, MR_NUMA_MISPLACED, NULL);
2108
	if (nr_remaining) {
2109 2110
		if (!list_empty(&migratepages)) {
			list_del(&page->lru);
2111 2112
			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
					page_is_file_lru(page), -nr_pages);
2113 2114
			putback_lru_page(page);
		}
2115 2116
		isolated = 0;
	} else
2117
		count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_pages);
2118 2119
	BUG_ON(!list_empty(&migratepages));
	return isolated;
2120 2121 2122 2123

out:
	put_page(page);
	return 0;
2124
}
2125
#endif /* CONFIG_NUMA_BALANCING */
2126
#endif /* CONFIG_NUMA */
2127

2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
/*
 * node_demotion[] example:
 *
 * Consider a system with two sockets.  Each socket has
 * three classes of memory attached: fast, medium and slow.
 * Each memory class is placed in its own NUMA node.  The
 * CPUs are placed in the node with the "fast" memory.  The
 * 6 NUMA nodes (0-5) might be split among the sockets like
 * this:
 *
 *	Socket A: 0, 1, 2
 *	Socket B: 3, 4, 5
 *
 * When Node 0 fills up, its memory should be migrated to
 * Node 1.  When Node 1 fills up, it should be migrated to
 * Node 2.  The migration path start on the nodes with the
 * processors (since allocations default to this node) and
 * fast memory, progress through medium and end with the
 * slow memory:
 *
 *	0 -> 1 -> 2 -> stop
 *	3 -> 4 -> 5 -> stop
 *
 * This is represented in the node_demotion[] like this:
 *
 *	{  nr=1, nodes[0]=1 }, // Node 0 migrates to 1
 *	{  nr=1, nodes[0]=2 }, // Node 1 migrates to 2
 *	{  nr=0, nodes[0]=-1 }, // Node 2 does not migrate
 *	{  nr=1, nodes[0]=4 }, // Node 3 migrates to 4
 *	{  nr=1, nodes[0]=5 }, // Node 4 migrates to 5
 *	{  nr=0, nodes[0]=-1 }, // Node 5 does not migrate
 *
 * Moreover some systems may have multiple slow memory nodes.
 * Suppose a system has one socket with 3 memory nodes, node 0
 * is fast memory type, and node 1/2 both are slow memory
 * type, and the distance between fast memory node and slow
 * memory node is same. So the migration path should be:
 *
 *	0 -> 1/2 -> stop
 *
 * This is represented in the node_demotion[] like this:
 *	{ nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2
 *	{ nr=0, nodes[0]=-1, }, // Node 1 dose not migrate
 *	{ nr=0, nodes[0]=-1, }, // Node 2 does not migrate
 */

/*
 * Writes to this array occur without locking.  Cycles are
 * not allowed: Node X demotes to Y which demotes to X...
 *
 * If multiple reads are performed, a single rcu_read_lock()
 * must be held over all reads to ensure that no cycles are
 * observed.
 */
#define DEFAULT_DEMOTION_TARGET_NODES 15

#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES
#define DEMOTION_TARGET_NODES	(MAX_NUMNODES - 1)
#else
#define DEMOTION_TARGET_NODES	DEFAULT_DEMOTION_TARGET_NODES
#endif

struct demotion_nodes {
	unsigned short nr;
	short nodes[DEMOTION_TARGET_NODES];
};

static struct demotion_nodes *node_demotion __read_mostly;

/**
 * next_demotion_node() - Get the next node in the demotion path
 * @node: The starting node to lookup the next node
 *
 * Return: node id for next memory node in the demotion path hierarchy
 * from @node; NUMA_NO_NODE if @node is terminal.  This does not keep
 * @node online or guarantee that it *continues* to be the next demotion
 * target.
 */
int next_demotion_node(int node)
{
	struct demotion_nodes *nd;
	unsigned short target_nr, index;
	int target;

	if (!node_demotion)
		return NUMA_NO_NODE;

	nd = &node_demotion[node];

	/*
	 * node_demotion[] is updated without excluding this
	 * function from running.  RCU doesn't provide any
	 * compiler barriers, so the READ_ONCE() is required
	 * to avoid compiler reordering or read merging.
	 *
	 * Make sure to use RCU over entire code blocks if
	 * node_demotion[] reads need to be consistent.
	 */
	rcu_read_lock();
	target_nr = READ_ONCE(nd->nr);

	switch (target_nr) {
	case 0:
		target = NUMA_NO_NODE;
		goto out;
	case 1:
		index = 0;
		break;
	default:
		/*
		 * If there are multiple target nodes, just select one
		 * target node randomly.
		 *
		 * In addition, we can also use round-robin to select
		 * target node, but we should introduce another variable
		 * for node_demotion[] to record last selected target node,
		 * that may cause cache ping-pong due to the changing of
		 * last target node. Or introducing per-cpu data to avoid
		 * caching issue, which seems more complicated. So selecting
		 * target node randomly seems better until now.
		 */
		index = get_random_int() % target_nr;
		break;
	}

	target = READ_ONCE(nd->nodes[index]);

out:
	rcu_read_unlock();
	return target;
}

2260
#if defined(CONFIG_HOTPLUG_CPU)
2261 2262 2263
/* Disable reclaim-based migration. */
static void __disable_all_migrate_targets(void)
{
2264
	int node, i;
2265

2266 2267
	if (!node_demotion)
		return;
2268

2269 2270 2271 2272 2273
	for_each_online_node(node) {
		node_demotion[node].nr = 0;
		for (i = 0; i < DEMOTION_TARGET_NODES; i++)
			node_demotion[node].nodes[i] = NUMA_NO_NODE;
	}
2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
}

static void disable_all_migrate_targets(void)
{
	__disable_all_migrate_targets();

	/*
	 * Ensure that the "disable" is visible across the system.
	 * Readers will see either a combination of before+disable
	 * state or disable+after.  They will never see before and
	 * after state together.
	 *
	 * The before+after state together might have cycles and
	 * could cause readers to do things like loop until this
	 * function finishes.  This ensures they can only see a
	 * single "bad" read and would, for instance, only loop
	 * once.
	 */
	synchronize_rcu();
}

/*
 * Find an automatic demotion target for 'node'.
 * Failing here is OK.  It might just indicate
 * being at the end of a chain.
 */
2300 2301
static int establish_migrate_target(int node, nodemask_t *used,
				    int best_distance)
2302
{
2303 2304
	int migration_target, index, val;
	struct demotion_nodes *nd;
2305

2306
	if (!node_demotion)
2307 2308
		return NUMA_NO_NODE;

2309 2310
	nd = &node_demotion[node];

2311 2312 2313 2314
	migration_target = find_next_best_node(node, used);
	if (migration_target == NUMA_NO_NODE)
		return NUMA_NO_NODE;

2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
	/*
	 * If the node has been set a migration target node before,
	 * which means it's the best distance between them. Still
	 * check if this node can be demoted to other target nodes
	 * if they have a same best distance.
	 */
	if (best_distance != -1) {
		val = node_distance(node, migration_target);
		if (val > best_distance)
			return NUMA_NO_NODE;
	}

	index = nd->nr;
	if (WARN_ONCE(index >= DEMOTION_TARGET_NODES,
		      "Exceeds maximum demotion target nodes\n"))
		return NUMA_NO_NODE;

	nd->nodes[index] = migration_target;
	nd->nr++;
2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348

	return migration_target;
}

/*
 * When memory fills up on a node, memory contents can be
 * automatically migrated to another node instead of
 * discarded at reclaim.
 *
 * Establish a "migration path" which will start at nodes
 * with CPUs and will follow the priorities used to build the
 * page allocator zonelists.
 *
 * The difference here is that cycles must be avoided.  If
 * node0 migrates to node1, then neither node1, nor anything
2349 2350 2351
 * node1 migrates to can migrate to node0. Also one node can
 * be migrated to multiple nodes if the target nodes all have
 * a same best-distance against the source node.
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362
 *
 * This function can run simultaneously with readers of
 * node_demotion[].  However, it can not run simultaneously
 * with itself.  Exclusion is provided by memory hotplug events
 * being single-threaded.
 */
static void __set_migration_target_nodes(void)
{
	nodemask_t next_pass	= NODE_MASK_NONE;
	nodemask_t this_pass	= NODE_MASK_NONE;
	nodemask_t used_targets = NODE_MASK_NONE;
2363
	int node, best_distance;
2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392

	/*
	 * Avoid any oddities like cycles that could occur
	 * from changes in the topology.  This will leave
	 * a momentary gap when migration is disabled.
	 */
	disable_all_migrate_targets();

	/*
	 * Allocations go close to CPUs, first.  Assume that
	 * the migration path starts at the nodes with CPUs.
	 */
	next_pass = node_states[N_CPU];
again:
	this_pass = next_pass;
	next_pass = NODE_MASK_NONE;
	/*
	 * To avoid cycles in the migration "graph", ensure
	 * that migration sources are not future targets by
	 * setting them in 'used_targets'.  Do this only
	 * once per pass so that multiple source nodes can
	 * share a target node.
	 *
	 * 'used_targets' will become unavailable in future
	 * passes.  This limits some opportunities for
	 * multiple source nodes to share a destination.
	 */
	nodes_or(used_targets, used_targets, this_pass);

2393 2394
	for_each_node_mask(node, this_pass) {
		best_distance = -1;
2395 2396

		/*
2397 2398 2399
		 * Try to set up the migration path for the node, and the target
		 * migration nodes can be multiple, so doing a loop to find all
		 * the target nodes if they all have a best node distance.
2400
		 */
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
		do {
			int target_node =
				establish_migrate_target(node, &used_targets,
							 best_distance);

			if (target_node == NUMA_NO_NODE)
				break;

			if (best_distance == -1)
				best_distance = node_distance(node, target_node);

			/*
			 * Visit targets from this pass in the next pass.
			 * Eventually, every node will have been part of
			 * a pass, and will become set in 'used_targets'.
			 */
			node_set(target_node, next_pass);
		} while (1);
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437
	}
	/*
	 * 'next_pass' contains nodes which became migration
	 * targets in this pass.  Make additional passes until
	 * no more migrations targets are available.
	 */
	if (!nodes_empty(next_pass))
		goto again;
}

/*
 * For callers that do not hold get_online_mems() already.
 */
static void set_migration_target_nodes(void)
{
	get_online_mems();
	__set_migration_target_nodes();
	put_online_mems();
}
2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450

/*
 * This leaves migrate-on-reclaim transiently disabled between
 * the MEM_GOING_OFFLINE and MEM_OFFLINE events.  This runs
 * whether reclaim-based migration is enabled or not, which
 * ensures that the user can turn reclaim-based migration at
 * any time without needing to recalculate migration targets.
 *
 * These callbacks already hold get_online_mems().  That is why
 * __set_migration_target_nodes() can be used as opposed to
 * set_migration_target_nodes().
 */
static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
2451
						 unsigned long action, void *_arg)
2452
{
2453 2454 2455 2456 2457 2458 2459 2460 2461 2462
	struct memory_notify *arg = _arg;

	/*
	 * Only update the node migration order when a node is
	 * changing status, like online->offline.  This avoids
	 * the overhead of synchronize_rcu() in most cases.
	 */
	if (arg->status_change_nid < 0)
		return notifier_from_errno(0);

2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
	switch (action) {
	case MEM_GOING_OFFLINE:
		/*
		 * Make sure there are not transient states where
		 * an offline node is a migration target.  This
		 * will leave migration disabled until the offline
		 * completes and the MEM_OFFLINE case below runs.
		 */
		disable_all_migrate_targets();
		break;
	case MEM_OFFLINE:
	case MEM_ONLINE:
		/*
		 * Recalculate the target nodes once the node
		 * reaches its final state (online or offline).
		 */
		__set_migration_target_nodes();
		break;
	case MEM_CANCEL_OFFLINE:
		/*
		 * MEM_GOING_OFFLINE disabled all the migration
		 * targets.  Reenable them.
		 */
		__set_migration_target_nodes();
		break;
	case MEM_GOING_ONLINE:
	case MEM_CANCEL_ONLINE:
		break;
	}

	return notifier_from_errno(0);
}

2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
/*
 * React to hotplug events that might affect the migration targets
 * like events that online or offline NUMA nodes.
 *
 * The ordering is also currently dependent on which nodes have
 * CPUs.  That means we need CPU on/offline notification too.
 */
static int migration_online_cpu(unsigned int cpu)
{
	set_migration_target_nodes();
	return 0;
}

static int migration_offline_cpu(unsigned int cpu)
{
	set_migration_target_nodes();
	return 0;
}

2515 2516 2517 2518
static int __init migrate_on_reclaim_init(void)
{
	int ret;

2519 2520 2521 2522 2523
	node_demotion = kmalloc_array(nr_node_ids,
				      sizeof(struct demotion_nodes),
				      GFP_KERNEL);
	WARN_ON(!node_demotion);

2524 2525
	ret = cpuhp_setup_state_nocalls(CPUHP_MM_DEMOTION_DEAD, "mm/demotion:offline",
					NULL, migration_offline_cpu);
2526 2527 2528 2529 2530 2531 2532
	/*
	 * In the unlikely case that this fails, the automatic
	 * migration targets may become suboptimal for nodes
	 * where N_CPU changes.  With such a small impact in a
	 * rare case, do not bother trying to do anything special.
	 */
	WARN_ON(ret < 0);
2533 2534 2535
	ret = cpuhp_setup_state(CPUHP_AP_MM_DEMOTION_ONLINE, "mm/demotion:online",
				migration_online_cpu, NULL);
	WARN_ON(ret < 0);
2536 2537 2538 2539 2540

	hotplug_memory_notifier(migrate_on_reclaim_callback, 100);
	return 0;
}
late_initcall(migrate_on_reclaim_init);
2541
#endif /* CONFIG_HOTPLUG_CPU */
2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602

bool numa_demotion_enabled = false;

#ifdef CONFIG_SYSFS
static ssize_t numa_demotion_enabled_show(struct kobject *kobj,
					  struct kobj_attribute *attr, char *buf)
{
	return sysfs_emit(buf, "%s\n",
			  numa_demotion_enabled ? "true" : "false");
}

static ssize_t numa_demotion_enabled_store(struct kobject *kobj,
					   struct kobj_attribute *attr,
					   const char *buf, size_t count)
{
	if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1))
		numa_demotion_enabled = true;
	else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1))
		numa_demotion_enabled = false;
	else
		return -EINVAL;

	return count;
}

static struct kobj_attribute numa_demotion_enabled_attr =
	__ATTR(demotion_enabled, 0644, numa_demotion_enabled_show,
	       numa_demotion_enabled_store);

static struct attribute *numa_attrs[] = {
	&numa_demotion_enabled_attr.attr,
	NULL,
};

static const struct attribute_group numa_attr_group = {
	.attrs = numa_attrs,
};

static int __init numa_init_sysfs(void)
{
	int err;
	struct kobject *numa_kobj;

	numa_kobj = kobject_create_and_add("numa", mm_kobj);
	if (!numa_kobj) {
		pr_err("failed to create numa kobject\n");
		return -ENOMEM;
	}
	err = sysfs_create_group(numa_kobj, &numa_attr_group);
	if (err) {
		pr_err("failed to register numa group\n");
		goto delete_obj;
	}
	return 0;

delete_obj:
	kobject_put(numa_kobj);
	return err;
}
subsys_initcall(numa_init_sysfs);
#endif
反馈
建议
客服 返回
顶部