memory.c 127.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 *  linux/mm/memory.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 */

/*
 * demand-loading started 01.12.91 - seems it is high on the list of
 * things wanted, and it should be easy to implement. - Linus
 */

/*
 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
 * pages started 02.12.91, seems to work. - Linus.
 *
 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
 * would have taken more than the 6M I have free, but it worked well as
 * far as I could see.
 *
 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
 */

/*
 * Real VM (paging to/from disk) started 18.12.91. Much more work and
 * thought has to go into this. Oh, well..
 * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
 *		Found it. Everything seems to work now.
 * 20.12.91  -  Ok, making the swap-device changeable like the root.
 */

/*
 * 05.04.94  -  Multi-page memory management added for v1.1.
T
Tobin C Harding 已提交
33
 *              Idea by Alex Bligh (alex@cconcepts.co.uk)
L
Linus Torvalds 已提交
34 35 36 37 38 39 40 41 42
 *
 * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
 *		(Gerhard.Wichert@pdb.siemens.de)
 *
 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
 */

#include <linux/kernel_stat.h>
#include <linux/mm.h>
43
#include <linux/sched/mm.h>
44
#include <linux/sched/coredump.h>
45
#include <linux/sched/numa_balancing.h>
46
#include <linux/sched/task.h>
L
Linus Torvalds 已提交
47 48 49 50 51
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
52
#include <linux/memremap.h>
H
Hugh Dickins 已提交
53
#include <linux/ksm.h>
L
Linus Torvalds 已提交
54
#include <linux/rmap.h>
55
#include <linux/export.h>
56
#include <linux/delayacct.h>
L
Linus Torvalds 已提交
57
#include <linux/init.h>
58
#include <linux/pfn_t.h>
P
Peter Zijlstra 已提交
59
#include <linux/writeback.h>
60
#include <linux/memcontrol.h>
A
Andrea Arcangeli 已提交
61
#include <linux/mmu_notifier.h>
62 63
#include <linux/swapops.h>
#include <linux/elf.h>
64
#include <linux/gfp.h>
65
#include <linux/migrate.h>
A
Andy Shevchenko 已提交
66
#include <linux/string.h>
67
#include <linux/dma-debug.h>
68
#include <linux/debugfs.h>
69
#include <linux/userfaultfd_k.h>
70
#include <linux/dax.h>
71
#include <linux/oom.h>
L
Linus Torvalds 已提交
72

A
Alexey Dobriyan 已提交
73
#include <asm/io.h>
74
#include <asm/mmu_context.h>
L
Linus Torvalds 已提交
75
#include <asm/pgalloc.h>
76
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
77 78 79 80
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/pgtable.h>

81 82
#include "internal.h"

83
#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
84
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
85 86
#endif

A
Andy Whitcroft 已提交
87
#ifndef CONFIG_NEED_MULTIPLE_NODES
L
Linus Torvalds 已提交
88 89 90
/* use the per-pgdat data instead for discontigmem - mbligh */
unsigned long max_mapnr;
EXPORT_SYMBOL(max_mapnr);
T
Tobin C Harding 已提交
91 92

struct page *mem_map;
L
Linus Torvalds 已提交
93 94 95 96 97 98 99 100 101 102
EXPORT_SYMBOL(mem_map);
#endif

/*
 * A number of key systems in x86 including ioremap() rely on the assumption
 * that high_memory defines the upper bound on direct map memory, then end
 * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and
 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
 * and ZONE_HIGHMEM.
 */
T
Tobin C Harding 已提交
103
void *high_memory;
L
Linus Torvalds 已提交
104 105
EXPORT_SYMBOL(high_memory);

106 107 108 109 110 111 112 113 114 115 116 117
/*
 * Randomize the address space (stacks, mmaps, brk, etc.).
 *
 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
 *   as ancient (libc5 based) binaries can segfault. )
 */
int randomize_va_space __read_mostly =
#ifdef CONFIG_COMPAT_BRK
					1;
#else
					2;
#endif
118 119 120 121

static int __init disable_randmaps(char *s)
{
	randomize_va_space = 0;
122
	return 1;
123 124 125
}
__setup("norandmaps", disable_randmaps);

H
Hugh Dickins 已提交
126
unsigned long zero_pfn __read_mostly;
127 128
EXPORT_SYMBOL(zero_pfn);

T
Tobin C Harding 已提交
129 130
unsigned long highest_memmap_pfn __read_mostly;

H
Hugh Dickins 已提交
131 132 133 134 135 136 137 138 139
/*
 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
 */
static int __init init_zero_pfn(void)
{
	zero_pfn = page_to_pfn(ZERO_PAGE(0));
	return 0;
}
core_initcall(init_zero_pfn);
140

K
KAMEZAWA Hiroyuki 已提交
141

142 143
#if defined(SPLIT_RSS_COUNTING)

144
void sync_mm_rss(struct mm_struct *mm)
145 146 147 148
{
	int i;

	for (i = 0; i < NR_MM_COUNTERS; i++) {
149 150 151
		if (current->rss_stat.count[i]) {
			add_mm_counter(mm, i, current->rss_stat.count[i]);
			current->rss_stat.count[i] = 0;
152 153
		}
	}
154
	current->rss_stat.events = 0;
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
}

static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
{
	struct task_struct *task = current;

	if (likely(task->mm == mm))
		task->rss_stat.count[member] += val;
	else
		add_mm_counter(mm, member, val);
}
#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)

/* sync counter once per 64 page faults */
#define TASK_RSS_EVENTS_THRESH	(64)
static void check_sync_rss_stat(struct task_struct *task)
{
	if (unlikely(task != current))
		return;
	if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
176
		sync_mm_rss(task->mm);
177
}
178
#else /* SPLIT_RSS_COUNTING */
179 180 181 182 183 184 185 186

#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)

static void check_sync_rss_stat(struct task_struct *task)
{
}

187 188 189 190
#endif /* SPLIT_RSS_COUNTING */

#ifdef HAVE_GENERIC_MMU_GATHER

191
static bool tlb_next_batch(struct mmu_gather *tlb)
192 193 194 195 196 197
{
	struct mmu_gather_batch *batch;

	batch = tlb->active;
	if (batch->next) {
		tlb->active = batch->next;
198
		return true;
199 200
	}

201
	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
202
		return false;
203

204 205
	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
	if (!batch)
206
		return false;
207

208
	tlb->batch_count++;
209 210 211 212 213 214 215
	batch->next = NULL;
	batch->nr   = 0;
	batch->max  = MAX_GATHER_BATCH;

	tlb->active->next = batch;
	tlb->active = batch;

216
	return true;
217 218
}

M
Minchan Kim 已提交
219 220
void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
				unsigned long start, unsigned long end)
221 222 223
{
	tlb->mm = mm;

224 225
	/* Is it from 0 to ~0? */
	tlb->fullmm     = !(start | (end+1));
226
	tlb->need_flush_all = 0;
227 228 229 230
	tlb->local.next = NULL;
	tlb->local.nr   = 0;
	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
	tlb->active     = &tlb->local;
231
	tlb->batch_count = 0;
232 233 234 235

#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	tlb->batch = NULL;
#endif
236
	tlb->page_size = 0;
237 238

	__tlb_reset_range(tlb);
239 240
}

241
static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
242
{
243 244 245
	if (!tlb->end)
		return;

246
	tlb_flush(tlb);
247
	mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
248 249
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	tlb_table_flush(tlb);
250
#endif
251
	__tlb_reset_range(tlb);
252 253 254 255 256
}

static void tlb_flush_mmu_free(struct mmu_gather *tlb)
{
	struct mmu_gather_batch *batch;
257

258
	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
259 260 261 262 263 264
		free_pages_and_swap_cache(batch->pages, batch->nr);
		batch->nr = 0;
	}
	tlb->active = &tlb->local;
}

265 266 267 268 269 270
void tlb_flush_mmu(struct mmu_gather *tlb)
{
	tlb_flush_mmu_tlbonly(tlb);
	tlb_flush_mmu_free(tlb);
}

271 272 273 274
/* tlb_finish_mmu
 *	Called at the end of the shootdown operation to free up any resources
 *	that were required.
 */
M
Minchan Kim 已提交
275
void arch_tlb_finish_mmu(struct mmu_gather *tlb,
276
		unsigned long start, unsigned long end, bool force)
277 278 279
{
	struct mmu_gather_batch *batch, *next;

280 281 282
	if (force)
		__tlb_adjust_range(tlb, start, end - start);

283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
	tlb_flush_mmu(tlb);

	/* keep the page table cache within bounds */
	check_pgt_cache();

	for (batch = tlb->local.next; batch; batch = next) {
		next = batch->next;
		free_pages((unsigned long)batch, 0);
	}
	tlb->local.next = NULL;
}

/* __tlb_remove_page
 *	Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
 *	handling the additional races in SMP caused by other CPUs caching valid
 *	mappings in their TLBs. Returns the number of free page slots left.
 *	When out of page slots we must call tlb_flush_mmu().
300
 *returns true if the caller should flush.
301
 */
302
bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
303 304 305
{
	struct mmu_gather_batch *batch;

306
	VM_BUG_ON(!tlb->end);
307
	VM_WARN_ON(tlb->page_size != page_size);
308

309
	batch = tlb->active;
310 311 312 313 314
	/*
	 * Add the page and check if we are full. If so
	 * force a flush.
	 */
	batch->pages[batch->nr++] = page;
315 316
	if (batch->nr == batch->max) {
		if (!tlb_next_batch(tlb))
317
			return true;
318
		batch = tlb->active;
319
	}
320
	VM_BUG_ON_PAGE(batch->nr > batch->max, page);
321

322
	return false;
323 324 325 326
}

#endif /* HAVE_GENERIC_MMU_GATHER */

327 328 329 330
#ifdef CONFIG_HAVE_RCU_TABLE_FREE

static void tlb_remove_table_smp_sync(void *arg)
{
331 332 333 334 335 336 337 338 339
	struct mm_struct __maybe_unused *mm = arg;
	/*
	 * On most architectures this does nothing. Simply delivering the
	 * interrupt is enough to prevent races with software page table
	 * walking like that done in get_user_pages_fast.
	 *
	 * See the comment near struct mmu_table_batch.
	 */
	tlb_flush_remove_tables_local(mm);
340 341
}

342
static void tlb_remove_table_one(void *table, struct mmu_gather *tlb)
343 344 345 346 347 348 349 350
{
	/*
	 * This isn't an RCU grace period and hence the page-tables cannot be
	 * assumed to be actually RCU-freed.
	 *
	 * It is however sufficient for software page-table walkers that rely on
	 * IRQ disabling. See the comment near struct mmu_table_batch.
	 */
351
	smp_call_function(tlb_remove_table_smp_sync, tlb->mm, 1);
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
	__tlb_remove_table(table);
}

static void tlb_remove_table_rcu(struct rcu_head *head)
{
	struct mmu_table_batch *batch;
	int i;

	batch = container_of(head, struct mmu_table_batch, rcu);

	for (i = 0; i < batch->nr; i++)
		__tlb_remove_table(batch->tables[i]);

	free_page((unsigned long)batch);
}

void tlb_table_flush(struct mmu_gather *tlb)
{
	struct mmu_table_batch **batch = &tlb->batch;

372 373
	tlb_flush_remove_tables(tlb->mm);

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
	if (*batch) {
		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
		*batch = NULL;
	}
}

void tlb_remove_table(struct mmu_gather *tlb, void *table)
{
	struct mmu_table_batch **batch = &tlb->batch;

	/*
	 * When there's less then two users of this mm there cannot be a
	 * concurrent page-table walk.
	 */
	if (atomic_read(&tlb->mm->mm_users) < 2) {
		__tlb_remove_table(table);
		return;
	}

	if (*batch == NULL) {
		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
		if (*batch == NULL) {
396
			tlb_remove_table_one(table, tlb);
397 398 399 400 401 402 403 404 405
			return;
		}
		(*batch)->nr = 0;
	}
	(*batch)->tables[(*batch)->nr++] = table;
	if ((*batch)->nr == MAX_TABLE_BATCH)
		tlb_table_flush(tlb);
}

406
#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
407

408 409 410 411 412 413 414 415 416 417 418
/**
 * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
 * @tlb: the mmu_gather structure to initialize
 * @mm: the mm_struct of the target address space
 * @start: start of the region that will be removed from the page-table
 * @end: end of the region that will be removed from the page-table
 *
 * Called to initialize an (on-stack) mmu_gather structure for page-table
 * tear-down from @mm. The @start and @end are set to 0 and -1
 * respectively when @mm is without users and we're going to destroy
 * the full address space (exit/execve).
M
Minchan Kim 已提交
419 420 421 422 423
 */
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
			unsigned long start, unsigned long end)
{
	arch_tlb_gather_mmu(tlb, mm, start, end);
424
	inc_tlb_flush_pending(tlb->mm);
M
Minchan Kim 已提交
425 426 427 428 429
}

void tlb_finish_mmu(struct mmu_gather *tlb,
		unsigned long start, unsigned long end)
{
430 431 432 433 434 435 436 437 438 439 440
	/*
	 * If there are parallel threads are doing PTE changes on same range
	 * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
	 * flush by batching, a thread has stable TLB entry can fail to flush
	 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
	 * forcefully if we detect parallel PTE batching threads.
	 */
	bool force = mm_tlb_flush_nested(tlb->mm);

	arch_tlb_finish_mmu(tlb, start, end, force);
	dec_tlb_flush_pending(tlb->mm);
M
Minchan Kim 已提交
441 442
}

L
Linus Torvalds 已提交
443 444 445 446
/*
 * Note: this doesn't free the actual pages themselves. That
 * has been handled earlier when unmapping all the memory regions.
 */
447 448
static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
			   unsigned long addr)
L
Linus Torvalds 已提交
449
{
450
	pgtable_t token = pmd_pgtable(*pmd);
451
	pmd_clear(pmd);
452
	pte_free_tlb(tlb, token, addr);
453
	mm_dec_nr_ptes(tlb->mm);
L
Linus Torvalds 已提交
454 455
}

456 457 458
static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
				unsigned long addr, unsigned long end,
				unsigned long floor, unsigned long ceiling)
L
Linus Torvalds 已提交
459 460 461
{
	pmd_t *pmd;
	unsigned long next;
462
	unsigned long start;
L
Linus Torvalds 已提交
463

464
	start = addr;
L
Linus Torvalds 已提交
465 466 467 468 469
	pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
		if (pmd_none_or_clear_bad(pmd))
			continue;
470
		free_pte_range(tlb, pmd, addr);
L
Linus Torvalds 已提交
471 472
	} while (pmd++, addr = next, addr != end);

473 474 475 476 477 478 479
	start &= PUD_MASK;
	if (start < floor)
		return;
	if (ceiling) {
		ceiling &= PUD_MASK;
		if (!ceiling)
			return;
L
Linus Torvalds 已提交
480
	}
481 482 483 484 485
	if (end - 1 > ceiling - 1)
		return;

	pmd = pmd_offset(pud, start);
	pud_clear(pud);
486
	pmd_free_tlb(tlb, pmd, start);
487
	mm_dec_nr_pmds(tlb->mm);
L
Linus Torvalds 已提交
488 489
}

490
static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
491 492
				unsigned long addr, unsigned long end,
				unsigned long floor, unsigned long ceiling)
L
Linus Torvalds 已提交
493 494 495
{
	pud_t *pud;
	unsigned long next;
496
	unsigned long start;
L
Linus Torvalds 已提交
497

498
	start = addr;
499
	pud = pud_offset(p4d, addr);
L
Linus Torvalds 已提交
500 501 502 503
	do {
		next = pud_addr_end(addr, end);
		if (pud_none_or_clear_bad(pud))
			continue;
504
		free_pmd_range(tlb, pud, addr, next, floor, ceiling);
L
Linus Torvalds 已提交
505 506
	} while (pud++, addr = next, addr != end);

507 508 509 510 511 512 513 514 515 516 517 518 519 520
	start &= P4D_MASK;
	if (start < floor)
		return;
	if (ceiling) {
		ceiling &= P4D_MASK;
		if (!ceiling)
			return;
	}
	if (end - 1 > ceiling - 1)
		return;

	pud = pud_offset(p4d, start);
	p4d_clear(p4d);
	pud_free_tlb(tlb, pud, start);
K
Kirill A. Shutemov 已提交
521
	mm_dec_nr_puds(tlb->mm);
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
}

static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
				unsigned long addr, unsigned long end,
				unsigned long floor, unsigned long ceiling)
{
	p4d_t *p4d;
	unsigned long next;
	unsigned long start;

	start = addr;
	p4d = p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		if (p4d_none_or_clear_bad(p4d))
			continue;
		free_pud_range(tlb, p4d, addr, next, floor, ceiling);
	} while (p4d++, addr = next, addr != end);

541 542 543 544 545 546 547
	start &= PGDIR_MASK;
	if (start < floor)
		return;
	if (ceiling) {
		ceiling &= PGDIR_MASK;
		if (!ceiling)
			return;
L
Linus Torvalds 已提交
548
	}
549 550 551
	if (end - 1 > ceiling - 1)
		return;

552
	p4d = p4d_offset(pgd, start);
553
	pgd_clear(pgd);
554
	p4d_free_tlb(tlb, p4d, start);
L
Linus Torvalds 已提交
555 556 557
}

/*
558
 * This function frees user-level page tables of a process.
L
Linus Torvalds 已提交
559
 */
560
void free_pgd_range(struct mmu_gather *tlb,
561 562
			unsigned long addr, unsigned long end,
			unsigned long floor, unsigned long ceiling)
L
Linus Torvalds 已提交
563 564 565
{
	pgd_t *pgd;
	unsigned long next;
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591

	/*
	 * The next few lines have given us lots of grief...
	 *
	 * Why are we testing PMD* at this top level?  Because often
	 * there will be no work to do at all, and we'd prefer not to
	 * go all the way down to the bottom just to discover that.
	 *
	 * Why all these "- 1"s?  Because 0 represents both the bottom
	 * of the address space and the top of it (using -1 for the
	 * top wouldn't help much: the masks would do the wrong thing).
	 * The rule is that addr 0 and floor 0 refer to the bottom of
	 * the address space, but end 0 and ceiling 0 refer to the top
	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
	 * that end 0 case should be mythical).
	 *
	 * Wherever addr is brought up or ceiling brought down, we must
	 * be careful to reject "the opposite 0" before it confuses the
	 * subsequent tests.  But what about where end is brought down
	 * by PMD_SIZE below? no, end can't go down to 0 there.
	 *
	 * Whereas we round start (addr) and ceiling down, by different
	 * masks at different levels, in order to test whether a table
	 * now has no other vmas using it, so can be freed, we don't
	 * bother to round floor or end up - the tests don't need that.
	 */
L
Linus Torvalds 已提交
592

593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
	addr &= PMD_MASK;
	if (addr < floor) {
		addr += PMD_SIZE;
		if (!addr)
			return;
	}
	if (ceiling) {
		ceiling &= PMD_MASK;
		if (!ceiling)
			return;
	}
	if (end - 1 > ceiling - 1)
		end -= PMD_SIZE;
	if (addr > end - 1)
		return;
608 609 610 611 612
	/*
	 * We add page table cache pages with PAGE_SIZE,
	 * (see pte_free_tlb()), flush the tlb if we need
	 */
	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
613
	pgd = pgd_offset(tlb->mm, addr);
L
Linus Torvalds 已提交
614 615 616 617
	do {
		next = pgd_addr_end(addr, end);
		if (pgd_none_or_clear_bad(pgd))
			continue;
618
		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
L
Linus Torvalds 已提交
619
	} while (pgd++, addr = next, addr != end);
620 621
}

622
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
623
		unsigned long floor, unsigned long ceiling)
624 625 626 627 628
{
	while (vma) {
		struct vm_area_struct *next = vma->vm_next;
		unsigned long addr = vma->vm_start;

629
		/*
N
npiggin@suse.de 已提交
630 631
		 * Hide vma from rmap and truncate_pagecache before freeing
		 * pgtables
632
		 */
633
		unlink_anon_vmas(vma);
634 635
		unlink_file_vma(vma);

636
		if (is_vm_hugetlb_page(vma)) {
637
			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
T
Tobin C Harding 已提交
638
				floor, next ? next->vm_start : ceiling);
639 640 641 642 643
		} else {
			/*
			 * Optimization: gather nearby vmas into one call down
			 */
			while (next && next->vm_start <= vma->vm_end + PMD_SIZE
644
			       && !is_vm_hugetlb_page(next)) {
645 646
				vma = next;
				next = vma->vm_next;
647
				unlink_anon_vmas(vma);
648
				unlink_file_vma(vma);
649 650
			}
			free_pgd_range(tlb, addr, vma->vm_end,
T
Tobin C Harding 已提交
651
				floor, next ? next->vm_start : ceiling);
652
		}
653 654
		vma = next;
	}
L
Linus Torvalds 已提交
655 656
}

657
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
L
Linus Torvalds 已提交
658
{
659
	spinlock_t *ptl;
660
	pgtable_t new = pte_alloc_one(mm, address);
661 662 663
	if (!new)
		return -ENOMEM;

664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
	/*
	 * Ensure all pte setup (eg. pte page lock and page clearing) are
	 * visible before the pte is made visible to other CPUs by being
	 * put into page tables.
	 *
	 * The other side of the story is the pointer chasing in the page
	 * table walking code (when walking the page table without locking;
	 * ie. most of the time). Fortunately, these data accesses consist
	 * of a chain of data-dependent loads, meaning most CPUs (alpha
	 * being the notable exception) will already guarantee loads are
	 * seen in-order. See the alpha page table accessors for the
	 * smp_read_barrier_depends() barriers in page table walking code.
	 */
	smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */

679
	ptl = pmd_lock(mm, pmd);
680
	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
681
		mm_inc_nr_ptes(mm);
L
Linus Torvalds 已提交
682
		pmd_populate(mm, pmd, new);
683
		new = NULL;
684
	}
685
	spin_unlock(ptl);
686 687
	if (new)
		pte_free(mm, new);
688
	return 0;
L
Linus Torvalds 已提交
689 690
}

691
int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
L
Linus Torvalds 已提交
692
{
693 694 695 696
	pte_t *new = pte_alloc_one_kernel(&init_mm, address);
	if (!new)
		return -ENOMEM;

697 698
	smp_wmb(); /* See comment in __pte_alloc */

699
	spin_lock(&init_mm.page_table_lock);
700
	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
701
		pmd_populate_kernel(&init_mm, pmd, new);
702
		new = NULL;
703
	}
704
	spin_unlock(&init_mm.page_table_lock);
705 706
	if (new)
		pte_free_kernel(&init_mm, new);
707
	return 0;
L
Linus Torvalds 已提交
708 709
}

K
KAMEZAWA Hiroyuki 已提交
710 711 712 713 714 715
static inline void init_rss_vec(int *rss)
{
	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
}

static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
716
{
K
KAMEZAWA Hiroyuki 已提交
717 718
	int i;

719
	if (current->mm == mm)
720
		sync_mm_rss(mm);
K
KAMEZAWA Hiroyuki 已提交
721 722 723
	for (i = 0; i < NR_MM_COUNTERS; i++)
		if (rss[i])
			add_mm_counter(mm, i, rss[i]);
724 725
}

N
Nick Piggin 已提交
726
/*
727 728 729
 * This function is called to print an error when a bad pte
 * is found. For example, we might have a PFN-mapped pte in
 * a region that doesn't allow it.
N
Nick Piggin 已提交
730 731 732
 *
 * The calling function must still handle the error.
 */
733 734
static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
			  pte_t pte, struct page *page)
N
Nick Piggin 已提交
735
{
736
	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
737 738
	p4d_t *p4d = p4d_offset(pgd, addr);
	pud_t *pud = pud_offset(p4d, addr);
739 740 741
	pmd_t *pmd = pmd_offset(pud, addr);
	struct address_space *mapping;
	pgoff_t index;
742 743 744 745 746 747 748 749 750 751 752 753 754 755
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

	/*
	 * Allow a burst of 60 reports, then keep quiet for that minute;
	 * or allow a steady drip of one report per second.
	 */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			return;
		}
		if (nr_unshown) {
756 757
			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
				 nr_unshown);
758 759 760 761 762 763
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;
764 765 766 767

	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
	index = linear_page_index(vma, addr);

768 769 770
	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
		 current->comm,
		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
771
	if (page)
772
		dump_page(page, "bad pte");
773 774
	pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
775 776 777 778 779
	pr_alert("file:%pD fault:%pf mmap:%pf readpage:%pf\n",
		 vma->vm_file,
		 vma->vm_ops ? vma->vm_ops->fault : NULL,
		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
		 mapping ? mapping->a_ops->readpage : NULL);
N
Nick Piggin 已提交
780
	dump_stack();
781
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
N
Nick Piggin 已提交
782 783
}

H
Hugh Dickins 已提交
784
/*
N
Nick Piggin 已提交
785
 * vm_normal_page -- This function gets the "struct page" associated with a pte.
786
 *
N
Nick Piggin 已提交
787 788 789
 * "Special" mappings do not wish to be associated with a "struct page" (either
 * it doesn't exist, or it exists but they don't want to touch it). In this
 * case, NULL is returned here. "Normal" mappings do have a struct page.
J
Jared Hulbert 已提交
790
 *
N
Nick Piggin 已提交
791 792 793 794 795 796 797 798
 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
 * pte bit, in which case this function is trivial. Secondly, an architecture
 * may not have a spare pte bit, which requires a more complicated scheme,
 * described below.
 *
 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
 * special mapping (even if there are underlying and valid "struct pages").
 * COWed pages of a VM_PFNMAP are always normal.
799
 *
J
Jared Hulbert 已提交
800 801
 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
N
Nick Piggin 已提交
802 803
 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
 * mapping will always honor the rule
804 805 806
 *
 *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
 *
N
Nick Piggin 已提交
807 808 809 810 811 812
 * And for normal mappings this is false.
 *
 * This restricts such mappings to be a linear translation from virtual address
 * to pfn. To get around this restriction, we allow arbitrary mappings so long
 * as the vma is not a COW mapping; in that case, we know that all ptes are
 * special (because none can have been COWed).
J
Jared Hulbert 已提交
813 814
 *
 *
N
Nick Piggin 已提交
815
 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
J
Jared Hulbert 已提交
816 817 818 819 820 821 822 823 824
 *
 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
 * page" backing, however the difference is that _all_ pages with a struct
 * page (that is, those where pfn_valid is true) are refcounted and considered
 * normal pages by the VM. The disadvantage is that pages are refcounted
 * (which can be slower and simply not an option for some PFNMAP users). The
 * advantage is that we don't have to follow the strict linearity rule of
 * PFNMAP mappings in order to support COWable mappings.
 *
H
Hugh Dickins 已提交
825
 */
826 827
struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
			     pte_t pte, bool with_public_device)
H
Hugh Dickins 已提交
828
{
829
	unsigned long pfn = pte_pfn(pte);
N
Nick Piggin 已提交
830

L
Laurent Dufour 已提交
831
	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
832
		if (likely(!pte_special(pte)))
833
			goto check_pfn;
834 835
		if (vma->vm_ops && vma->vm_ops->find_special_page)
			return vma->vm_ops->find_special_page(vma, addr);
H
Hugh Dickins 已提交
836 837
		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
			return NULL;
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
		if (is_zero_pfn(pfn))
			return NULL;

		/*
		 * Device public pages are special pages (they are ZONE_DEVICE
		 * pages but different from persistent memory). They behave
		 * allmost like normal pages. The difference is that they are
		 * not on the lru and thus should never be involve with any-
		 * thing that involve lru manipulation (mlock, numa balancing,
		 * ...).
		 *
		 * This is why we still want to return NULL for such page from
		 * vm_normal_page() so that we do not have to special case all
		 * call site of vm_normal_page().
		 */
853
		if (likely(pfn <= highest_memmap_pfn)) {
854 855 856 857 858 859 860 861
			struct page *page = pfn_to_page(pfn);

			if (is_device_public_page(page)) {
				if (with_public_device)
					return page;
				return NULL;
			}
		}
862 863 864 865

		if (pte_devmap(pte))
			return NULL;

866
		print_bad_pte(vma, addr, pte, NULL);
N
Nick Piggin 已提交
867 868 869
		return NULL;
	}

L
Laurent Dufour 已提交
870
	/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
N
Nick Piggin 已提交
871

J
Jared Hulbert 已提交
872 873 874 875 876 877
	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
		if (vma->vm_flags & VM_MIXEDMAP) {
			if (!pfn_valid(pfn))
				return NULL;
			goto out;
		} else {
N
Nick Piggin 已提交
878 879
			unsigned long off;
			off = (addr - vma->vm_start) >> PAGE_SHIFT;
J
Jared Hulbert 已提交
880 881 882 883 884
			if (pfn == vma->vm_pgoff + off)
				return NULL;
			if (!is_cow_mapping(vma->vm_flags))
				return NULL;
		}
885 886
	}

887 888
	if (is_zero_pfn(pfn))
		return NULL;
L
Laurent Dufour 已提交
889

890 891 892 893 894
check_pfn:
	if (unlikely(pfn > highest_memmap_pfn)) {
		print_bad_pte(vma, addr, pte, NULL);
		return NULL;
	}
895 896

	/*
N
Nick Piggin 已提交
897 898
	 * NOTE! We still have PageReserved() pages in the page tables.
	 * eg. VDSO mappings can cause them to exist.
899
	 */
J
Jared Hulbert 已提交
900
out:
901
	return pfn_to_page(pfn);
H
Hugh Dickins 已提交
902 903
}

904 905 906 907 908 909 910 911 912
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
				pmd_t pmd)
{
	unsigned long pfn = pmd_pfn(pmd);

	/*
	 * There is no pmd_special() but there may be special pmds, e.g.
	 * in a direct-access (dax) mapping, so let's just replicate the
L
Laurent Dufour 已提交
913
	 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
	 */
	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
		if (vma->vm_flags & VM_MIXEDMAP) {
			if (!pfn_valid(pfn))
				return NULL;
			goto out;
		} else {
			unsigned long off;
			off = (addr - vma->vm_start) >> PAGE_SHIFT;
			if (pfn == vma->vm_pgoff + off)
				return NULL;
			if (!is_cow_mapping(vma->vm_flags))
				return NULL;
		}
	}

930 931
	if (pmd_devmap(pmd))
		return NULL;
932 933 934 935 936 937 938 939 940 941 942 943 944 945
	if (is_zero_pfn(pfn))
		return NULL;
	if (unlikely(pfn > highest_memmap_pfn))
		return NULL;

	/*
	 * NOTE! We still have PageReserved() pages in the page tables.
	 * eg. VDSO mappings can cause them to exist.
	 */
out:
	return pfn_to_page(pfn);
}
#endif

L
Linus Torvalds 已提交
946 947 948 949 950 951
/*
 * copy one vm_area from one task to the other. Assumes the page tables
 * already present in the new task to be cleared in the whole range
 * covered by this vma.
 */

H
Hugh Dickins 已提交
952
static inline unsigned long
L
Linus Torvalds 已提交
953
copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
N
Nick Piggin 已提交
954
		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
H
Hugh Dickins 已提交
955
		unsigned long addr, int *rss)
L
Linus Torvalds 已提交
956
{
N
Nick Piggin 已提交
957
	unsigned long vm_flags = vma->vm_flags;
L
Linus Torvalds 已提交
958 959 960 961 962
	pte_t pte = *src_pte;
	struct page *page;

	/* pte contains position in swap or file, so copy. */
	if (unlikely(!pte_present(pte))) {
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
		swp_entry_t entry = pte_to_swp_entry(pte);

		if (likely(!non_swap_entry(entry))) {
			if (swap_duplicate(entry) < 0)
				return entry.val;

			/* make sure dst_mm is on swapoff's mmlist. */
			if (unlikely(list_empty(&dst_mm->mmlist))) {
				spin_lock(&mmlist_lock);
				if (list_empty(&dst_mm->mmlist))
					list_add(&dst_mm->mmlist,
							&src_mm->mmlist);
				spin_unlock(&mmlist_lock);
			}
			rss[MM_SWAPENTS]++;
		} else if (is_migration_entry(entry)) {
			page = migration_entry_to_page(entry);

981
			rss[mm_counter(page)]++;
982 983 984 985 986 987 988 989 990 991 992 993

			if (is_write_migration_entry(entry) &&
					is_cow_mapping(vm_flags)) {
				/*
				 * COW mappings require pages in both
				 * parent and child to be set to read.
				 */
				make_migration_entry_read(&entry);
				pte = swp_entry_to_pte(entry);
				if (pte_swp_soft_dirty(*src_pte))
					pte = pte_swp_mksoft_dirty(pte);
				set_pte_at(src_mm, addr, src_pte, pte);
994
			}
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
		} else if (is_device_private_entry(entry)) {
			page = device_private_entry_to_page(entry);

			/*
			 * Update rss count even for unaddressable pages, as
			 * they should treated just like normal pages in this
			 * respect.
			 *
			 * We will likely want to have some new rss counters
			 * for unaddressable pages, at some point. But for now
			 * keep things as they are.
			 */
			get_page(page);
			rss[mm_counter(page)]++;
			page_dup_rmap(page, false);

			/*
			 * We do not preserve soft-dirty information, because so
			 * far, checkpoint/restore is the only feature that
			 * requires that. And checkpoint/restore does not work
			 * when a device driver is involved (you cannot easily
			 * save and restore device driver state).
			 */
			if (is_write_device_private_entry(entry) &&
			    is_cow_mapping(vm_flags)) {
				make_device_private_entry_read(&entry);
				pte = swp_entry_to_pte(entry);
				set_pte_at(src_mm, addr, src_pte, pte);
			}
L
Linus Torvalds 已提交
1024
		}
1025
		goto out_set_pte;
L
Linus Torvalds 已提交
1026 1027 1028 1029 1030 1031
	}

	/*
	 * If it's a COW mapping, write protect it both
	 * in the parent and the child
	 */
1032
	if (is_cow_mapping(vm_flags)) {
L
Linus Torvalds 已提交
1033
		ptep_set_wrprotect(src_mm, addr, src_pte);
1034
		pte = pte_wrprotect(pte);
L
Linus Torvalds 已提交
1035 1036 1037 1038 1039 1040 1041 1042 1043
	}

	/*
	 * If it's a shared mapping, mark it clean in
	 * the child
	 */
	if (vm_flags & VM_SHARED)
		pte = pte_mkclean(pte);
	pte = pte_mkold(pte);
1044 1045 1046 1047

	page = vm_normal_page(vma, addr, pte);
	if (page) {
		get_page(page);
1048
		page_dup_rmap(page, false);
1049
		rss[mm_counter(page)]++;
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
	} else if (pte_devmap(pte)) {
		page = pte_page(pte);

		/*
		 * Cache coherent device memory behave like regular page and
		 * not like persistent memory page. For more informations see
		 * MEMORY_DEVICE_CACHE_COHERENT in memory_hotplug.h
		 */
		if (is_device_public_page(page)) {
			get_page(page);
			page_dup_rmap(page, false);
			rss[mm_counter(page)]++;
		}
1063
	}
1064 1065 1066

out_set_pte:
	set_pte_at(dst_mm, addr, dst_pte, pte);
H
Hugh Dickins 已提交
1067
	return 0;
L
Linus Torvalds 已提交
1068 1069
}

1070
static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1071 1072
		   pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
		   unsigned long addr, unsigned long end)
L
Linus Torvalds 已提交
1073
{
1074
	pte_t *orig_src_pte, *orig_dst_pte;
L
Linus Torvalds 已提交
1075
	pte_t *src_pte, *dst_pte;
H
Hugh Dickins 已提交
1076
	spinlock_t *src_ptl, *dst_ptl;
1077
	int progress = 0;
K
KAMEZAWA Hiroyuki 已提交
1078
	int rss[NR_MM_COUNTERS];
H
Hugh Dickins 已提交
1079
	swp_entry_t entry = (swp_entry_t){0};
L
Linus Torvalds 已提交
1080 1081

again:
K
KAMEZAWA Hiroyuki 已提交
1082 1083
	init_rss_vec(rss);

H
Hugh Dickins 已提交
1084
	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
L
Linus Torvalds 已提交
1085 1086
	if (!dst_pte)
		return -ENOMEM;
P
Peter Zijlstra 已提交
1087
	src_pte = pte_offset_map(src_pmd, addr);
H
Hugh Dickins 已提交
1088
	src_ptl = pte_lockptr(src_mm, src_pmd);
I
Ingo Molnar 已提交
1089
	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1090 1091
	orig_src_pte = src_pte;
	orig_dst_pte = dst_pte;
1092
	arch_enter_lazy_mmu_mode();
L
Linus Torvalds 已提交
1093 1094 1095 1096 1097 1098

	do {
		/*
		 * We are holding two locks at this point - either of them
		 * could generate latencies in another task on another CPU.
		 */
1099 1100 1101
		if (progress >= 32) {
			progress = 0;
			if (need_resched() ||
N
Nick Piggin 已提交
1102
			    spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1103 1104
				break;
		}
L
Linus Torvalds 已提交
1105 1106 1107 1108
		if (pte_none(*src_pte)) {
			progress++;
			continue;
		}
H
Hugh Dickins 已提交
1109 1110 1111 1112
		entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
							vma, addr, rss);
		if (entry.val)
			break;
L
Linus Torvalds 已提交
1113 1114 1115
		progress += 8;
	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);

1116
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
1117
	spin_unlock(src_ptl);
P
Peter Zijlstra 已提交
1118
	pte_unmap(orig_src_pte);
K
KAMEZAWA Hiroyuki 已提交
1119
	add_mm_rss_vec(dst_mm, rss);
1120
	pte_unmap_unlock(orig_dst_pte, dst_ptl);
H
Hugh Dickins 已提交
1121
	cond_resched();
H
Hugh Dickins 已提交
1122 1123 1124 1125 1126 1127

	if (entry.val) {
		if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
			return -ENOMEM;
		progress = 0;
	}
L
Linus Torvalds 已提交
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
	if (addr != end)
		goto again;
	return 0;
}

static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
		unsigned long addr, unsigned long end)
{
	pmd_t *src_pmd, *dst_pmd;
	unsigned long next;

	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
	if (!dst_pmd)
		return -ENOMEM;
	src_pmd = pmd_offset(src_pud, addr);
	do {
		next = pmd_addr_end(addr, end);
1146 1147
		if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
			|| pmd_devmap(*src_pmd)) {
1148
			int err;
1149
			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma);
1150 1151 1152 1153 1154 1155 1156 1157
			err = copy_huge_pmd(dst_mm, src_mm,
					    dst_pmd, src_pmd, addr, vma);
			if (err == -ENOMEM)
				return -ENOMEM;
			if (!err)
				continue;
			/* fall through */
		}
L
Linus Torvalds 已提交
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
		if (pmd_none_or_clear_bad(src_pmd))
			continue;
		if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
						vma, addr, next))
			return -ENOMEM;
	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
	return 0;
}

static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1168
		p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
L
Linus Torvalds 已提交
1169 1170 1171 1172 1173
		unsigned long addr, unsigned long end)
{
	pud_t *src_pud, *dst_pud;
	unsigned long next;

1174
	dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
L
Linus Torvalds 已提交
1175 1176
	if (!dst_pud)
		return -ENOMEM;
1177
	src_pud = pud_offset(src_p4d, addr);
L
Linus Torvalds 已提交
1178 1179
	do {
		next = pud_addr_end(addr, end);
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
		if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
			int err;

			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma);
			err = copy_huge_pud(dst_mm, src_mm,
					    dst_pud, src_pud, addr, vma);
			if (err == -ENOMEM)
				return -ENOMEM;
			if (!err)
				continue;
			/* fall through */
		}
L
Linus Torvalds 已提交
1192 1193 1194 1195 1196 1197 1198 1199 1200
		if (pud_none_or_clear_bad(src_pud))
			continue;
		if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
						vma, addr, next))
			return -ENOMEM;
	} while (dst_pud++, src_pud++, addr = next, addr != end);
	return 0;
}

1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
		unsigned long addr, unsigned long end)
{
	p4d_t *src_p4d, *dst_p4d;
	unsigned long next;

	dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
	if (!dst_p4d)
		return -ENOMEM;
	src_p4d = p4d_offset(src_pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		if (p4d_none_or_clear_bad(src_p4d))
			continue;
		if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
						vma, addr, next))
			return -ENOMEM;
	} while (dst_p4d++, src_p4d++, addr = next, addr != end);
	return 0;
}

L
Linus Torvalds 已提交
1223 1224 1225 1226 1227 1228 1229
int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		struct vm_area_struct *vma)
{
	pgd_t *src_pgd, *dst_pgd;
	unsigned long next;
	unsigned long addr = vma->vm_start;
	unsigned long end = vma->vm_end;
1230 1231 1232
	unsigned long mmun_start;	/* For mmu_notifiers */
	unsigned long mmun_end;		/* For mmu_notifiers */
	bool is_cow;
A
Andrea Arcangeli 已提交
1233
	int ret;
L
Linus Torvalds 已提交
1234

1235 1236 1237 1238 1239 1240
	/*
	 * Don't copy ptes where a page fault will fill them correctly.
	 * Fork becomes much lighter when there are big shared or private
	 * readonly mappings. The tradeoff is that copy_page_range is more
	 * efficient than faulting.
	 */
1241 1242 1243
	if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
			!vma->anon_vma)
		return 0;
1244

L
Linus Torvalds 已提交
1245 1246 1247
	if (is_vm_hugetlb_page(vma))
		return copy_hugetlb_page_range(dst_mm, src_mm, vma);

1248
	if (unlikely(vma->vm_flags & VM_PFNMAP)) {
1249 1250 1251 1252
		/*
		 * We do not free on error cases below as remove_vma
		 * gets called on error from higher level routine
		 */
1253
		ret = track_pfn_copy(vma);
1254 1255 1256 1257
		if (ret)
			return ret;
	}

A
Andrea Arcangeli 已提交
1258 1259 1260 1261 1262 1263
	/*
	 * We need to invalidate the secondary MMU mappings only when
	 * there could be a permission downgrade on the ptes of the
	 * parent mm. And a permission downgrade will only happen if
	 * is_cow_mapping() returns true.
	 */
1264 1265 1266 1267 1268 1269
	is_cow = is_cow_mapping(vma->vm_flags);
	mmun_start = addr;
	mmun_end   = end;
	if (is_cow)
		mmu_notifier_invalidate_range_start(src_mm, mmun_start,
						    mmun_end);
A
Andrea Arcangeli 已提交
1270 1271

	ret = 0;
L
Linus Torvalds 已提交
1272 1273 1274 1275 1276 1277
	dst_pgd = pgd_offset(dst_mm, addr);
	src_pgd = pgd_offset(src_mm, addr);
	do {
		next = pgd_addr_end(addr, end);
		if (pgd_none_or_clear_bad(src_pgd))
			continue;
1278
		if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
A
Andrea Arcangeli 已提交
1279 1280 1281 1282
					    vma, addr, next))) {
			ret = -ENOMEM;
			break;
		}
L
Linus Torvalds 已提交
1283
	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
A
Andrea Arcangeli 已提交
1284

1285 1286
	if (is_cow)
		mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
A
Andrea Arcangeli 已提交
1287
	return ret;
L
Linus Torvalds 已提交
1288 1289
}

1290
static unsigned long zap_pte_range(struct mmu_gather *tlb,
N
Nick Piggin 已提交
1291
				struct vm_area_struct *vma, pmd_t *pmd,
L
Linus Torvalds 已提交
1292
				unsigned long addr, unsigned long end,
1293
				struct zap_details *details)
L
Linus Torvalds 已提交
1294
{
N
Nick Piggin 已提交
1295
	struct mm_struct *mm = tlb->mm;
P
Peter Zijlstra 已提交
1296
	int force_flush = 0;
K
KAMEZAWA Hiroyuki 已提交
1297
	int rss[NR_MM_COUNTERS];
1298
	spinlock_t *ptl;
1299
	pte_t *start_pte;
1300
	pte_t *pte;
1301
	swp_entry_t entry;
K
KAMEZAWA Hiroyuki 已提交
1302

1303
	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
P
Peter Zijlstra 已提交
1304
again:
1305
	init_rss_vec(rss);
1306 1307
	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
	pte = start_pte;
1308
	flush_tlb_batched_pending(mm);
1309
	arch_enter_lazy_mmu_mode();
L
Linus Torvalds 已提交
1310 1311
	do {
		pte_t ptent = *pte;
T
Tobin C Harding 已提交
1312
		if (pte_none(ptent))
L
Linus Torvalds 已提交
1313
			continue;
1314

L
Linus Torvalds 已提交
1315
		if (pte_present(ptent)) {
H
Hugh Dickins 已提交
1316
			struct page *page;
1317

1318
			page = _vm_normal_page(vma, addr, ptent, true);
L
Linus Torvalds 已提交
1319 1320 1321 1322 1323 1324 1325
			if (unlikely(details) && page) {
				/*
				 * unmap_shared_mapping_pages() wants to
				 * invalidate cache without truncating:
				 * unmap shared but keep private pages.
				 */
				if (details->check_mapping &&
1326
				    details->check_mapping != page_rmapping(page))
L
Linus Torvalds 已提交
1327 1328
					continue;
			}
N
Nick Piggin 已提交
1329
			ptent = ptep_get_and_clear_full(mm, addr, pte,
1330
							tlb->fullmm);
L
Linus Torvalds 已提交
1331 1332 1333
			tlb_remove_tlb_entry(tlb, pte, addr);
			if (unlikely(!page))
				continue;
1334 1335

			if (!PageAnon(page)) {
1336 1337
				if (pte_dirty(ptent)) {
					force_flush = 1;
1338
					set_page_dirty(page);
1339
				}
1340
				if (pte_young(ptent) &&
1341
				    likely(!(vma->vm_flags & VM_SEQ_READ)))
1342
					mark_page_accessed(page);
1343
			}
1344
			rss[mm_counter(page)]--;
1345
			page_remove_rmap(page, false);
1346 1347
			if (unlikely(page_mapcount(page) < 0))
				print_bad_pte(vma, addr, ptent, page);
1348
			if (unlikely(__tlb_remove_page(tlb, page))) {
1349
				force_flush = 1;
1350
				addr += PAGE_SIZE;
P
Peter Zijlstra 已提交
1351
				break;
1352
			}
L
Linus Torvalds 已提交
1353 1354
			continue;
		}
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377

		entry = pte_to_swp_entry(ptent);
		if (non_swap_entry(entry) && is_device_private_entry(entry)) {
			struct page *page = device_private_entry_to_page(entry);

			if (unlikely(details && details->check_mapping)) {
				/*
				 * unmap_shared_mapping_pages() wants to
				 * invalidate cache without truncating:
				 * unmap shared but keep private pages.
				 */
				if (details->check_mapping !=
				    page_rmapping(page))
					continue;
			}

			pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
			rss[mm_counter(page)]--;
			page_remove_rmap(page, false);
			put_page(page);
			continue;
		}

1378 1379
		/* If details->check_mapping, we leave swap entries. */
		if (unlikely(details))
L
Linus Torvalds 已提交
1380
			continue;
K
KAMEZAWA Hiroyuki 已提交
1381

1382 1383 1384 1385 1386
		entry = pte_to_swp_entry(ptent);
		if (!non_swap_entry(entry))
			rss[MM_SWAPENTS]--;
		else if (is_migration_entry(entry)) {
			struct page *page;
1387

1388
			page = migration_entry_to_page(entry);
1389
			rss[mm_counter(page)]--;
K
KAMEZAWA Hiroyuki 已提交
1390
		}
1391 1392
		if (unlikely(!free_swap_and_cache(entry)))
			print_bad_pte(vma, addr, ptent, NULL);
1393
		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1394
	} while (pte++, addr += PAGE_SIZE, addr != end);
1395

K
KAMEZAWA Hiroyuki 已提交
1396
	add_mm_rss_vec(mm, rss);
1397
	arch_leave_lazy_mmu_mode();
1398

1399
	/* Do the actual TLB flush before dropping ptl */
1400
	if (force_flush)
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
		tlb_flush_mmu_tlbonly(tlb);
	pte_unmap_unlock(start_pte, ptl);

	/*
	 * If we forced a TLB flush (either due to running out of
	 * batch buffers or because we needed to flush dirty TLB
	 * entries before releasing the ptl), free the batched
	 * memory too. Restart if we didn't do everything.
	 */
	if (force_flush) {
		force_flush = 0;
		tlb_flush_mmu_free(tlb);
1413
		if (addr != end)
P
Peter Zijlstra 已提交
1414 1415 1416
			goto again;
	}

1417
	return addr;
L
Linus Torvalds 已提交
1418 1419
}

1420
static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
N
Nick Piggin 已提交
1421
				struct vm_area_struct *vma, pud_t *pud,
L
Linus Torvalds 已提交
1422
				unsigned long addr, unsigned long end,
1423
				struct zap_details *details)
L
Linus Torvalds 已提交
1424 1425 1426 1427 1428 1429 1430
{
	pmd_t *pmd;
	unsigned long next;

	pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
1431
		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1432
			if (next - addr != HPAGE_PMD_SIZE)
1433
				__split_huge_pmd(vma, pmd, addr, false, NULL);
1434
			else if (zap_huge_pmd(tlb, vma, pmd, addr))
1435
				goto next;
1436 1437
			/* fall through */
		}
1438 1439 1440 1441 1442 1443 1444 1445 1446
		/*
		 * Here there can be other concurrent MADV_DONTNEED or
		 * trans huge page faults running, and if the pmd is
		 * none or trans huge it can change under us. This is
		 * because MADV_DONTNEED holds the mmap_sem in read
		 * mode.
		 */
		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
			goto next;
1447
		next = zap_pte_range(tlb, vma, pmd, addr, next, details);
1448
next:
1449 1450
		cond_resched();
	} while (pmd++, addr = next, addr != end);
1451 1452

	return addr;
L
Linus Torvalds 已提交
1453 1454
}

1455
static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1456
				struct vm_area_struct *vma, p4d_t *p4d,
L
Linus Torvalds 已提交
1457
				unsigned long addr, unsigned long end,
1458
				struct zap_details *details)
L
Linus Torvalds 已提交
1459 1460 1461 1462
{
	pud_t *pud;
	unsigned long next;

1463
	pud = pud_offset(p4d, addr);
L
Linus Torvalds 已提交
1464 1465
	do {
		next = pud_addr_end(addr, end);
1466 1467 1468 1469 1470 1471 1472 1473
		if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
			if (next - addr != HPAGE_PUD_SIZE) {
				VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);
				split_huge_pud(vma, pud, addr);
			} else if (zap_huge_pud(tlb, vma, pud, addr))
				goto next;
			/* fall through */
		}
1474
		if (pud_none_or_clear_bad(pud))
L
Linus Torvalds 已提交
1475
			continue;
1476
		next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1477 1478
next:
		cond_resched();
1479
	} while (pud++, addr = next, addr != end);
1480 1481

	return addr;
L
Linus Torvalds 已提交
1482 1483
}

1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
				struct vm_area_struct *vma, pgd_t *pgd,
				unsigned long addr, unsigned long end,
				struct zap_details *details)
{
	p4d_t *p4d;
	unsigned long next;

	p4d = p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		if (p4d_none_or_clear_bad(p4d))
			continue;
		next = zap_pud_range(tlb, vma, p4d, addr, next, details);
	} while (p4d++, addr = next, addr != end);

	return addr;
}

M
Michal Hocko 已提交
1503
void unmap_page_range(struct mmu_gather *tlb,
A
Al Viro 已提交
1504 1505 1506
			     struct vm_area_struct *vma,
			     unsigned long addr, unsigned long end,
			     struct zap_details *details)
L
Linus Torvalds 已提交
1507 1508 1509 1510 1511 1512 1513 1514 1515
{
	pgd_t *pgd;
	unsigned long next;

	BUG_ON(addr >= end);
	tlb_start_vma(tlb, vma);
	pgd = pgd_offset(vma->vm_mm, addr);
	do {
		next = pgd_addr_end(addr, end);
1516
		if (pgd_none_or_clear_bad(pgd))
L
Linus Torvalds 已提交
1517
			continue;
1518
		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1519
	} while (pgd++, addr = next, addr != end);
L
Linus Torvalds 已提交
1520 1521
	tlb_end_vma(tlb, vma);
}
1522

1523 1524 1525

static void unmap_single_vma(struct mmu_gather *tlb,
		struct vm_area_struct *vma, unsigned long start_addr,
1526
		unsigned long end_addr,
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
		struct zap_details *details)
{
	unsigned long start = max(vma->vm_start, start_addr);
	unsigned long end;

	if (start >= vma->vm_end)
		return;
	end = min(vma->vm_end, end_addr);
	if (end <= vma->vm_start)
		return;

1538 1539 1540
	if (vma->vm_file)
		uprobe_munmap(vma, start, end);

1541
	if (unlikely(vma->vm_flags & VM_PFNMAP))
1542
		untrack_pfn(vma, 0, 0);
1543 1544 1545 1546 1547 1548 1549

	if (start != end) {
		if (unlikely(is_vm_hugetlb_page(vma))) {
			/*
			 * It is undesirable to test vma->vm_file as it
			 * should be non-null for valid hugetlb area.
			 * However, vm_file will be NULL in the error
1550
			 * cleanup path of mmap_region. When
1551
			 * hugetlbfs ->mmap method fails,
1552
			 * mmap_region() nullifies vma->vm_file
1553 1554 1555 1556
			 * before calling this function to clean up.
			 * Since no pte has actually been setup, it is
			 * safe to do nothing in this case.
			 */
1557
			if (vma->vm_file) {
1558
				i_mmap_lock_write(vma->vm_file->f_mapping);
1559
				__unmap_hugepage_range_final(tlb, vma, start, end, NULL);
1560
				i_mmap_unlock_write(vma->vm_file->f_mapping);
1561
			}
1562 1563 1564
		} else
			unmap_page_range(tlb, vma, start, end, details);
	}
L
Linus Torvalds 已提交
1565 1566 1567 1568
}

/**
 * unmap_vmas - unmap a range of memory covered by a list of vma's
1569
 * @tlb: address of the caller's struct mmu_gather
L
Linus Torvalds 已提交
1570 1571 1572 1573
 * @vma: the starting vma
 * @start_addr: virtual address at which to start unmapping
 * @end_addr: virtual address at which to end unmapping
 *
1574
 * Unmap all pages in the vma list.
L
Linus Torvalds 已提交
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
 *
 * Only addresses between `start' and `end' will be unmapped.
 *
 * The VMA list must be sorted in ascending virtual address order.
 *
 * unmap_vmas() assumes that the caller will flush the whole unmapped address
 * range after unmap_vmas() returns.  So the only responsibility here is to
 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
 * drops the lock and schedules.
 */
A
Al Viro 已提交
1585
void unmap_vmas(struct mmu_gather *tlb,
L
Linus Torvalds 已提交
1586
		struct vm_area_struct *vma, unsigned long start_addr,
1587
		unsigned long end_addr)
L
Linus Torvalds 已提交
1588
{
A
Andrea Arcangeli 已提交
1589
	struct mm_struct *mm = vma->vm_mm;
L
Linus Torvalds 已提交
1590

A
Andrea Arcangeli 已提交
1591
	mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
1592
	for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
1593
		unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
A
Andrea Arcangeli 已提交
1594
	mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
L
Linus Torvalds 已提交
1595 1596 1597 1598 1599
}

/**
 * zap_page_range - remove user pages in a given range
 * @vma: vm_area_struct holding the applicable pages
1600
 * @start: starting address of pages to zap
L
Linus Torvalds 已提交
1601
 * @size: number of bytes to zap
1602 1603
 *
 * Caller must protect the VMA list
L
Linus Torvalds 已提交
1604
 */
1605
void zap_page_range(struct vm_area_struct *vma, unsigned long start,
1606
		unsigned long size)
L
Linus Torvalds 已提交
1607 1608
{
	struct mm_struct *mm = vma->vm_mm;
P
Peter Zijlstra 已提交
1609
	struct mmu_gather tlb;
1610
	unsigned long end = start + size;
L
Linus Torvalds 已提交
1611 1612

	lru_add_drain();
1613
	tlb_gather_mmu(&tlb, mm, start, end);
1614
	update_hiwater_rss(mm);
1615
	mmu_notifier_invalidate_range_start(mm, start, end);
1616
	for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
1617
		unmap_single_vma(&tlb, vma, start, end, NULL);
1618 1619
	mmu_notifier_invalidate_range_end(mm, start, end);
	tlb_finish_mmu(&tlb, start, end);
L
Linus Torvalds 已提交
1620 1621
}

1622 1623 1624 1625 1626
/**
 * zap_page_range_single - remove user pages in a given range
 * @vma: vm_area_struct holding the applicable pages
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
1627
 * @details: details of shared cache invalidation
1628 1629
 *
 * The range must fit into one VMA.
L
Linus Torvalds 已提交
1630
 */
1631
static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
L
Linus Torvalds 已提交
1632 1633 1634
		unsigned long size, struct zap_details *details)
{
	struct mm_struct *mm = vma->vm_mm;
P
Peter Zijlstra 已提交
1635
	struct mmu_gather tlb;
L
Linus Torvalds 已提交
1636 1637 1638
	unsigned long end = address + size;

	lru_add_drain();
1639
	tlb_gather_mmu(&tlb, mm, address, end);
1640
	update_hiwater_rss(mm);
1641
	mmu_notifier_invalidate_range_start(mm, address, end);
1642
	unmap_single_vma(&tlb, vma, address, end, details);
1643
	mmu_notifier_invalidate_range_end(mm, address, end);
P
Peter Zijlstra 已提交
1644
	tlb_finish_mmu(&tlb, address, end);
L
Linus Torvalds 已提交
1645 1646
}

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
/**
 * zap_vma_ptes - remove ptes mapping the vma
 * @vma: vm_area_struct holding ptes to be zapped
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
 *
 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
 *
 * The entire address range must be fully contained within the vma.
 *
 */
1658
void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1659 1660 1661 1662
		unsigned long size)
{
	if (address < vma->vm_start || address + size > vma->vm_end ||
	    		!(vma->vm_flags & VM_PFNMAP))
1663 1664
		return;

1665
	zap_page_range_single(vma, address, size, NULL);
1666 1667 1668
}
EXPORT_SYMBOL_GPL(zap_vma_ptes);

1669
pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
H
Harvey Harrison 已提交
1670
			spinlock_t **ptl)
1671
{
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	p4d = p4d_alloc(mm, pgd, addr);
	if (!p4d)
		return NULL;
	pud = pud_alloc(mm, p4d, addr);
	if (!pud)
		return NULL;
	pmd = pmd_alloc(mm, pud, addr);
	if (!pmd)
		return NULL;

	VM_BUG_ON(pmd_trans_huge(*pmd));
	return pte_alloc_map_lock(mm, pmd, addr, ptl);
1690 1691
}

1692 1693 1694 1695 1696 1697 1698
/*
 * This is the old fallback for page remapping.
 *
 * For historical reasons, it only allows reserved pages. Only
 * old drivers should use this, and they needed to mark their
 * pages reserved for the old functions anyway.
 */
N
Nick Piggin 已提交
1699 1700
static int insert_page(struct vm_area_struct *vma, unsigned long addr,
			struct page *page, pgprot_t prot)
1701
{
N
Nick Piggin 已提交
1702
	struct mm_struct *mm = vma->vm_mm;
1703
	int retval;
1704
	pte_t *pte;
1705 1706
	spinlock_t *ptl;

1707
	retval = -EINVAL;
1708
	if (PageAnon(page))
1709
		goto out;
1710 1711
	retval = -ENOMEM;
	flush_dcache_page(page);
1712
	pte = get_locked_pte(mm, addr, &ptl);
1713
	if (!pte)
1714
		goto out;
1715 1716 1717 1718 1719 1720
	retval = -EBUSY;
	if (!pte_none(*pte))
		goto out_unlock;

	/* Ok, finally just insert the thing.. */
	get_page(page);
1721
	inc_mm_counter_fast(mm, mm_counter_file(page));
K
Kirill A. Shutemov 已提交
1722
	page_add_file_rmap(page, false);
1723 1724 1725
	set_pte_at(mm, addr, pte, mk_pte(page, prot));

	retval = 0;
1726 1727
	pte_unmap_unlock(pte, ptl);
	return retval;
1728 1729 1730 1731 1732 1733
out_unlock:
	pte_unmap_unlock(pte, ptl);
out:
	return retval;
}

1734 1735 1736 1737 1738 1739
/**
 * vm_insert_page - insert single page into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @page: source kernel page
 *
1740 1741 1742 1743 1744 1745
 * This allows drivers to insert individual pages they've allocated
 * into a user vma.
 *
 * The page has to be a nice clean _individual_ kernel allocation.
 * If you allocate a compound page, you need to have marked it as
 * such (__GFP_COMP), or manually just split the page up yourself
N
Nick Piggin 已提交
1746
 * (see split_page()).
1747 1748 1749 1750 1751 1752 1753 1754
 *
 * NOTE! Traditionally this was done with "remap_pfn_range()" which
 * took an arbitrary page protection parameter. This doesn't allow
 * that. Your vma protection will have to be set up correctly, which
 * means that if you want a shared writable mapping, you'd better
 * ask for a shared writable mapping!
 *
 * The page does not need to be reserved.
1755 1756 1757 1758 1759
 *
 * Usually this function is called from f_op->mmap() handler
 * under mm->mmap_sem write-lock, so it can change vma->vm_flags.
 * Caller must set VM_MIXEDMAP on vma if it wants to call this
 * function from other places, for example from page-fault handler.
1760
 */
N
Nick Piggin 已提交
1761 1762
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
			struct page *page)
1763 1764 1765 1766 1767
{
	if (addr < vma->vm_start || addr >= vma->vm_end)
		return -EFAULT;
	if (!page_count(page))
		return -EINVAL;
1768 1769 1770 1771 1772
	if (!(vma->vm_flags & VM_MIXEDMAP)) {
		BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem));
		BUG_ON(vma->vm_flags & VM_PFNMAP);
		vma->vm_flags |= VM_MIXEDMAP;
	}
N
Nick Piggin 已提交
1773
	return insert_page(vma, addr, page, vma->vm_page_prot);
1774
}
1775
EXPORT_SYMBOL(vm_insert_page);
1776

N
Nick Piggin 已提交
1777
static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
R
Ross Zwisler 已提交
1778
			pfn_t pfn, pgprot_t prot, bool mkwrite)
N
Nick Piggin 已提交
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
{
	struct mm_struct *mm = vma->vm_mm;
	int retval;
	pte_t *pte, entry;
	spinlock_t *ptl;

	retval = -ENOMEM;
	pte = get_locked_pte(mm, addr, &ptl);
	if (!pte)
		goto out;
	retval = -EBUSY;
R
Ross Zwisler 已提交
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
	if (!pte_none(*pte)) {
		if (mkwrite) {
			/*
			 * For read faults on private mappings the PFN passed
			 * in may not match the PFN we have mapped if the
			 * mapped PFN is a writeable COW page.  In the mkwrite
			 * case we are creating a writable PTE for a shared
			 * mapping and we expect the PFNs to match.
			 */
			if (WARN_ON_ONCE(pte_pfn(*pte) != pfn_t_to_pfn(pfn)))
				goto out_unlock;
			entry = *pte;
			goto out_mkwrite;
		} else
			goto out_unlock;
	}
N
Nick Piggin 已提交
1806 1807

	/* Ok, finally just insert the thing.. */
1808 1809 1810 1811
	if (pfn_t_devmap(pfn))
		entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
	else
		entry = pte_mkspecial(pfn_t_pte(pfn, prot));
R
Ross Zwisler 已提交
1812 1813 1814 1815 1816 1817 1818

out_mkwrite:
	if (mkwrite) {
		entry = pte_mkyoung(entry);
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
	}

N
Nick Piggin 已提交
1819
	set_pte_at(mm, addr, pte, entry);
1820
	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
N
Nick Piggin 已提交
1821 1822 1823 1824 1825 1826 1827 1828

	retval = 0;
out_unlock:
	pte_unmap_unlock(pte, ptl);
out:
	return retval;
}

N
Nick Piggin 已提交
1829 1830 1831 1832 1833 1834
/**
 * vm_insert_pfn - insert single pfn into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 *
1835
 * Similar to vm_insert_page, this allows drivers to insert individual pages
N
Nick Piggin 已提交
1836 1837 1838 1839
 * they've allocated into a user vma. Same comments apply.
 *
 * This function should only be called from a vm_ops->fault handler, and
 * in that case the handler should return NULL.
N
Nick Piggin 已提交
1840 1841 1842 1843 1844
 *
 * vma cannot be a COW mapping.
 *
 * As this is called only for pages that do not currently exist, we
 * do not need to flush old virtual caches or the TLB.
N
Nick Piggin 已提交
1845 1846
 */
int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
N
Nick Piggin 已提交
1847
			unsigned long pfn)
A
Andy Lutomirski 已提交
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
{
	return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_pfn);

/**
 * vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 * @pgprot: pgprot flags for the inserted page
 *
 * This is exactly like vm_insert_pfn, except that it allows drivers to
 * to override pgprot on a per-page basis.
 *
 * This only makes sense for IO mappings, and it makes no sense for
 * cow mappings.  In general, using multiple vmas is preferable;
 * vm_insert_pfn_prot should only be used if using multiple VMAs is
 * impractical.
 */
int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
			unsigned long pfn, pgprot_t pgprot)
N
Nick Piggin 已提交
1870
{
1871
	int ret;
N
Nick Piggin 已提交
1872 1873 1874 1875 1876 1877
	/*
	 * Technically, architectures with pte_special can avoid all these
	 * restrictions (same for remap_pfn_range).  However we would like
	 * consistency in testing and feature parity among all, so we should
	 * try to keep these invariants in place for everybody.
	 */
J
Jared Hulbert 已提交
1878 1879 1880 1881 1882
	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
						(VM_PFNMAP|VM_MIXEDMAP));
	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
N
Nick Piggin 已提交
1883

N
Nick Piggin 已提交
1884 1885
	if (addr < vma->vm_start || addr >= vma->vm_end)
		return -EFAULT;
1886

1887 1888 1889
	if (!pfn_modify_allowed(pfn, pgprot))
		return -EACCES;

1890
	track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
1891

R
Ross Zwisler 已提交
1892 1893
	ret = insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
			false);
1894 1895

	return ret;
N
Nick Piggin 已提交
1896
}
A
Andy Lutomirski 已提交
1897
EXPORT_SYMBOL(vm_insert_pfn_prot);
N
Nick Piggin 已提交
1898

1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
{
	/* these checks mirror the abort conditions in vm_normal_page */
	if (vma->vm_flags & VM_MIXEDMAP)
		return true;
	if (pfn_t_devmap(pfn))
		return true;
	if (pfn_t_special(pfn))
		return true;
	if (is_zero_pfn(pfn_t_to_pfn(pfn)))
		return true;
	return false;
}

R
Ross Zwisler 已提交
1913 1914
static int __vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
			pfn_t pfn, bool mkwrite)
N
Nick Piggin 已提交
1915
{
1916 1917
	pgprot_t pgprot = vma->vm_page_prot;

1918
	BUG_ON(!vm_mixed_ok(vma, pfn));
N
Nick Piggin 已提交
1919

N
Nick Piggin 已提交
1920 1921
	if (addr < vma->vm_start || addr >= vma->vm_end)
		return -EFAULT;
1922 1923

	track_pfn_insert(vma, &pgprot, pfn);
N
Nick Piggin 已提交
1924

1925 1926 1927
	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
		return -EACCES;

N
Nick Piggin 已提交
1928 1929 1930 1931
	/*
	 * If we don't have pte special, then we have to use the pfn_valid()
	 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
	 * refcount the page if pfn_valid is true (hence insert_page rather
H
Hugh Dickins 已提交
1932 1933
	 * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
	 * without pte special, it would there be refcounted as a normal page.
N
Nick Piggin 已提交
1934
	 */
L
Laurent Dufour 已提交
1935 1936
	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
	    !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
N
Nick Piggin 已提交
1937 1938
		struct page *page;

1939 1940 1941 1942 1943 1944
		/*
		 * At this point we are committed to insert_page()
		 * regardless of whether the caller specified flags that
		 * result in pfn_t_has_page() == false.
		 */
		page = pfn_to_page(pfn_t_to_pfn(pfn));
1945
		return insert_page(vma, addr, page, pgprot);
N
Nick Piggin 已提交
1946
	}
R
Ross Zwisler 已提交
1947 1948 1949 1950 1951 1952 1953 1954
	return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
}

int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
			pfn_t pfn)
{
	return __vm_insert_mixed(vma, addr, pfn, false);

N
Nick Piggin 已提交
1955
}
N
Nick Piggin 已提交
1956
EXPORT_SYMBOL(vm_insert_mixed);
N
Nick Piggin 已提交
1957

1958 1959 1960 1961 1962 1963 1964 1965
/*
 *  If the insertion of PTE failed because someone else already added a
 *  different entry in the mean time, we treat that as success as we assume
 *  the same entry was actually inserted.
 */

vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
		unsigned long addr, pfn_t pfn)
R
Ross Zwisler 已提交
1966
{
1967 1968 1969 1970 1971 1972 1973 1974
	int err;

	err =  __vm_insert_mixed(vma, addr, pfn, true);
	if (err == -ENOMEM)
		return VM_FAULT_OOM;
	if (err < 0 && err != -EBUSY)
		return VM_FAULT_SIGBUS;
	return VM_FAULT_NOPAGE;
R
Ross Zwisler 已提交
1975
}
1976
EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
R
Ross Zwisler 已提交
1977

L
Linus Torvalds 已提交
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987
/*
 * maps a range of physical memory into the requested pages. the old
 * mappings are removed. any references to nonexistent pages results
 * in null mappings (currently treated as "copy-on-access")
 */
static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned long pfn, pgprot_t prot)
{
	pte_t *pte;
H
Hugh Dickins 已提交
1988
	spinlock_t *ptl;
1989
	int err = 0;
L
Linus Torvalds 已提交
1990

H
Hugh Dickins 已提交
1991
	pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
L
Linus Torvalds 已提交
1992 1993
	if (!pte)
		return -ENOMEM;
1994
	arch_enter_lazy_mmu_mode();
L
Linus Torvalds 已提交
1995 1996
	do {
		BUG_ON(!pte_none(*pte));
1997 1998 1999 2000
		if (!pfn_modify_allowed(pfn, prot)) {
			err = -EACCES;
			break;
		}
N
Nick Piggin 已提交
2001
		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
L
Linus Torvalds 已提交
2002 2003
		pfn++;
	} while (pte++, addr += PAGE_SIZE, addr != end);
2004
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
2005
	pte_unmap_unlock(pte - 1, ptl);
2006
	return err;
L
Linus Torvalds 已提交
2007 2008 2009 2010 2011 2012 2013 2014
}

static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
			unsigned long addr, unsigned long end,
			unsigned long pfn, pgprot_t prot)
{
	pmd_t *pmd;
	unsigned long next;
2015
	int err;
L
Linus Torvalds 已提交
2016 2017 2018 2019 2020

	pfn -= addr >> PAGE_SHIFT;
	pmd = pmd_alloc(mm, pud, addr);
	if (!pmd)
		return -ENOMEM;
2021
	VM_BUG_ON(pmd_trans_huge(*pmd));
L
Linus Torvalds 已提交
2022 2023
	do {
		next = pmd_addr_end(addr, end);
2024 2025 2026 2027
		err = remap_pte_range(mm, pmd, addr, next,
				pfn + (addr >> PAGE_SHIFT), prot);
		if (err)
			return err;
L
Linus Torvalds 已提交
2028 2029 2030 2031
	} while (pmd++, addr = next, addr != end);
	return 0;
}

2032
static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
L
Linus Torvalds 已提交
2033 2034 2035 2036 2037
			unsigned long addr, unsigned long end,
			unsigned long pfn, pgprot_t prot)
{
	pud_t *pud;
	unsigned long next;
2038
	int err;
L
Linus Torvalds 已提交
2039 2040

	pfn -= addr >> PAGE_SHIFT;
2041
	pud = pud_alloc(mm, p4d, addr);
L
Linus Torvalds 已提交
2042 2043 2044 2045
	if (!pud)
		return -ENOMEM;
	do {
		next = pud_addr_end(addr, end);
2046 2047 2048 2049
		err = remap_pmd_range(mm, pud, addr, next,
				pfn + (addr >> PAGE_SHIFT), prot);
		if (err)
			return err;
L
Linus Torvalds 已提交
2050 2051 2052 2053
	} while (pud++, addr = next, addr != end);
	return 0;
}

2054 2055 2056 2057 2058 2059
static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
			unsigned long addr, unsigned long end,
			unsigned long pfn, pgprot_t prot)
{
	p4d_t *p4d;
	unsigned long next;
2060
	int err;
2061 2062 2063 2064 2065 2066 2067

	pfn -= addr >> PAGE_SHIFT;
	p4d = p4d_alloc(mm, pgd, addr);
	if (!p4d)
		return -ENOMEM;
	do {
		next = p4d_addr_end(addr, end);
2068 2069 2070 2071
		err = remap_pud_range(mm, p4d, addr, next,
				pfn + (addr >> PAGE_SHIFT), prot);
		if (err)
			return err;
2072 2073 2074 2075
	} while (p4d++, addr = next, addr != end);
	return 0;
}

2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
/**
 * remap_pfn_range - remap kernel memory to userspace
 * @vma: user vma to map to
 * @addr: target user address to start at
 * @pfn: physical address of kernel memory
 * @size: size of map area
 * @prot: page protection flags for this mapping
 *
 *  Note: this is only safe if the mm semaphore is held when called.
 */
L
Linus Torvalds 已提交
2086 2087 2088 2089 2090
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
		    unsigned long pfn, unsigned long size, pgprot_t prot)
{
	pgd_t *pgd;
	unsigned long next;
2091
	unsigned long end = addr + PAGE_ALIGN(size);
L
Linus Torvalds 已提交
2092
	struct mm_struct *mm = vma->vm_mm;
2093
	unsigned long remap_pfn = pfn;
L
Linus Torvalds 已提交
2094 2095 2096 2097 2098 2099 2100
	int err;

	/*
	 * Physically remapped pages are special. Tell the
	 * rest of the world about it:
	 *   VM_IO tells people not to look at these pages
	 *	(accesses can have side effects).
2101 2102 2103
	 *   VM_PFNMAP tells the core MM that the base pages are just
	 *	raw PFN mappings, and do not have a "struct page" associated
	 *	with them.
2104 2105 2106 2107
	 *   VM_DONTEXPAND
	 *      Disable vma merging and expanding with mremap().
	 *   VM_DONTDUMP
	 *      Omit vma from core dump, even when VM_IO turned off.
L
Linus Torvalds 已提交
2108 2109 2110 2111
	 *
	 * There's a horrible special case to handle copy-on-write
	 * behaviour that some programs depend on. We mark the "original"
	 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2112
	 * See vm_normal_page() for details.
L
Linus Torvalds 已提交
2113
	 */
2114 2115 2116
	if (is_cow_mapping(vma->vm_flags)) {
		if (addr != vma->vm_start || end != vma->vm_end)
			return -EINVAL;
L
Linus Torvalds 已提交
2117
		vma->vm_pgoff = pfn;
2118 2119
	}

2120
	err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
2121
	if (err)
2122
		return -EINVAL;
L
Linus Torvalds 已提交
2123

2124
	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
L
Linus Torvalds 已提交
2125 2126 2127 2128 2129 2130 2131

	BUG_ON(addr >= end);
	pfn -= addr >> PAGE_SHIFT;
	pgd = pgd_offset(mm, addr);
	flush_cache_range(vma, addr, end);
	do {
		next = pgd_addr_end(addr, end);
2132
		err = remap_p4d_range(mm, pgd, addr, next,
L
Linus Torvalds 已提交
2133 2134 2135 2136
				pfn + (addr >> PAGE_SHIFT), prot);
		if (err)
			break;
	} while (pgd++, addr = next, addr != end);
2137 2138

	if (err)
2139
		untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
2140

L
Linus Torvalds 已提交
2141 2142 2143 2144
	return err;
}
EXPORT_SYMBOL(remap_pfn_range);

2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
/**
 * vm_iomap_memory - remap memory to userspace
 * @vma: user vma to map to
 * @start: start of area
 * @len: size of area
 *
 * This is a simplified io_remap_pfn_range() for common driver use. The
 * driver just needs to give us the physical memory range to be mapped,
 * we'll figure out the rest from the vma information.
 *
 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
 * whatever write-combining details or similar.
 */
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
	unsigned long vm_len, pfn, pages;

	/* Check that the physical memory area passed in looks valid */
	if (start + len < start)
		return -EINVAL;
	/*
	 * You *really* shouldn't map things that aren't page-aligned,
	 * but we've historically allowed it because IO memory might
	 * just have smaller alignment.
	 */
	len += start & ~PAGE_MASK;
	pfn = start >> PAGE_SHIFT;
	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
	if (pfn + pages < pfn)
		return -EINVAL;

	/* We start the mapping 'vm_pgoff' pages into the area */
	if (vma->vm_pgoff > pages)
		return -EINVAL;
	pfn += vma->vm_pgoff;
	pages -= vma->vm_pgoff;

	/* Can we fit all of the mapping? */
	vm_len = vma->vm_end - vma->vm_start;
	if (vm_len >> PAGE_SHIFT > pages)
		return -EINVAL;

	/* Ok, let it rip */
	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);

2192 2193 2194 2195 2196 2197
static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
				     unsigned long addr, unsigned long end,
				     pte_fn_t fn, void *data)
{
	pte_t *pte;
	int err;
2198
	pgtable_t token;
2199
	spinlock_t *uninitialized_var(ptl);
2200 2201 2202 2203 2204 2205 2206 2207 2208

	pte = (mm == &init_mm) ?
		pte_alloc_kernel(pmd, addr) :
		pte_alloc_map_lock(mm, pmd, addr, &ptl);
	if (!pte)
		return -ENOMEM;

	BUG_ON(pmd_huge(*pmd));

2209 2210
	arch_enter_lazy_mmu_mode();

2211
	token = pmd_pgtable(*pmd);
2212 2213

	do {
2214
		err = fn(pte++, token, addr, data);
2215 2216
		if (err)
			break;
2217
	} while (addr += PAGE_SIZE, addr != end);
2218

2219 2220
	arch_leave_lazy_mmu_mode();

2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
	if (mm != &init_mm)
		pte_unmap_unlock(pte-1, ptl);
	return err;
}

static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
				     unsigned long addr, unsigned long end,
				     pte_fn_t fn, void *data)
{
	pmd_t *pmd;
	unsigned long next;
	int err;

A
Andi Kleen 已提交
2234 2235
	BUG_ON(pud_huge(*pud));

2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
	pmd = pmd_alloc(mm, pud, addr);
	if (!pmd)
		return -ENOMEM;
	do {
		next = pmd_addr_end(addr, end);
		err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
		if (err)
			break;
	} while (pmd++, addr = next, addr != end);
	return err;
}

2248
static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2249 2250 2251 2252 2253 2254 2255
				     unsigned long addr, unsigned long end,
				     pte_fn_t fn, void *data)
{
	pud_t *pud;
	unsigned long next;
	int err;

2256
	pud = pud_alloc(mm, p4d, addr);
2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
	if (!pud)
		return -ENOMEM;
	do {
		next = pud_addr_end(addr, end);
		err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
		if (err)
			break;
	} while (pud++, addr = next, addr != end);
	return err;
}

2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
				     unsigned long addr, unsigned long end,
				     pte_fn_t fn, void *data)
{
	p4d_t *p4d;
	unsigned long next;
	int err;

	p4d = p4d_alloc(mm, pgd, addr);
	if (!p4d)
		return -ENOMEM;
	do {
		next = p4d_addr_end(addr, end);
		err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
		if (err)
			break;
	} while (p4d++, addr = next, addr != end);
	return err;
}

2288 2289 2290 2291 2292 2293 2294 2295 2296
/*
 * Scan a region of virtual memory, filling in page tables as necessary
 * and calling a provided function on each leaf page table.
 */
int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
			unsigned long size, pte_fn_t fn, void *data)
{
	pgd_t *pgd;
	unsigned long next;
2297
	unsigned long end = addr + size;
2298 2299
	int err;

2300 2301 2302
	if (WARN_ON(addr >= end))
		return -EINVAL;

2303 2304 2305
	pgd = pgd_offset(mm, addr);
	do {
		next = pgd_addr_end(addr, end);
2306
		err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
2307 2308 2309
		if (err)
			break;
	} while (pgd++, addr = next, addr != end);
2310

2311 2312 2313 2314
	return err;
}
EXPORT_SYMBOL_GPL(apply_to_page_range);

2315
/*
2316 2317 2318 2319 2320
 * handle_pte_fault chooses page fault handler according to an entry which was
 * read non-atomically.  Before making any commitment, on those architectures
 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
 * parts, do_swap_page must check under lock before unmapping the pte and
 * proceeding (but do_wp_page is only called after already making such a check;
2321
 * and do_anonymous_page can safely check later on).
2322
 */
H
Hugh Dickins 已提交
2323
static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
2324 2325 2326 2327 2328
				pte_t *page_table, pte_t orig_pte)
{
	int same = 1;
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
	if (sizeof(pte_t) > sizeof(unsigned long)) {
H
Hugh Dickins 已提交
2329 2330
		spinlock_t *ptl = pte_lockptr(mm, pmd);
		spin_lock(ptl);
2331
		same = pte_same(*page_table, orig_pte);
H
Hugh Dickins 已提交
2332
		spin_unlock(ptl);
2333 2334 2335 2336 2337 2338
	}
#endif
	pte_unmap(page_table);
	return same;
}

2339
static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
2340
{
2341 2342
	debug_dma_assert_idle(src);

2343 2344 2345 2346 2347 2348 2349
	/*
	 * If the source page was a PFN mapping, we don't have
	 * a "struct page" for it. We do a best-effort copy by
	 * just copying from the original user address. If that
	 * fails, we just zero-fill it. Live with it.
	 */
	if (unlikely(!src)) {
2350
		void *kaddr = kmap_atomic(dst);
L
Linus Torvalds 已提交
2351 2352 2353 2354 2355 2356 2357 2358 2359
		void __user *uaddr = (void __user *)(va & PAGE_MASK);

		/*
		 * This really shouldn't fail, because the page is there
		 * in the page tables. But it might just be unreadable,
		 * in which case we just give up and fill the result with
		 * zeroes.
		 */
		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
2360
			clear_page(kaddr);
2361
		kunmap_atomic(kaddr);
2362
		flush_dcache_page(dst);
N
Nick Piggin 已提交
2363 2364
	} else
		copy_user_highpage(dst, src, va, vma);
2365 2366
}

2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380
static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
{
	struct file *vm_file = vma->vm_file;

	if (vm_file)
		return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;

	/*
	 * Special mappings (e.g. VDSO) do not have any file so fake
	 * a default GFP_KERNEL for them.
	 */
	return GFP_KERNEL;
}

2381 2382 2383 2384 2385 2386
/*
 * Notify the address space that the page is about to become writable so that
 * it can prohibit this or wait for the page to get into an appropriate state.
 *
 * We do this without the lock held, so that it can sleep if it needs to.
 */
2387
static int do_page_mkwrite(struct vm_fault *vmf)
2388 2389
{
	int ret;
2390 2391
	struct page *page = vmf->page;
	unsigned int old_flags = vmf->flags;
2392

2393
	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
2394

2395
	ret = vmf->vma->vm_ops->page_mkwrite(vmf);
2396 2397
	/* Restore original flags so that caller is not surprised */
	vmf->flags = old_flags;
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
		return ret;
	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
		lock_page(page);
		if (!page->mapping) {
			unlock_page(page);
			return 0; /* retry */
		}
		ret |= VM_FAULT_LOCKED;
	} else
		VM_BUG_ON_PAGE(!PageLocked(page), page);
	return ret;
}

2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
/*
 * Handle dirtying of a page in shared file mapping on a write fault.
 *
 * The function expects the page to be locked and unlocks it.
 */
static void fault_dirty_shared_page(struct vm_area_struct *vma,
				    struct page *page)
{
	struct address_space *mapping;
	bool dirtied;
	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;

	dirtied = set_page_dirty(page);
	VM_BUG_ON_PAGE(PageAnon(page), page);
	/*
	 * Take a local copy of the address_space - page.mapping may be zeroed
	 * by truncate after unlock_page().   The address_space itself remains
	 * pinned by vma->vm_file's reference.  We rely on unlock_page()'s
	 * release semantics to prevent the compiler from undoing this copying.
	 */
	mapping = page_rmapping(page);
	unlock_page(page);

	if ((dirtied || page_mkwrite) && mapping) {
		/*
		 * Some device drivers do not set page.mapping
		 * but still dirty their pages
		 */
		balance_dirty_pages_ratelimited(mapping);
	}

	if (!page_mkwrite)
		file_update_time(vma->vm_file);
}

2447 2448 2449 2450 2451 2452 2453 2454
/*
 * Handle write page faults for pages that can be reused in the current vma
 *
 * This can happen either due to the mapping being with the VM_SHARED flag,
 * or due to us being the last reference standing to the page. In either
 * case, all we need to do here is to mark the page as writable and update
 * any related book-keeping.
 */
2455
static inline void wp_page_reuse(struct vm_fault *vmf)
J
Jan Kara 已提交
2456
	__releases(vmf->ptl)
2457
{
J
Jan Kara 已提交
2458
	struct vm_area_struct *vma = vmf->vma;
J
Jan Kara 已提交
2459
	struct page *page = vmf->page;
2460 2461 2462 2463 2464 2465 2466 2467 2468
	pte_t entry;
	/*
	 * Clear the pages cpupid information as the existing
	 * information potentially belongs to a now completely
	 * unrelated process.
	 */
	if (page)
		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);

J
Jan Kara 已提交
2469 2470
	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
	entry = pte_mkyoung(vmf->orig_pte);
2471
	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
J
Jan Kara 已提交
2472 2473 2474
	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
		update_mmu_cache(vma, vmf->address, vmf->pte);
	pte_unmap_unlock(vmf->pte, vmf->ptl);
2475 2476
}

2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492
/*
 * Handle the case of a page which we actually need to copy to a new page.
 *
 * Called with mmap_sem locked and the old page referenced, but
 * without the ptl held.
 *
 * High level logic flow:
 *
 * - Allocate a page, copy the content of the old page to the new one.
 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
 * - Take the PTL. If the pte changed, bail out and release the allocated page
 * - If the pte is still the way we remember it, update the page table and all
 *   relevant references. This includes dropping the reference the page-table
 *   held to the old page, as well as updating the rmap.
 * - In any case, unlock the PTL and drop the reference we took to the old page.
 */
J
Jan Kara 已提交
2493
static int wp_page_copy(struct vm_fault *vmf)
2494
{
J
Jan Kara 已提交
2495
	struct vm_area_struct *vma = vmf->vma;
K
Kirill A. Shutemov 已提交
2496
	struct mm_struct *mm = vma->vm_mm;
J
Jan Kara 已提交
2497
	struct page *old_page = vmf->page;
2498 2499 2500
	struct page *new_page = NULL;
	pte_t entry;
	int page_copied = 0;
J
Jan Kara 已提交
2501
	const unsigned long mmun_start = vmf->address & PAGE_MASK;
K
Kirill A. Shutemov 已提交
2502
	const unsigned long mmun_end = mmun_start + PAGE_SIZE;
2503 2504 2505 2506 2507
	struct mem_cgroup *memcg;

	if (unlikely(anon_vma_prepare(vma)))
		goto oom;

J
Jan Kara 已提交
2508
	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
J
Jan Kara 已提交
2509 2510
		new_page = alloc_zeroed_user_highpage_movable(vma,
							      vmf->address);
2511 2512 2513
		if (!new_page)
			goto oom;
	} else {
K
Kirill A. Shutemov 已提交
2514
		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
J
Jan Kara 已提交
2515
				vmf->address);
2516 2517
		if (!new_page)
			goto oom;
J
Jan Kara 已提交
2518
		cow_user_page(new_page, old_page, vmf->address, vma);
2519 2520
	}

2521
	if (mem_cgroup_try_charge_delay(new_page, mm, GFP_KERNEL, &memcg, false))
2522 2523
		goto oom_free_new;

2524 2525
	__SetPageUptodate(new_page);

2526 2527 2528 2529 2530
	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);

	/*
	 * Re-check the pte - we dropped the lock
	 */
J
Jan Kara 已提交
2531
	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
J
Jan Kara 已提交
2532
	if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
2533 2534
		if (old_page) {
			if (!PageAnon(old_page)) {
2535 2536
				dec_mm_counter_fast(mm,
						mm_counter_file(old_page));
2537 2538 2539 2540 2541
				inc_mm_counter_fast(mm, MM_ANONPAGES);
			}
		} else {
			inc_mm_counter_fast(mm, MM_ANONPAGES);
		}
J
Jan Kara 已提交
2542
		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
2543 2544 2545 2546 2547 2548 2549 2550
		entry = mk_pte(new_page, vma->vm_page_prot);
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
		/*
		 * Clear the pte entry and flush it first, before updating the
		 * pte with the new entry. This will avoid a race condition
		 * seen in the presence of one thread doing SMC and another
		 * thread doing COW.
		 */
J
Jan Kara 已提交
2551 2552
		ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
		page_add_new_anon_rmap(new_page, vma, vmf->address, false);
2553
		mem_cgroup_commit_charge(new_page, memcg, false, false);
2554 2555 2556 2557 2558 2559
		lru_cache_add_active_or_unevictable(new_page, vma);
		/*
		 * We call the notify macro here because, when using secondary
		 * mmu page tables (such as kvm shadow page tables), we want the
		 * new page to be mapped directly into the secondary page table.
		 */
J
Jan Kara 已提交
2560 2561
		set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
		update_mmu_cache(vma, vmf->address, vmf->pte);
2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584
		if (old_page) {
			/*
			 * Only after switching the pte to the new page may
			 * we remove the mapcount here. Otherwise another
			 * process may come and find the rmap count decremented
			 * before the pte is switched to the new page, and
			 * "reuse" the old page writing into it while our pte
			 * here still points into it and can be read by other
			 * threads.
			 *
			 * The critical issue is to order this
			 * page_remove_rmap with the ptp_clear_flush above.
			 * Those stores are ordered by (if nothing else,)
			 * the barrier present in the atomic_add_negative
			 * in page_remove_rmap.
			 *
			 * Then the TLB flush in ptep_clear_flush ensures that
			 * no process can access the old page before the
			 * decremented mapcount is visible. And the old page
			 * cannot be reused until after the decremented
			 * mapcount is visible. So transitively, TLBs to
			 * old page will be flushed before it can be reused.
			 */
2585
			page_remove_rmap(old_page, false);
2586 2587 2588 2589 2590 2591
		}

		/* Free the old page.. */
		new_page = old_page;
		page_copied = 1;
	} else {
2592
		mem_cgroup_cancel_charge(new_page, memcg, false);
2593 2594 2595
	}

	if (new_page)
2596
		put_page(new_page);
2597

J
Jan Kara 已提交
2598
	pte_unmap_unlock(vmf->pte, vmf->ptl);
2599 2600 2601 2602 2603
	/*
	 * No need to double call mmu_notifier->invalidate_range() callback as
	 * the above ptep_clear_flush_notify() did already call it.
	 */
	mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
2604 2605 2606 2607 2608 2609 2610
	if (old_page) {
		/*
		 * Don't let another task, with possibly unlocked vma,
		 * keep the mlocked page.
		 */
		if (page_copied && (vma->vm_flags & VM_LOCKED)) {
			lock_page(old_page);	/* LRU manipulation */
2611 2612
			if (PageMlocked(old_page))
				munlock_vma_page(old_page);
2613 2614
			unlock_page(old_page);
		}
2615
		put_page(old_page);
2616 2617 2618
	}
	return page_copied ? VM_FAULT_WRITE : 0;
oom_free_new:
2619
	put_page(new_page);
2620 2621
oom:
	if (old_page)
2622
		put_page(old_page);
2623 2624 2625
	return VM_FAULT_OOM;
}

2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651
/**
 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
 *			  writeable once the page is prepared
 *
 * @vmf: structure describing the fault
 *
 * This function handles all that is needed to finish a write page fault in a
 * shared mapping due to PTE being read-only once the mapped page is prepared.
 * It handles locking of PTE and modifying it. The function returns
 * VM_FAULT_WRITE on success, 0 when PTE got changed before we acquired PTE
 * lock.
 *
 * The function expects the page to be locked or other protection against
 * concurrent faults / writeback (such as DAX radix tree locks).
 */
int finish_mkwrite_fault(struct vm_fault *vmf)
{
	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
				       &vmf->ptl);
	/*
	 * We might have raced with another page fault while we released the
	 * pte_offset_map_lock.
	 */
	if (!pte_same(*vmf->pte, vmf->orig_pte)) {
		pte_unmap_unlock(vmf->pte, vmf->ptl);
2652
		return VM_FAULT_NOPAGE;
2653 2654
	}
	wp_page_reuse(vmf);
2655
	return 0;
2656 2657
}

2658 2659 2660 2661
/*
 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
 * mapping
 */
J
Jan Kara 已提交
2662
static int wp_pfn_shared(struct vm_fault *vmf)
2663
{
J
Jan Kara 已提交
2664
	struct vm_area_struct *vma = vmf->vma;
K
Kirill A. Shutemov 已提交
2665

2666 2667 2668
	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
		int ret;

J
Jan Kara 已提交
2669
		pte_unmap_unlock(vmf->pte, vmf->ptl);
2670
		vmf->flags |= FAULT_FLAG_MKWRITE;
2671
		ret = vma->vm_ops->pfn_mkwrite(vmf);
2672
		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
2673
			return ret;
2674
		return finish_mkwrite_fault(vmf);
2675
	}
2676 2677
	wp_page_reuse(vmf);
	return VM_FAULT_WRITE;
2678 2679
}

J
Jan Kara 已提交
2680
static int wp_page_shared(struct vm_fault *vmf)
J
Jan Kara 已提交
2681
	__releases(vmf->ptl)
2682
{
J
Jan Kara 已提交
2683
	struct vm_area_struct *vma = vmf->vma;
2684

J
Jan Kara 已提交
2685
	get_page(vmf->page);
2686 2687 2688 2689

	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
		int tmp;

J
Jan Kara 已提交
2690
		pte_unmap_unlock(vmf->pte, vmf->ptl);
2691
		tmp = do_page_mkwrite(vmf);
2692 2693
		if (unlikely(!tmp || (tmp &
				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
J
Jan Kara 已提交
2694
			put_page(vmf->page);
2695 2696
			return tmp;
		}
2697
		tmp = finish_mkwrite_fault(vmf);
2698
		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
J
Jan Kara 已提交
2699 2700
			unlock_page(vmf->page);
			put_page(vmf->page);
2701
			return tmp;
2702
		}
2703 2704
	} else {
		wp_page_reuse(vmf);
2705
		lock_page(vmf->page);
2706
	}
2707 2708
	fault_dirty_shared_page(vma, vmf->page);
	put_page(vmf->page);
2709

2710
	return VM_FAULT_WRITE;
2711 2712
}

L
Linus Torvalds 已提交
2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726
/*
 * This routine handles present pages, when users try to write
 * to a shared page. It is done by copying the page to a new address
 * and decrementing the shared-page counter for the old page.
 *
 * Note that this routine assumes that the protection checks have been
 * done by the caller (the low-level page fault routine in most cases).
 * Thus we can safely just mark it writable once we've done any necessary
 * COW.
 *
 * We also mark the page dirty at this point even though the page will
 * change only once the write actually happens. This avoids a few races,
 * and potentially makes it more efficient.
 *
2727 2728 2729
 * We enter with non-exclusive mmap_sem (to exclude vma changes,
 * but allow concurrent faults), with pte both mapped and locked.
 * We return with mmap_sem still held, but pte unmapped and unlocked.
L
Linus Torvalds 已提交
2730
 */
J
Jan Kara 已提交
2731
static int do_wp_page(struct vm_fault *vmf)
J
Jan Kara 已提交
2732
	__releases(vmf->ptl)
L
Linus Torvalds 已提交
2733
{
J
Jan Kara 已提交
2734
	struct vm_area_struct *vma = vmf->vma;
L
Linus Torvalds 已提交
2735

J
Jan Kara 已提交
2736 2737
	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
	if (!vmf->page) {
2738
		/*
2739 2740
		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
		 * VM_PFNMAP VMA.
2741 2742
		 *
		 * We should not cow pages in a shared writeable mapping.
2743
		 * Just mark the pages writable and/or call ops->pfn_mkwrite.
2744 2745 2746
		 */
		if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
				     (VM_WRITE|VM_SHARED))
J
Jan Kara 已提交
2747
			return wp_pfn_shared(vmf);
2748

J
Jan Kara 已提交
2749
		pte_unmap_unlock(vmf->pte, vmf->ptl);
J
Jan Kara 已提交
2750
		return wp_page_copy(vmf);
2751
	}
L
Linus Torvalds 已提交
2752

2753
	/*
P
Peter Zijlstra 已提交
2754 2755
	 * Take out anonymous pages first, anonymous shared vmas are
	 * not dirty accountable.
2756
	 */
J
Jan Kara 已提交
2757
	if (PageAnon(vmf->page) && !PageKsm(vmf->page)) {
2758
		int total_map_swapcount;
J
Jan Kara 已提交
2759 2760
		if (!trylock_page(vmf->page)) {
			get_page(vmf->page);
J
Jan Kara 已提交
2761
			pte_unmap_unlock(vmf->pte, vmf->ptl);
J
Jan Kara 已提交
2762
			lock_page(vmf->page);
J
Jan Kara 已提交
2763 2764
			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
					vmf->address, &vmf->ptl);
J
Jan Kara 已提交
2765
			if (!pte_same(*vmf->pte, vmf->orig_pte)) {
J
Jan Kara 已提交
2766
				unlock_page(vmf->page);
J
Jan Kara 已提交
2767
				pte_unmap_unlock(vmf->pte, vmf->ptl);
J
Jan Kara 已提交
2768
				put_page(vmf->page);
2769
				return 0;
2770
			}
J
Jan Kara 已提交
2771
			put_page(vmf->page);
P
Peter Zijlstra 已提交
2772
		}
2773 2774
		if (reuse_swap_page(vmf->page, &total_map_swapcount)) {
			if (total_map_swapcount == 1) {
2775 2776 2777 2778 2779 2780 2781
				/*
				 * The page is all ours. Move it to
				 * our anon_vma so the rmap code will
				 * not search our parent or siblings.
				 * Protected against the rmap code by
				 * the page lock.
				 */
J
Jan Kara 已提交
2782
				page_move_anon_rmap(vmf->page, vma);
2783
			}
J
Jan Kara 已提交
2784
			unlock_page(vmf->page);
2785 2786
			wp_page_reuse(vmf);
			return VM_FAULT_WRITE;
2787
		}
J
Jan Kara 已提交
2788
		unlock_page(vmf->page);
P
Peter Zijlstra 已提交
2789
	} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2790
					(VM_WRITE|VM_SHARED))) {
J
Jan Kara 已提交
2791
		return wp_page_shared(vmf);
L
Linus Torvalds 已提交
2792 2793 2794 2795 2796
	}

	/*
	 * Ok, we need to copy. Oh, well..
	 */
J
Jan Kara 已提交
2797
	get_page(vmf->page);
2798

J
Jan Kara 已提交
2799
	pte_unmap_unlock(vmf->pte, vmf->ptl);
J
Jan Kara 已提交
2800
	return wp_page_copy(vmf);
L
Linus Torvalds 已提交
2801 2802
}

2803
static void unmap_mapping_range_vma(struct vm_area_struct *vma,
L
Linus Torvalds 已提交
2804 2805 2806
		unsigned long start_addr, unsigned long end_addr,
		struct zap_details *details)
{
2807
	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
L
Linus Torvalds 已提交
2808 2809
}

2810
static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
L
Linus Torvalds 已提交
2811 2812 2813 2814 2815
					    struct zap_details *details)
{
	struct vm_area_struct *vma;
	pgoff_t vba, vea, zba, zea;

2816
	vma_interval_tree_foreach(vma, root,
L
Linus Torvalds 已提交
2817 2818 2819
			details->first_index, details->last_index) {

		vba = vma->vm_pgoff;
2820
		vea = vba + vma_pages(vma) - 1;
L
Linus Torvalds 已提交
2821 2822 2823 2824 2825 2826 2827
		zba = details->first_index;
		if (zba < vba)
			zba = vba;
		zea = details->last_index;
		if (zea > vea)
			zea = vea;

2828
		unmap_mapping_range_vma(vma,
L
Linus Torvalds 已提交
2829 2830
			((zba - vba) << PAGE_SHIFT) + vma->vm_start,
			((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
2831
				details);
L
Linus Torvalds 已提交
2832 2833 2834
	}
}

M
Matthew Wilcox 已提交
2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
/**
 * unmap_mapping_pages() - Unmap pages from processes.
 * @mapping: The address space containing pages to be unmapped.
 * @start: Index of first page to be unmapped.
 * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
 * @even_cows: Whether to unmap even private COWed pages.
 *
 * Unmap the pages in this address space from any userspace process which
 * has them mmaped.  Generally, you want to remove COWed pages as well when
 * a file is being truncated, but not when invalidating pages from the page
 * cache.
 */
void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
		pgoff_t nr, bool even_cows)
{
	struct zap_details details = { };

	details.check_mapping = even_cows ? NULL : mapping;
	details.first_index = start;
	details.last_index = start + nr - 1;
	if (details.last_index < details.first_index)
		details.last_index = ULONG_MAX;

	i_mmap_lock_write(mapping);
	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
		unmap_mapping_range_tree(&mapping->i_mmap, &details);
	i_mmap_unlock_write(mapping);
}

L
Linus Torvalds 已提交
2864
/**
2865
 * unmap_mapping_range - unmap the portion of all mmaps in the specified
M
Matthew Wilcox 已提交
2866
 * address_space corresponding to the specified byte range in the underlying
2867 2868
 * file.
 *
M
Martin Waitz 已提交
2869
 * @mapping: the address space containing mmaps to be unmapped.
L
Linus Torvalds 已提交
2870 2871
 * @holebegin: byte in first page to unmap, relative to the start of
 * the underlying file.  This will be rounded down to a PAGE_SIZE
N
npiggin@suse.de 已提交
2872
 * boundary.  Note that this is different from truncate_pagecache(), which
L
Linus Torvalds 已提交
2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
 * must keep the partial page.  In contrast, we must get rid of
 * partial pages.
 * @holelen: size of prospective hole in bytes.  This will be rounded
 * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
 * end of the file.
 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
 * but 0 when invalidating pagecache, don't throw away private data.
 */
void unmap_mapping_range(struct address_space *mapping,
		loff_t const holebegin, loff_t const holelen, int even_cows)
{
	pgoff_t hba = holebegin >> PAGE_SHIFT;
	pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;

	/* Check for overflow. */
	if (sizeof(holelen) > sizeof(hlen)) {
		long long holeend =
			(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
		if (holeend & ~(long long)ULONG_MAX)
			hlen = ULONG_MAX - hba + 1;
	}

M
Matthew Wilcox 已提交
2895
	unmap_mapping_pages(mapping, hba, hlen, even_cows);
L
Linus Torvalds 已提交
2896 2897 2898 2899
}
EXPORT_SYMBOL(unmap_mapping_range);

/*
2900 2901
 * We enter with non-exclusive mmap_sem (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
2902 2903 2904 2905
 * We return with pte unmapped and unlocked.
 *
 * We return with the mmap_sem locked or unlocked in the same cases
 * as does filemap_fault().
L
Linus Torvalds 已提交
2906
 */
J
Jan Kara 已提交
2907
int do_swap_page(struct vm_fault *vmf)
L
Linus Torvalds 已提交
2908
{
J
Jan Kara 已提交
2909
	struct vm_area_struct *vma = vmf->vma;
M
Minchan Kim 已提交
2910
	struct page *page = NULL, *swapcache;
2911
	struct mem_cgroup *memcg;
2912
	swp_entry_t entry;
L
Linus Torvalds 已提交
2913
	pte_t pte;
2914
	int locked;
2915
	int exclusive = 0;
N
Nick Piggin 已提交
2916
	int ret = 0;
L
Linus Torvalds 已提交
2917

M
Minchan Kim 已提交
2918
	if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
2919
		goto out;
2920

J
Jan Kara 已提交
2921
	entry = pte_to_swp_entry(vmf->orig_pte);
2922 2923
	if (unlikely(non_swap_entry(entry))) {
		if (is_migration_entry(entry)) {
J
Jan Kara 已提交
2924 2925
			migration_entry_wait(vma->vm_mm, vmf->pmd,
					     vmf->address);
2926 2927 2928 2929 2930 2931 2932 2933
		} else if (is_device_private_entry(entry)) {
			/*
			 * For un-addressable device memory we call the pgmap
			 * fault handler callback. The callback must migrate
			 * the page back to some CPU accessible page.
			 */
			ret = device_private_entry_fault(vma, vmf->address, entry,
						 vmf->flags, vmf->pmd);
2934 2935 2936
		} else if (is_hwpoison_entry(entry)) {
			ret = VM_FAULT_HWPOISON;
		} else {
J
Jan Kara 已提交
2937
			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
H
Hugh Dickins 已提交
2938
			ret = VM_FAULT_SIGBUS;
2939
		}
2940 2941
		goto out;
	}
2942 2943


2944
	delayacct_set_flag(DELAYACCT_PF_SWAPIN);
M
Minchan Kim 已提交
2945 2946
	page = lookup_swap_cache(entry, vma, vmf->address);
	swapcache = page;
2947

L
Linus Torvalds 已提交
2948
	if (!page) {
2949 2950
		struct swap_info_struct *si = swp_swap_info(entry);

2951 2952
		if (si->flags & SWP_SYNCHRONOUS_IO &&
				__swap_count(si, entry) == 1) {
2953
			/* skip swapcache */
2954 2955
			page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
							vmf->address);
2956 2957 2958 2959 2960 2961 2962
			if (page) {
				__SetPageLocked(page);
				__SetPageSwapBacked(page);
				set_page_private(page, entry.val);
				lru_cache_add_anon(page);
				swap_readpage(page, true);
			}
2963
		} else {
2964 2965
			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
						vmf);
2966
			swapcache = page;
2967 2968
		}

L
Linus Torvalds 已提交
2969 2970
		if (!page) {
			/*
2971 2972
			 * Back out if somebody else faulted in this pte
			 * while we released the pte lock.
L
Linus Torvalds 已提交
2973
			 */
J
Jan Kara 已提交
2974 2975
			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
					vmf->address, &vmf->ptl);
J
Jan Kara 已提交
2976
			if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
L
Linus Torvalds 已提交
2977
				ret = VM_FAULT_OOM;
2978
			delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2979
			goto unlock;
L
Linus Torvalds 已提交
2980 2981 2982 2983
		}

		/* Had to read the page from swap area: Major fault */
		ret = VM_FAULT_MAJOR;
2984
		count_vm_event(PGMAJFAULT);
2985
		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
2986
	} else if (PageHWPoison(page)) {
2987 2988 2989 2990
		/*
		 * hwpoisoned dirty swapcache pages are kept for killing
		 * owner processes (which may be unknown at hwpoison time)
		 */
2991 2992
		ret = VM_FAULT_HWPOISON;
		delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2993
		goto out_release;
L
Linus Torvalds 已提交
2994 2995
	}

J
Jan Kara 已提交
2996
	locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
R
Rik van Riel 已提交
2997

2998
	delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2999 3000 3001 3002
	if (!locked) {
		ret |= VM_FAULT_RETRY;
		goto out_release;
	}
3003

A
Andrea Arcangeli 已提交
3004
	/*
3005 3006 3007 3008
	 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
	 * release the swapcache from under us.  The page pin, and pte_same
	 * test below, are not enough to exclude that.  Even if it is still
	 * swapcache, we need to check that the page's swap has not changed.
A
Andrea Arcangeli 已提交
3009
	 */
3010 3011
	if (unlikely((!PageSwapCache(page) ||
			page_private(page) != entry.val)) && swapcache)
A
Andrea Arcangeli 已提交
3012 3013
		goto out_page;

J
Jan Kara 已提交
3014
	page = ksm_might_need_to_copy(page, vma, vmf->address);
3015 3016 3017 3018
	if (unlikely(!page)) {
		ret = VM_FAULT_OOM;
		page = swapcache;
		goto out_page;
H
Hugh Dickins 已提交
3019 3020
	}

3021 3022
	if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL,
					&memcg, false)) {
3023
		ret = VM_FAULT_OOM;
3024
		goto out_page;
3025 3026
	}

L
Linus Torvalds 已提交
3027
	/*
3028
	 * Back out if somebody else already faulted in this pte.
L
Linus Torvalds 已提交
3029
	 */
J
Jan Kara 已提交
3030 3031
	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
			&vmf->ptl);
J
Jan Kara 已提交
3032
	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
3033 3034 3035 3036 3037
		goto out_nomap;

	if (unlikely(!PageUptodate(page))) {
		ret = VM_FAULT_SIGBUS;
		goto out_nomap;
L
Linus Torvalds 已提交
3038 3039
	}

3040 3041 3042 3043 3044 3045 3046 3047 3048
	/*
	 * The page isn't present yet, go ahead with the fault.
	 *
	 * Be careful about the sequence of operations here.
	 * To get its accounting right, reuse_swap_page() must be called
	 * while the page is counted on swap but not yet in mapcount i.e.
	 * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
	 * must be called after the swap_free(), or it will never succeed.
	 */
L
Linus Torvalds 已提交
3049

K
Kirill A. Shutemov 已提交
3050 3051
	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
	dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
L
Linus Torvalds 已提交
3052
	pte = mk_pte(page, vma->vm_page_prot);
J
Jan Kara 已提交
3053
	if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
L
Linus Torvalds 已提交
3054
		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
J
Jan Kara 已提交
3055
		vmf->flags &= ~FAULT_FLAG_WRITE;
3056
		ret |= VM_FAULT_WRITE;
3057
		exclusive = RMAP_EXCLUSIVE;
L
Linus Torvalds 已提交
3058 3059
	}
	flush_icache_page(vma, page);
J
Jan Kara 已提交
3060
	if (pte_swp_soft_dirty(vmf->orig_pte))
3061
		pte = pte_mksoft_dirty(pte);
J
Jan Kara 已提交
3062
	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
3063
	arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
J
Jan Kara 已提交
3064
	vmf->orig_pte = pte;
3065 3066 3067

	/* ksm created a completely new copy */
	if (unlikely(page != swapcache && swapcache)) {
J
Jan Kara 已提交
3068
		page_add_new_anon_rmap(page, vma, vmf->address, false);
3069
		mem_cgroup_commit_charge(page, memcg, false, false);
3070
		lru_cache_add_active_or_unevictable(page, vma);
3071 3072 3073 3074
	} else {
		do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
		mem_cgroup_commit_charge(page, memcg, true, false);
		activate_page(page);
3075
	}
L
Linus Torvalds 已提交
3076

3077
	swap_free(entry);
3078 3079
	if (mem_cgroup_swap_full(page) ||
	    (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
3080
		try_to_free_swap(page);
3081
	unlock_page(page);
3082
	if (page != swapcache && swapcache) {
A
Andrea Arcangeli 已提交
3083 3084 3085 3086 3087 3088 3089 3090 3091
		/*
		 * Hold the lock to avoid the swap entry to be reused
		 * until we take the PT lock for the pte_same() check
		 * (to avoid false positives from pte_same). For
		 * further safety release the lock after the swap_free
		 * so that the swap count won't change under a
		 * parallel locked swapcache.
		 */
		unlock_page(swapcache);
3092
		put_page(swapcache);
A
Andrea Arcangeli 已提交
3093
	}
3094

J
Jan Kara 已提交
3095
	if (vmf->flags & FAULT_FLAG_WRITE) {
J
Jan Kara 已提交
3096
		ret |= do_wp_page(vmf);
3097 3098
		if (ret & VM_FAULT_ERROR)
			ret &= VM_FAULT_ERROR;
L
Linus Torvalds 已提交
3099 3100 3101 3102
		goto out;
	}

	/* No need to invalidate - it was non-present before */
J
Jan Kara 已提交
3103
	update_mmu_cache(vma, vmf->address, vmf->pte);
3104
unlock:
J
Jan Kara 已提交
3105
	pte_unmap_unlock(vmf->pte, vmf->ptl);
L
Linus Torvalds 已提交
3106 3107
out:
	return ret;
3108
out_nomap:
3109
	mem_cgroup_cancel_charge(page, memcg, false);
J
Jan Kara 已提交
3110
	pte_unmap_unlock(vmf->pte, vmf->ptl);
3111
out_page:
3112
	unlock_page(page);
3113
out_release:
3114
	put_page(page);
3115
	if (page != swapcache && swapcache) {
A
Andrea Arcangeli 已提交
3116
		unlock_page(swapcache);
3117
		put_page(swapcache);
A
Andrea Arcangeli 已提交
3118
	}
3119
	return ret;
L
Linus Torvalds 已提交
3120 3121 3122
}

/*
3123 3124 3125
 * We enter with non-exclusive mmap_sem (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
 * We return with mmap_sem still held, but pte unmapped and unlocked.
L
Linus Torvalds 已提交
3126
 */
J
Jan Kara 已提交
3127
static int do_anonymous_page(struct vm_fault *vmf)
L
Linus Torvalds 已提交
3128
{
J
Jan Kara 已提交
3129
	struct vm_area_struct *vma = vmf->vma;
3130
	struct mem_cgroup *memcg;
3131
	struct page *page;
3132
	int ret = 0;
L
Linus Torvalds 已提交
3133 3134
	pte_t entry;

3135 3136 3137 3138
	/* File mapping without ->vm_ops ? */
	if (vma->vm_flags & VM_SHARED)
		return VM_FAULT_SIGBUS;

3139 3140 3141 3142 3143 3144 3145 3146 3147 3148
	/*
	 * Use pte_alloc() instead of pte_alloc_map().  We can't run
	 * pte_offset_map() on pmds where a huge pmd might be created
	 * from a different thread.
	 *
	 * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
	 * parallel threads are excluded by other means.
	 *
	 * Here we only have down_read(mmap_sem).
	 */
J
Jan Kara 已提交
3149
	if (pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))
3150 3151 3152
		return VM_FAULT_OOM;

	/* See the comment in pte_alloc_one_map() */
J
Jan Kara 已提交
3153
	if (unlikely(pmd_trans_unstable(vmf->pmd)))
3154 3155
		return 0;

3156
	/* Use the zero-page for reads */
J
Jan Kara 已提交
3157
	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
K
Kirill A. Shutemov 已提交
3158
			!mm_forbids_zeropage(vma->vm_mm)) {
J
Jan Kara 已提交
3159
		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
H
Hugh Dickins 已提交
3160
						vma->vm_page_prot));
J
Jan Kara 已提交
3161 3162 3163
		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
				vmf->address, &vmf->ptl);
		if (!pte_none(*vmf->pte))
H
Hugh Dickins 已提交
3164
			goto unlock;
3165 3166 3167
		ret = check_stable_address_space(vma->vm_mm);
		if (ret)
			goto unlock;
3168 3169
		/* Deliver the page fault to userland, check inside PT lock */
		if (userfaultfd_missing(vma)) {
J
Jan Kara 已提交
3170 3171
			pte_unmap_unlock(vmf->pte, vmf->ptl);
			return handle_userfault(vmf, VM_UFFD_MISSING);
3172
		}
H
Hugh Dickins 已提交
3173 3174 3175
		goto setpte;
	}

N
Nick Piggin 已提交
3176 3177 3178
	/* Allocate our own private page. */
	if (unlikely(anon_vma_prepare(vma)))
		goto oom;
J
Jan Kara 已提交
3179
	page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
N
Nick Piggin 已提交
3180 3181
	if (!page)
		goto oom;
3182

3183 3184
	if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, &memcg,
					false))
3185 3186
		goto oom_free_page;

3187 3188 3189 3190 3191
	/*
	 * The memory barrier inside __SetPageUptodate makes sure that
	 * preceeding stores to the page contents become visible before
	 * the set_pte_at() write.
	 */
N
Nick Piggin 已提交
3192
	__SetPageUptodate(page);
3193

N
Nick Piggin 已提交
3194
	entry = mk_pte(page, vma->vm_page_prot);
H
Hugh Dickins 已提交
3195 3196
	if (vma->vm_flags & VM_WRITE)
		entry = pte_mkwrite(pte_mkdirty(entry));
L
Linus Torvalds 已提交
3197

J
Jan Kara 已提交
3198 3199 3200
	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
			&vmf->ptl);
	if (!pte_none(*vmf->pte))
N
Nick Piggin 已提交
3201
		goto release;
H
Hugh Dickins 已提交
3202

3203 3204 3205 3206
	ret = check_stable_address_space(vma->vm_mm);
	if (ret)
		goto release;

3207 3208
	/* Deliver the page fault to userland, check inside PT lock */
	if (userfaultfd_missing(vma)) {
J
Jan Kara 已提交
3209
		pte_unmap_unlock(vmf->pte, vmf->ptl);
3210
		mem_cgroup_cancel_charge(page, memcg, false);
3211
		put_page(page);
J
Jan Kara 已提交
3212
		return handle_userfault(vmf, VM_UFFD_MISSING);
3213 3214
	}

K
Kirill A. Shutemov 已提交
3215
	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
J
Jan Kara 已提交
3216
	page_add_new_anon_rmap(page, vma, vmf->address, false);
3217
	mem_cgroup_commit_charge(page, memcg, false, false);
3218
	lru_cache_add_active_or_unevictable(page, vma);
H
Hugh Dickins 已提交
3219
setpte:
J
Jan Kara 已提交
3220
	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
L
Linus Torvalds 已提交
3221 3222

	/* No need to invalidate - it was non-present before */
J
Jan Kara 已提交
3223
	update_mmu_cache(vma, vmf->address, vmf->pte);
3224
unlock:
J
Jan Kara 已提交
3225
	pte_unmap_unlock(vmf->pte, vmf->ptl);
3226
	return ret;
3227
release:
3228
	mem_cgroup_cancel_charge(page, memcg, false);
3229
	put_page(page);
3230
	goto unlock;
3231
oom_free_page:
3232
	put_page(page);
3233
oom:
L
Linus Torvalds 已提交
3234 3235 3236
	return VM_FAULT_OOM;
}

3237 3238 3239 3240 3241
/*
 * The mmap_sem must have been held on entry, and may have been
 * released depending on flags and vma->vm_ops->fault() return value.
 * See filemap_fault() and __lock_page_retry().
 */
J
Jan Kara 已提交
3242
static int __do_fault(struct vm_fault *vmf)
3243
{
J
Jan Kara 已提交
3244
	struct vm_area_struct *vma = vmf->vma;
3245 3246
	int ret;

3247
	ret = vma->vm_ops->fault(vmf);
3248
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
3249
			    VM_FAULT_DONE_COW)))
3250
		return ret;
3251

3252
	if (unlikely(PageHWPoison(vmf->page))) {
3253
		if (ret & VM_FAULT_LOCKED)
3254 3255
			unlock_page(vmf->page);
		put_page(vmf->page);
J
Jan Kara 已提交
3256
		vmf->page = NULL;
3257 3258 3259 3260
		return VM_FAULT_HWPOISON;
	}

	if (unlikely(!(ret & VM_FAULT_LOCKED)))
3261
		lock_page(vmf->page);
3262
	else
3263
		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
3264 3265 3266 3267

	return ret;
}

3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
/*
 * The ordering of these checks is important for pmds with _PAGE_DEVMAP set.
 * If we check pmd_trans_unstable() first we will trip the bad_pmd() check
 * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly
 * returning 1 but not before it spams dmesg with the pmd_clear_bad() output.
 */
static int pmd_devmap_trans_unstable(pmd_t *pmd)
{
	return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
}

J
Jan Kara 已提交
3279
static int pte_alloc_one_map(struct vm_fault *vmf)
3280
{
J
Jan Kara 已提交
3281
	struct vm_area_struct *vma = vmf->vma;
3282

J
Jan Kara 已提交
3283
	if (!pmd_none(*vmf->pmd))
3284
		goto map_pte;
J
Jan Kara 已提交
3285 3286 3287 3288
	if (vmf->prealloc_pte) {
		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
		if (unlikely(!pmd_none(*vmf->pmd))) {
			spin_unlock(vmf->ptl);
3289 3290 3291
			goto map_pte;
		}

3292
		mm_inc_nr_ptes(vma->vm_mm);
J
Jan Kara 已提交
3293 3294
		pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
		spin_unlock(vmf->ptl);
3295
		vmf->prealloc_pte = NULL;
J
Jan Kara 已提交
3296
	} else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))) {
3297 3298 3299 3300 3301
		return VM_FAULT_OOM;
	}
map_pte:
	/*
	 * If a huge pmd materialized under us just retry later.  Use
3302 3303 3304 3305 3306 3307 3308 3309
	 * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of
	 * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge
	 * under us and then back to pmd_none, as a result of MADV_DONTNEED
	 * running immediately after a huge pmd fault in a different thread of
	 * this mm, in turn leading to a misleading pmd_trans_huge() retval.
	 * All we have to ensure is that it is a regular pmd that we can walk
	 * with pte_offset_map() and we can do that through an atomic read in
	 * C, which is what pmd_trans_unstable() provides.
3310
	 */
3311
	if (pmd_devmap_trans_unstable(vmf->pmd))
3312 3313
		return VM_FAULT_NOPAGE;

3314 3315 3316 3317 3318 3319 3320 3321 3322
	/*
	 * At this point we know that our vmf->pmd points to a page of ptes
	 * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge()
	 * for the duration of the fault.  If a racing MADV_DONTNEED runs and
	 * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still
	 * be valid and we will re-check to make sure the vmf->pte isn't
	 * pte_none() under vmf->ptl protection when we return to
	 * alloc_set_pte().
	 */
J
Jan Kara 已提交
3323 3324
	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
			&vmf->ptl);
3325 3326 3327
	return 0;
}

3328
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
K
Kirill A. Shutemov 已提交
3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341

#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
		unsigned long haddr)
{
	if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=
			(vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))
		return false;
	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
		return false;
	return true;
}

J
Jan Kara 已提交
3342
static void deposit_prealloc_pte(struct vm_fault *vmf)
3343
{
J
Jan Kara 已提交
3344
	struct vm_area_struct *vma = vmf->vma;
3345

J
Jan Kara 已提交
3346
	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
3347 3348 3349 3350
	/*
	 * We are going to consume the prealloc table,
	 * count that as nr_ptes.
	 */
3351
	mm_inc_nr_ptes(vma->vm_mm);
3352
	vmf->prealloc_pte = NULL;
3353 3354
}

J
Jan Kara 已提交
3355
static int do_set_pmd(struct vm_fault *vmf, struct page *page)
K
Kirill A. Shutemov 已提交
3356
{
J
Jan Kara 已提交
3357 3358 3359
	struct vm_area_struct *vma = vmf->vma;
	bool write = vmf->flags & FAULT_FLAG_WRITE;
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
K
Kirill A. Shutemov 已提交
3360 3361 3362 3363 3364 3365 3366 3367 3368
	pmd_t entry;
	int i, ret;

	if (!transhuge_vma_suitable(vma, haddr))
		return VM_FAULT_FALLBACK;

	ret = VM_FAULT_FALLBACK;
	page = compound_head(page);

3369 3370 3371 3372
	/*
	 * Archs like ppc64 need additonal space to store information
	 * related to pte entry. Use the preallocated table for that.
	 */
J
Jan Kara 已提交
3373 3374 3375
	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm, vmf->address);
		if (!vmf->prealloc_pte)
3376 3377 3378 3379
			return VM_FAULT_OOM;
		smp_wmb(); /* See comment in __pte_alloc() */
	}

J
Jan Kara 已提交
3380 3381
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_none(*vmf->pmd)))
K
Kirill A. Shutemov 已提交
3382 3383 3384 3385 3386 3387 3388
		goto out;

	for (i = 0; i < HPAGE_PMD_NR; i++)
		flush_icache_page(vma, page + i);

	entry = mk_huge_pmd(page, vma->vm_page_prot);
	if (write)
3389
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
K
Kirill A. Shutemov 已提交
3390

3391
	add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
K
Kirill A. Shutemov 已提交
3392
	page_add_file_rmap(page, true);
3393 3394 3395 3396
	/*
	 * deposit and withdraw with pmd lock held
	 */
	if (arch_needs_pgtable_deposit())
J
Jan Kara 已提交
3397
		deposit_prealloc_pte(vmf);
K
Kirill A. Shutemov 已提交
3398

J
Jan Kara 已提交
3399
	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
K
Kirill A. Shutemov 已提交
3400

J
Jan Kara 已提交
3401
	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
K
Kirill A. Shutemov 已提交
3402 3403 3404

	/* fault is handled */
	ret = 0;
3405
	count_vm_event(THP_FILE_MAPPED);
K
Kirill A. Shutemov 已提交
3406
out:
J
Jan Kara 已提交
3407
	spin_unlock(vmf->ptl);
K
Kirill A. Shutemov 已提交
3408 3409 3410
	return ret;
}
#else
J
Jan Kara 已提交
3411
static int do_set_pmd(struct vm_fault *vmf, struct page *page)
K
Kirill A. Shutemov 已提交
3412 3413 3414 3415 3416 3417
{
	BUILD_BUG();
	return 0;
}
#endif

3418
/**
3419 3420
 * alloc_set_pte - setup new PTE entry for given page and add reverse page
 * mapping. If needed, the fucntion allocates page table or use pre-allocated.
3421
 *
J
Jan Kara 已提交
3422
 * @vmf: fault environment
3423
 * @memcg: memcg to charge page (only for private mappings)
3424 3425
 * @page: page to map
 *
J
Jan Kara 已提交
3426 3427
 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
 * return.
3428 3429 3430 3431
 *
 * Target users are page handler itself and implementations of
 * vm_ops->map_pages.
 */
J
Jan Kara 已提交
3432
int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
3433
		struct page *page)
3434
{
J
Jan Kara 已提交
3435 3436
	struct vm_area_struct *vma = vmf->vma;
	bool write = vmf->flags & FAULT_FLAG_WRITE;
3437
	pte_t entry;
K
Kirill A. Shutemov 已提交
3438 3439
	int ret;

J
Jan Kara 已提交
3440
	if (pmd_none(*vmf->pmd) && PageTransCompound(page) &&
3441
			IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
K
Kirill A. Shutemov 已提交
3442 3443 3444
		/* THP on COW? */
		VM_BUG_ON_PAGE(memcg, page);

J
Jan Kara 已提交
3445
		ret = do_set_pmd(vmf, page);
K
Kirill A. Shutemov 已提交
3446
		if (ret != VM_FAULT_FALLBACK)
H
Hugh Dickins 已提交
3447
			return ret;
K
Kirill A. Shutemov 已提交
3448
	}
3449

J
Jan Kara 已提交
3450 3451
	if (!vmf->pte) {
		ret = pte_alloc_one_map(vmf);
3452
		if (ret)
H
Hugh Dickins 已提交
3453
			return ret;
3454 3455 3456
	}

	/* Re-check under ptl */
H
Hugh Dickins 已提交
3457 3458
	if (unlikely(!pte_none(*vmf->pte)))
		return VM_FAULT_NOPAGE;
3459

3460 3461 3462 3463
	flush_icache_page(vma, page);
	entry = mk_pte(page, vma->vm_page_prot);
	if (write)
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
K
Kirill A. Shutemov 已提交
3464 3465
	/* copy-on-write page */
	if (write && !(vma->vm_flags & VM_SHARED)) {
3466
		inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
J
Jan Kara 已提交
3467
		page_add_new_anon_rmap(page, vma, vmf->address, false);
3468 3469
		mem_cgroup_commit_charge(page, memcg, false, false);
		lru_cache_add_active_or_unevictable(page, vma);
3470
	} else {
3471
		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
K
Kirill A. Shutemov 已提交
3472
		page_add_file_rmap(page, false);
3473
	}
J
Jan Kara 已提交
3474
	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
3475 3476

	/* no need to invalidate: a not-present page won't be cached */
J
Jan Kara 已提交
3477
	update_mmu_cache(vma, vmf->address, vmf->pte);
3478

H
Hugh Dickins 已提交
3479
	return 0;
3480 3481
}

3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499

/**
 * finish_fault - finish page fault once we have prepared the page to fault
 *
 * @vmf: structure describing the fault
 *
 * This function handles all that is needed to finish a page fault once the
 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
 * given page, adds reverse page mapping, handles memcg charges and LRU
 * addition. The function returns 0 on success, VM_FAULT_ code in case of
 * error.
 *
 * The function expects the page to be locked and on success it consumes a
 * reference of a page being mapped (for the PTE which maps it).
 */
int finish_fault(struct vm_fault *vmf)
{
	struct page *page;
3500
	int ret = 0;
3501 3502 3503 3504 3505 3506 3507

	/* Did we COW the page? */
	if ((vmf->flags & FAULT_FLAG_WRITE) &&
	    !(vmf->vma->vm_flags & VM_SHARED))
		page = vmf->cow_page;
	else
		page = vmf->page;
3508 3509 3510 3511 3512 3513 3514 3515 3516

	/*
	 * check even for read faults because we might have lost our CoWed
	 * page
	 */
	if (!(vmf->vma->vm_flags & VM_SHARED))
		ret = check_stable_address_space(vmf->vma->vm_mm);
	if (!ret)
		ret = alloc_set_pte(vmf, vmf->memcg, page);
3517 3518 3519 3520 3521
	if (vmf->pte)
		pte_unmap_unlock(vmf->pte, vmf->ptl);
	return ret;
}

3522 3523
static unsigned long fault_around_bytes __read_mostly =
	rounddown_pow_of_two(65536);
3524 3525 3526

#ifdef CONFIG_DEBUG_FS
static int fault_around_bytes_get(void *data, u64 *val)
3527
{
3528
	*val = fault_around_bytes;
3529 3530 3531
	return 0;
}

3532
/*
3533 3534
 * fault_around_bytes must be rounded down to the nearest page order as it's
 * what do_fault_around() expects to see.
3535
 */
3536
static int fault_around_bytes_set(void *data, u64 val)
3537
{
3538
	if (val / PAGE_SIZE > PTRS_PER_PTE)
3539
		return -EINVAL;
3540 3541 3542 3543
	if (val > PAGE_SIZE)
		fault_around_bytes = rounddown_pow_of_two(val);
	else
		fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
3544 3545
	return 0;
}
3546
DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
3547
		fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
3548 3549 3550 3551 3552

static int __init fault_around_debugfs(void)
{
	void *ret;

3553
	ret = debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
3554
			&fault_around_bytes_fops);
3555
	if (!ret)
3556
		pr_warn("Failed to create fault_around_bytes in debugfs");
3557 3558 3559 3560
	return 0;
}
late_initcall(fault_around_debugfs);
#endif
3561

3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576
/*
 * do_fault_around() tries to map few pages around the fault address. The hope
 * is that the pages will be needed soon and this will lower the number of
 * faults to handle.
 *
 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
 * not ready to be mapped: not up-to-date, locked, etc.
 *
 * This function is called with the page table lock taken. In the split ptlock
 * case the page table lock only protects only those entries which belong to
 * the page table corresponding to the fault address.
 *
 * This function doesn't cross the VMA boundaries, in order to call map_pages()
 * only once.
 *
3577 3578 3579
 * fault_around_bytes defines how many bytes we'll try to map.
 * do_fault_around() expects it to be set to a power of two less than or equal
 * to PTRS_PER_PTE.
3580
 *
3581 3582 3583 3584
 * The virtual address of the area that we map is naturally aligned to
 * fault_around_bytes rounded down to the machine page size
 * (and therefore to page order).  This way it's easier to guarantee
 * that we don't cross page table boundaries.
3585
 */
3586
static int do_fault_around(struct vm_fault *vmf)
3587
{
J
Jan Kara 已提交
3588
	unsigned long address = vmf->address, nr_pages, mask;
3589
	pgoff_t start_pgoff = vmf->pgoff;
K
Kirill A. Shutemov 已提交
3590
	pgoff_t end_pgoff;
3591
	int off, ret = 0;
3592

3593
	nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
3594 3595
	mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;

J
Jan Kara 已提交
3596 3597
	vmf->address = max(address & mask, vmf->vma->vm_start);
	off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
K
Kirill A. Shutemov 已提交
3598
	start_pgoff -= off;
3599 3600

	/*
3601 3602
	 *  end_pgoff is either the end of the page table, the end of
	 *  the vma or nr_pages from start_pgoff, depending what is nearest.
3603
	 */
K
Kirill A. Shutemov 已提交
3604
	end_pgoff = start_pgoff -
J
Jan Kara 已提交
3605
		((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
3606
		PTRS_PER_PTE - 1;
J
Jan Kara 已提交
3607
	end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
K
Kirill A. Shutemov 已提交
3608
			start_pgoff + nr_pages - 1);
3609

J
Jan Kara 已提交
3610 3611 3612 3613
	if (pmd_none(*vmf->pmd)) {
		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
						  vmf->address);
		if (!vmf->prealloc_pte)
3614
			goto out;
3615
		smp_wmb(); /* See comment in __pte_alloc() */
3616 3617
	}

J
Jan Kara 已提交
3618
	vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
3619 3620

	/* Huge page is mapped? Page fault is solved */
J
Jan Kara 已提交
3621
	if (pmd_trans_huge(*vmf->pmd)) {
3622 3623 3624 3625 3626
		ret = VM_FAULT_NOPAGE;
		goto out;
	}

	/* ->map_pages() haven't done anything useful. Cold page cache? */
J
Jan Kara 已提交
3627
	if (!vmf->pte)
3628 3629 3630
		goto out;

	/* check if the page fault is solved */
J
Jan Kara 已提交
3631 3632
	vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
	if (!pte_none(*vmf->pte))
3633
		ret = VM_FAULT_NOPAGE;
J
Jan Kara 已提交
3634
	pte_unmap_unlock(vmf->pte, vmf->ptl);
K
Kirill A. Shutemov 已提交
3635
out:
J
Jan Kara 已提交
3636 3637
	vmf->address = address;
	vmf->pte = NULL;
3638
	return ret;
3639 3640
}

3641
static int do_read_fault(struct vm_fault *vmf)
3642
{
J
Jan Kara 已提交
3643
	struct vm_area_struct *vma = vmf->vma;
3644 3645 3646 3647 3648 3649 3650
	int ret = 0;

	/*
	 * Let's call ->map_pages() first and use ->fault() as fallback
	 * if page by the offset is not ready to be mapped (cold cache or
	 * something).
	 */
3651
	if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
3652
		ret = do_fault_around(vmf);
3653 3654
		if (ret)
			return ret;
3655
	}
3656

J
Jan Kara 已提交
3657
	ret = __do_fault(vmf);
3658 3659 3660
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
		return ret;

3661
	ret |= finish_fault(vmf);
J
Jan Kara 已提交
3662
	unlock_page(vmf->page);
3663
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
J
Jan Kara 已提交
3664
		put_page(vmf->page);
3665 3666 3667
	return ret;
}

3668
static int do_cow_fault(struct vm_fault *vmf)
3669
{
J
Jan Kara 已提交
3670
	struct vm_area_struct *vma = vmf->vma;
3671 3672 3673 3674 3675
	int ret;

	if (unlikely(anon_vma_prepare(vma)))
		return VM_FAULT_OOM;

J
Jan Kara 已提交
3676 3677
	vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
	if (!vmf->cow_page)
3678 3679
		return VM_FAULT_OOM;

3680
	if (mem_cgroup_try_charge_delay(vmf->cow_page, vma->vm_mm, GFP_KERNEL,
3681
				&vmf->memcg, false)) {
J
Jan Kara 已提交
3682
		put_page(vmf->cow_page);
3683 3684 3685
		return VM_FAULT_OOM;
	}

J
Jan Kara 已提交
3686
	ret = __do_fault(vmf);
3687 3688
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
		goto uncharge_out;
3689 3690
	if (ret & VM_FAULT_DONE_COW)
		return ret;
3691

3692
	copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
J
Jan Kara 已提交
3693
	__SetPageUptodate(vmf->cow_page);
3694

3695
	ret |= finish_fault(vmf);
3696 3697
	unlock_page(vmf->page);
	put_page(vmf->page);
3698 3699
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
		goto uncharge_out;
3700 3701
	return ret;
uncharge_out:
3702
	mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false);
J
Jan Kara 已提交
3703
	put_page(vmf->cow_page);
3704 3705 3706
	return ret;
}

3707
static int do_shared_fault(struct vm_fault *vmf)
L
Linus Torvalds 已提交
3708
{
J
Jan Kara 已提交
3709
	struct vm_area_struct *vma = vmf->vma;
3710
	int ret, tmp;
3711

J
Jan Kara 已提交
3712
	ret = __do_fault(vmf);
3713
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3714
		return ret;
L
Linus Torvalds 已提交
3715 3716

	/*
3717 3718
	 * Check if the backing address space wants to know that the page is
	 * about to become writable
L
Linus Torvalds 已提交
3719
	 */
3720
	if (vma->vm_ops->page_mkwrite) {
J
Jan Kara 已提交
3721
		unlock_page(vmf->page);
3722
		tmp = do_page_mkwrite(vmf);
3723 3724
		if (unlikely(!tmp ||
				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
J
Jan Kara 已提交
3725
			put_page(vmf->page);
3726
			return tmp;
3727
		}
3728 3729
	}

3730
	ret |= finish_fault(vmf);
3731 3732
	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
					VM_FAULT_RETRY))) {
J
Jan Kara 已提交
3733 3734
		unlock_page(vmf->page);
		put_page(vmf->page);
3735
		return ret;
L
Linus Torvalds 已提交
3736
	}
N
Nick Piggin 已提交
3737

3738
	fault_dirty_shared_page(vma, vmf->page);
3739
	return ret;
3740
}
3741

3742 3743 3744 3745 3746 3747
/*
 * We enter with non-exclusive mmap_sem (to exclude vma changes,
 * but allow concurrent faults).
 * The mmap_sem may have been released depending on flags and our
 * return value.  See filemap_fault() and __lock_page_or_retry().
 */
J
Jan Kara 已提交
3748
static int do_fault(struct vm_fault *vmf)
3749
{
J
Jan Kara 已提交
3750
	struct vm_area_struct *vma = vmf->vma;
H
Hugh Dickins 已提交
3751
	int ret;
3752

3753 3754
	/* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
	if (!vma->vm_ops->fault)
H
Hugh Dickins 已提交
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765
		ret = VM_FAULT_SIGBUS;
	else if (!(vmf->flags & FAULT_FLAG_WRITE))
		ret = do_read_fault(vmf);
	else if (!(vma->vm_flags & VM_SHARED))
		ret = do_cow_fault(vmf);
	else
		ret = do_shared_fault(vmf);

	/* preallocated pagetable is unused: free it */
	if (vmf->prealloc_pte) {
		pte_free(vma->vm_mm, vmf->prealloc_pte);
3766
		vmf->prealloc_pte = NULL;
H
Hugh Dickins 已提交
3767 3768
	}
	return ret;
3769 3770
}

3771
static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
3772 3773
				unsigned long addr, int page_nid,
				int *flags)
3774 3775 3776 3777
{
	get_page(page);

	count_vm_numa_event(NUMA_HINT_FAULTS);
3778
	if (page_nid == numa_node_id()) {
3779
		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
3780 3781
		*flags |= TNF_FAULT_LOCAL;
	}
3782 3783 3784 3785

	return mpol_misplaced(page, vma, addr);
}

J
Jan Kara 已提交
3786
static int do_numa_page(struct vm_fault *vmf)
3787
{
J
Jan Kara 已提交
3788
	struct vm_area_struct *vma = vmf->vma;
3789
	struct page *page = NULL;
3790
	int page_nid = -1;
3791
	int last_cpupid;
3792
	int target_nid;
3793
	bool migrated = false;
3794
	pte_t pte;
3795
	bool was_writable = pte_savedwrite(vmf->orig_pte);
3796
	int flags = 0;
3797 3798

	/*
T
Tobin C Harding 已提交
3799 3800 3801 3802
	 * The "pte" at this point cannot be used safely without
	 * validation through pte_unmap_same(). It's of NUMA type but
	 * the pfn may be screwed if the read is non atomic.
	 */
J
Jan Kara 已提交
3803 3804
	vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
	spin_lock(vmf->ptl);
3805
	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
J
Jan Kara 已提交
3806
		pte_unmap_unlock(vmf->pte, vmf->ptl);
3807 3808 3809
		goto out;
	}

3810 3811 3812 3813 3814
	/*
	 * Make it present again, Depending on how arch implementes non
	 * accessible ptes, some can allow access by kernel mode.
	 */
	pte = ptep_modify_prot_start(vma->vm_mm, vmf->address, vmf->pte);
3815 3816
	pte = pte_modify(pte, vma->vm_page_prot);
	pte = pte_mkyoung(pte);
3817 3818
	if (was_writable)
		pte = pte_mkwrite(pte);
3819
	ptep_modify_prot_commit(vma->vm_mm, vmf->address, vmf->pte, pte);
J
Jan Kara 已提交
3820
	update_mmu_cache(vma, vmf->address, vmf->pte);
3821

J
Jan Kara 已提交
3822
	page = vm_normal_page(vma, vmf->address, pte);
3823
	if (!page) {
J
Jan Kara 已提交
3824
		pte_unmap_unlock(vmf->pte, vmf->ptl);
3825 3826 3827
		return 0;
	}

3828 3829
	/* TODO: handle PTE-mapped THP */
	if (PageCompound(page)) {
J
Jan Kara 已提交
3830
		pte_unmap_unlock(vmf->pte, vmf->ptl);
3831 3832 3833
		return 0;
	}

3834
	/*
3835 3836 3837 3838 3839 3840
	 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
	 * much anyway since they can be in shared cache state. This misses
	 * the case where a mapping is writable but the process never writes
	 * to it but pte_write gets cleared during protection updates and
	 * pte_dirty has unpredictable behaviour between PTE scan updates,
	 * background writeback, dirty balancing and application behaviour.
3841
	 */
3842
	if (!pte_write(pte))
3843 3844
		flags |= TNF_NO_GROUP;

3845 3846 3847 3848 3849 3850 3851
	/*
	 * Flag if the page is shared between multiple address spaces. This
	 * is later used when determining whether to group tasks together
	 */
	if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
		flags |= TNF_SHARED;

3852
	last_cpupid = page_cpupid_last(page);
3853
	page_nid = page_to_nid(page);
J
Jan Kara 已提交
3854
	target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
K
Kirill A. Shutemov 已提交
3855
			&flags);
J
Jan Kara 已提交
3856
	pte_unmap_unlock(vmf->pte, vmf->ptl);
3857 3858 3859 3860 3861 3862
	if (target_nid == -1) {
		put_page(page);
		goto out;
	}

	/* Migrate to the requested node */
3863
	migrated = migrate_misplaced_page(page, vma, target_nid);
3864
	if (migrated) {
3865
		page_nid = target_nid;
3866
		flags |= TNF_MIGRATED;
3867 3868
	} else
		flags |= TNF_MIGRATE_FAIL;
3869 3870

out:
3871
	if (page_nid != -1)
3872
		task_numa_fault(last_cpupid, page_nid, 1, flags);
3873 3874 3875
	return 0;
}

3876
static inline int create_huge_pmd(struct vm_fault *vmf)
M
Matthew Wilcox 已提交
3877
{
3878
	if (vma_is_anonymous(vmf->vma))
J
Jan Kara 已提交
3879
		return do_huge_pmd_anonymous_page(vmf);
3880
	if (vmf->vma->vm_ops->huge_fault)
3881
		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
M
Matthew Wilcox 已提交
3882 3883 3884
	return VM_FAULT_FALLBACK;
}

3885 3886
/* `inline' is required to avoid gcc 4.1.2 build error */
static inline int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
M
Matthew Wilcox 已提交
3887
{
J
Jan Kara 已提交
3888 3889
	if (vma_is_anonymous(vmf->vma))
		return do_huge_pmd_wp_page(vmf, orig_pmd);
3890
	if (vmf->vma->vm_ops->huge_fault)
3891
		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
K
Kirill A. Shutemov 已提交
3892 3893

	/* COW handled on pte level: split pmd */
J
Jan Kara 已提交
3894 3895
	VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
	__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
K
Kirill A. Shutemov 已提交
3896

M
Matthew Wilcox 已提交
3897 3898 3899
	return VM_FAULT_FALLBACK;
}

3900 3901 3902 3903 3904
static inline bool vma_is_accessible(struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE);
}

3905 3906 3907 3908 3909 3910 3911
static int create_huge_pud(struct vm_fault *vmf)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	/* No support for anonymous transparent PUD pages yet */
	if (vma_is_anonymous(vmf->vma))
		return VM_FAULT_FALLBACK;
	if (vmf->vma->vm_ops->huge_fault)
3912
		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	return VM_FAULT_FALLBACK;
}

static int wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	/* No support for anonymous transparent PUD pages yet */
	if (vma_is_anonymous(vmf->vma))
		return VM_FAULT_FALLBACK;
	if (vmf->vma->vm_ops->huge_fault)
3924
		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
3925 3926 3927 3928
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	return VM_FAULT_FALLBACK;
}

L
Linus Torvalds 已提交
3929 3930 3931 3932 3933 3934 3935 3936 3937
/*
 * These routines also need to handle stuff like marking pages dirty
 * and/or accessed for architectures that don't do it in hardware (most
 * RISC architectures).  The early dirtying is also good on the i386.
 *
 * There is also a hook called "update_mmu_cache()" that architectures
 * with external mmu caches can use to update those (ie the Sparc or
 * PowerPC hashed page tables that act as extended TLBs).
 *
3938 3939
 * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow
 * concurrent faults).
3940
 *
3941 3942
 * The mmap_sem may have been released depending on flags and our return value.
 * See filemap_fault() and __lock_page_or_retry().
L
Linus Torvalds 已提交
3943
 */
J
Jan Kara 已提交
3944
static int handle_pte_fault(struct vm_fault *vmf)
L
Linus Torvalds 已提交
3945 3946 3947
{
	pte_t entry;

J
Jan Kara 已提交
3948
	if (unlikely(pmd_none(*vmf->pmd))) {
3949 3950 3951 3952 3953 3954
		/*
		 * Leave __pte_alloc() until later: because vm_ops->fault may
		 * want to allocate huge page, and if we expose page table
		 * for an instant, it will be difficult to retract from
		 * concurrent faults and from rmap lookups.
		 */
J
Jan Kara 已提交
3955
		vmf->pte = NULL;
3956 3957
	} else {
		/* See comment in pte_alloc_one_map() */
3958
		if (pmd_devmap_trans_unstable(vmf->pmd))
3959 3960 3961 3962 3963 3964 3965
			return 0;
		/*
		 * A regular pmd is established and it can't morph into a huge
		 * pmd from under us anymore at this point because we hold the
		 * mmap_sem read mode and khugepaged takes it in write mode.
		 * So now it's safe to run pte_offset_map().
		 */
J
Jan Kara 已提交
3966
		vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
J
Jan Kara 已提交
3967
		vmf->orig_pte = *vmf->pte;
3968 3969 3970 3971

		/*
		 * some architectures can have larger ptes than wordsize,
		 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
3972 3973 3974
		 * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
		 * accesses.  The code below just needs a consistent view
		 * for the ifs and we later double check anyway with the
3975 3976 3977
		 * ptl lock held. So here a barrier will do.
		 */
		barrier();
J
Jan Kara 已提交
3978
		if (pte_none(vmf->orig_pte)) {
J
Jan Kara 已提交
3979 3980
			pte_unmap(vmf->pte);
			vmf->pte = NULL;
3981
		}
L
Linus Torvalds 已提交
3982 3983
	}

J
Jan Kara 已提交
3984 3985 3986
	if (!vmf->pte) {
		if (vma_is_anonymous(vmf->vma))
			return do_anonymous_page(vmf);
3987
		else
J
Jan Kara 已提交
3988
			return do_fault(vmf);
3989 3990
	}

J
Jan Kara 已提交
3991 3992
	if (!pte_present(vmf->orig_pte))
		return do_swap_page(vmf);
3993

J
Jan Kara 已提交
3994 3995
	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
		return do_numa_page(vmf);
3996

J
Jan Kara 已提交
3997 3998
	vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
	spin_lock(vmf->ptl);
J
Jan Kara 已提交
3999
	entry = vmf->orig_pte;
J
Jan Kara 已提交
4000
	if (unlikely(!pte_same(*vmf->pte, entry)))
4001
		goto unlock;
J
Jan Kara 已提交
4002
	if (vmf->flags & FAULT_FLAG_WRITE) {
4003
		if (!pte_write(entry))
J
Jan Kara 已提交
4004
			return do_wp_page(vmf);
L
Linus Torvalds 已提交
4005 4006 4007
		entry = pte_mkdirty(entry);
	}
	entry = pte_mkyoung(entry);
J
Jan Kara 已提交
4008 4009 4010
	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
				vmf->flags & FAULT_FLAG_WRITE)) {
		update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
4011 4012 4013 4014 4015 4016 4017
	} else {
		/*
		 * This is needed only for protection faults but the arch code
		 * is not yet telling us if this is a protection fault or not.
		 * This still avoids useless tlb flushes for .text page faults
		 * with threads.
		 */
J
Jan Kara 已提交
4018 4019
		if (vmf->flags & FAULT_FLAG_WRITE)
			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
4020
	}
4021
unlock:
J
Jan Kara 已提交
4022
	pte_unmap_unlock(vmf->pte, vmf->ptl);
N
Nick Piggin 已提交
4023
	return 0;
L
Linus Torvalds 已提交
4024 4025 4026 4027
}

/*
 * By the time we get here, we already hold the mm semaphore
4028 4029 4030
 *
 * The mmap_sem may have been released depending on flags and our
 * return value.  See filemap_fault() and __lock_page_or_retry().
L
Linus Torvalds 已提交
4031
 */
4032 4033
static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
		unsigned int flags)
L
Linus Torvalds 已提交
4034
{
J
Jan Kara 已提交
4035
	struct vm_fault vmf = {
K
Kirill A. Shutemov 已提交
4036
		.vma = vma,
4037
		.address = address & PAGE_MASK,
K
Kirill A. Shutemov 已提交
4038
		.flags = flags,
4039
		.pgoff = linear_page_index(vma, address),
4040
		.gfp_mask = __get_fault_gfp_mask(vma),
K
Kirill A. Shutemov 已提交
4041
	};
4042
	unsigned int dirty = flags & FAULT_FLAG_WRITE;
4043
	struct mm_struct *mm = vma->vm_mm;
L
Linus Torvalds 已提交
4044
	pgd_t *pgd;
4045
	p4d_t *p4d;
4046
	int ret;
L
Linus Torvalds 已提交
4047 4048

	pgd = pgd_offset(mm, address);
4049 4050 4051
	p4d = p4d_alloc(mm, pgd, address);
	if (!p4d)
		return VM_FAULT_OOM;
4052

4053
	vmf.pud = pud_alloc(mm, p4d, address);
4054
	if (!vmf.pud)
H
Hugh Dickins 已提交
4055
		return VM_FAULT_OOM;
4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067
	if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
		ret = create_huge_pud(&vmf);
		if (!(ret & VM_FAULT_FALLBACK))
			return ret;
	} else {
		pud_t orig_pud = *vmf.pud;

		barrier();
		if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {

			/* NUMA case for anonymous PUDs would go here */

4068
			if (dirty && !pud_write(orig_pud)) {
4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079
				ret = wp_huge_pud(&vmf, orig_pud);
				if (!(ret & VM_FAULT_FALLBACK))
					return ret;
			} else {
				huge_pud_set_accessed(&vmf, orig_pud);
				return 0;
			}
		}
	}

	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
J
Jan Kara 已提交
4080
	if (!vmf.pmd)
H
Hugh Dickins 已提交
4081
		return VM_FAULT_OOM;
J
Jan Kara 已提交
4082
	if (pmd_none(*vmf.pmd) && transparent_hugepage_enabled(vma)) {
4083
		ret = create_huge_pmd(&vmf);
4084 4085
		if (!(ret & VM_FAULT_FALLBACK))
			return ret;
4086
	} else {
J
Jan Kara 已提交
4087
		pmd_t orig_pmd = *vmf.pmd;
4088

4089
		barrier();
4090 4091 4092 4093 4094 4095 4096
		if (unlikely(is_swap_pmd(orig_pmd))) {
			VM_BUG_ON(thp_migration_supported() &&
					  !is_pmd_migration_entry(orig_pmd));
			if (is_pmd_migration_entry(orig_pmd))
				pmd_migration_entry_wait(mm, vmf.pmd);
			return 0;
		}
4097
		if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
4098
			if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
J
Jan Kara 已提交
4099
				return do_huge_pmd_numa_page(&vmf, orig_pmd);
4100

4101
			if (dirty && !pmd_write(orig_pmd)) {
J
Jan Kara 已提交
4102
				ret = wp_huge_pmd(&vmf, orig_pmd);
4103 4104
				if (!(ret & VM_FAULT_FALLBACK))
					return ret;
4105
			} else {
J
Jan Kara 已提交
4106
				huge_pmd_set_accessed(&vmf, orig_pmd);
4107
				return 0;
4108
			}
4109 4110 4111
		}
	}

J
Jan Kara 已提交
4112
	return handle_pte_fault(&vmf);
L
Linus Torvalds 已提交
4113 4114
}

4115 4116 4117 4118 4119 4120
/*
 * By the time we get here, we already hold the mm semaphore
 *
 * The mmap_sem may have been released depending on flags and our
 * return value.  See filemap_fault() and __lock_page_or_retry().
 */
4121 4122
int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
		unsigned int flags)
4123 4124 4125 4126 4127 4128
{
	int ret;

	__set_current_state(TASK_RUNNING);

	count_vm_event(PGFAULT);
4129
	count_memcg_event_mm(vma->vm_mm, PGFAULT);
4130 4131 4132 4133

	/* do counter updates before entering really critical section. */
	check_sync_rss_stat(current);

4134 4135 4136 4137 4138
	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
					    flags & FAULT_FLAG_INSTRUCTION,
					    flags & FAULT_FLAG_REMOTE))
		return VM_FAULT_SIGSEGV;

4139 4140 4141 4142 4143
	/*
	 * Enable the memcg OOM handling for faults triggered in user
	 * space.  Kernel faults are handled more gracefully.
	 */
	if (flags & FAULT_FLAG_USER)
4144
		mem_cgroup_enter_user_fault();
4145

K
Kirill A. Shutemov 已提交
4146 4147 4148 4149
	if (unlikely(is_vm_hugetlb_page(vma)))
		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
	else
		ret = __handle_mm_fault(vma, address, flags);
4150

4151
	if (flags & FAULT_FLAG_USER) {
4152
		mem_cgroup_exit_user_fault();
T
Tobin C Harding 已提交
4153 4154 4155 4156 4157 4158 4159 4160
		/*
		 * The task may have entered a memcg OOM situation but
		 * if the allocation error was handled gracefully (no
		 * VM_FAULT_OOM), there is no need to kill anything.
		 * Just clean up the OOM state peacefully.
		 */
		if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
			mem_cgroup_oom_synchronize(false);
4161
	}
4162

4163 4164
	return ret;
}
4165
EXPORT_SYMBOL_GPL(handle_mm_fault);
4166

K
Kirill A. Shutemov 已提交
4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189
#ifndef __PAGETABLE_P4D_FOLDED
/*
 * Allocate p4d page table.
 * We've already handled the fast-path in-line.
 */
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
{
	p4d_t *new = p4d_alloc_one(mm, address);
	if (!new)
		return -ENOMEM;

	smp_wmb(); /* See comment in __pte_alloc */

	spin_lock(&mm->page_table_lock);
	if (pgd_present(*pgd))		/* Another has populated it */
		p4d_free(mm, new);
	else
		pgd_populate(mm, pgd, new);
	spin_unlock(&mm->page_table_lock);
	return 0;
}
#endif /* __PAGETABLE_P4D_FOLDED */

L
Linus Torvalds 已提交
4190 4191 4192
#ifndef __PAGETABLE_PUD_FOLDED
/*
 * Allocate page upper directory.
H
Hugh Dickins 已提交
4193
 * We've already handled the fast-path in-line.
L
Linus Torvalds 已提交
4194
 */
4195
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
L
Linus Torvalds 已提交
4196
{
H
Hugh Dickins 已提交
4197 4198
	pud_t *new = pud_alloc_one(mm, address);
	if (!new)
4199
		return -ENOMEM;
L
Linus Torvalds 已提交
4200

4201 4202
	smp_wmb(); /* See comment in __pte_alloc */

H
Hugh Dickins 已提交
4203
	spin_lock(&mm->page_table_lock);
4204
#ifndef __ARCH_HAS_5LEVEL_HACK
K
Kirill A. Shutemov 已提交
4205 4206
	if (!p4d_present(*p4d)) {
		mm_inc_nr_puds(mm);
4207
		p4d_populate(mm, p4d, new);
K
Kirill A. Shutemov 已提交
4208
	} else	/* Another has populated it */
4209
		pud_free(mm, new);
K
Kirill A. Shutemov 已提交
4210 4211 4212
#else
	if (!pgd_present(*p4d)) {
		mm_inc_nr_puds(mm);
4213
		pgd_populate(mm, p4d, new);
K
Kirill A. Shutemov 已提交
4214 4215
	} else	/* Another has populated it */
		pud_free(mm, new);
4216
#endif /* __ARCH_HAS_5LEVEL_HACK */
H
Hugh Dickins 已提交
4217
	spin_unlock(&mm->page_table_lock);
4218
	return 0;
L
Linus Torvalds 已提交
4219 4220 4221 4222 4223 4224
}
#endif /* __PAGETABLE_PUD_FOLDED */

#ifndef __PAGETABLE_PMD_FOLDED
/*
 * Allocate page middle directory.
H
Hugh Dickins 已提交
4225
 * We've already handled the fast-path in-line.
L
Linus Torvalds 已提交
4226
 */
4227
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
L
Linus Torvalds 已提交
4228
{
4229
	spinlock_t *ptl;
H
Hugh Dickins 已提交
4230 4231
	pmd_t *new = pmd_alloc_one(mm, address);
	if (!new)
4232
		return -ENOMEM;
L
Linus Torvalds 已提交
4233

4234 4235
	smp_wmb(); /* See comment in __pte_alloc */

4236
	ptl = pud_lock(mm, pud);
L
Linus Torvalds 已提交
4237
#ifndef __ARCH_HAS_4LEVEL_HACK
4238 4239
	if (!pud_present(*pud)) {
		mm_inc_nr_pmds(mm);
4240
		pud_populate(mm, pud, new);
4241
	} else	/* Another has populated it */
4242
		pmd_free(mm, new);
4243 4244 4245
#else
	if (!pgd_present(*pud)) {
		mm_inc_nr_pmds(mm);
4246
		pgd_populate(mm, pud, new);
4247 4248
	} else /* Another has populated it */
		pmd_free(mm, new);
L
Linus Torvalds 已提交
4249
#endif /* __ARCH_HAS_4LEVEL_HACK */
4250
	spin_unlock(ptl);
4251
	return 0;
4252
}
L
Linus Torvalds 已提交
4253 4254
#endif /* __PAGETABLE_PMD_FOLDED */

R
Ross Zwisler 已提交
4255
static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
4256 4257
			    unsigned long *start, unsigned long *end,
			    pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
J
Johannes Weiner 已提交
4258 4259
{
	pgd_t *pgd;
4260
	p4d_t *p4d;
J
Johannes Weiner 已提交
4261 4262 4263 4264 4265 4266 4267 4268
	pud_t *pud;
	pmd_t *pmd;
	pte_t *ptep;

	pgd = pgd_offset(mm, address);
	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
		goto out;

4269 4270 4271 4272 4273
	p4d = p4d_offset(pgd, address);
	if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
		goto out;

	pud = pud_offset(p4d, address);
J
Johannes Weiner 已提交
4274 4275 4276 4277
	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
		goto out;

	pmd = pmd_offset(pud, address);
4278
	VM_BUG_ON(pmd_trans_huge(*pmd));
J
Johannes Weiner 已提交
4279

R
Ross Zwisler 已提交
4280 4281 4282 4283
	if (pmd_huge(*pmd)) {
		if (!pmdpp)
			goto out;

4284 4285 4286 4287 4288
		if (start && end) {
			*start = address & PMD_MASK;
			*end = *start + PMD_SIZE;
			mmu_notifier_invalidate_range_start(mm, *start, *end);
		}
R
Ross Zwisler 已提交
4289 4290 4291 4292 4293 4294
		*ptlp = pmd_lock(mm, pmd);
		if (pmd_huge(*pmd)) {
			*pmdpp = pmd;
			return 0;
		}
		spin_unlock(*ptlp);
4295 4296
		if (start && end)
			mmu_notifier_invalidate_range_end(mm, *start, *end);
R
Ross Zwisler 已提交
4297 4298 4299
	}

	if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
J
Johannes Weiner 已提交
4300 4301
		goto out;

4302 4303 4304 4305 4306
	if (start && end) {
		*start = address & PAGE_MASK;
		*end = *start + PAGE_SIZE;
		mmu_notifier_invalidate_range_start(mm, *start, *end);
	}
J
Johannes Weiner 已提交
4307 4308 4309 4310 4311 4312 4313
	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
	if (!pte_present(*ptep))
		goto unlock;
	*ptepp = ptep;
	return 0;
unlock:
	pte_unmap_unlock(ptep, *ptlp);
4314 4315
	if (start && end)
		mmu_notifier_invalidate_range_end(mm, *start, *end);
J
Johannes Weiner 已提交
4316 4317 4318 4319
out:
	return -EINVAL;
}

4320 4321
static inline int follow_pte(struct mm_struct *mm, unsigned long address,
			     pte_t **ptepp, spinlock_t **ptlp)
4322 4323 4324 4325 4326
{
	int res;

	/* (void) is needed to make gcc happy */
	(void) __cond_lock(*ptlp,
4327 4328
			   !(res = __follow_pte_pmd(mm, address, NULL, NULL,
						    ptepp, NULL, ptlp)));
R
Ross Zwisler 已提交
4329 4330 4331 4332
	return res;
}

int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
4333
			     unsigned long *start, unsigned long *end,
R
Ross Zwisler 已提交
4334 4335 4336 4337 4338 4339
			     pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
{
	int res;

	/* (void) is needed to make gcc happy */
	(void) __cond_lock(*ptlp,
4340 4341
			   !(res = __follow_pte_pmd(mm, address, start, end,
						    ptepp, pmdpp, ptlp)));
4342 4343
	return res;
}
R
Ross Zwisler 已提交
4344
EXPORT_SYMBOL(follow_pte_pmd);
4345

J
Johannes Weiner 已提交
4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374
/**
 * follow_pfn - look up PFN at a user virtual address
 * @vma: memory mapping
 * @address: user virtual address
 * @pfn: location to store found PFN
 *
 * Only IO mappings and raw PFN mappings are allowed.
 *
 * Returns zero and the pfn at @pfn on success, -ve otherwise.
 */
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
	unsigned long *pfn)
{
	int ret = -EINVAL;
	spinlock_t *ptl;
	pte_t *ptep;

	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
		return ret;

	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
	if (ret)
		return ret;
	*pfn = pte_pfn(*ptep);
	pte_unmap_unlock(ptep, ptl);
	return 0;
}
EXPORT_SYMBOL(follow_pfn);

4375
#ifdef CONFIG_HAVE_IOREMAP_PROT
4376 4377 4378
int follow_phys(struct vm_area_struct *vma,
		unsigned long address, unsigned int flags,
		unsigned long *prot, resource_size_t *phys)
4379
{
4380
	int ret = -EINVAL;
4381 4382 4383
	pte_t *ptep, pte;
	spinlock_t *ptl;

4384 4385
	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
		goto out;
4386

4387
	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
4388
		goto out;
4389
	pte = *ptep;
4390

4391
	if ((flags & FOLL_WRITE) && !pte_write(pte))
4392 4393 4394
		goto unlock;

	*prot = pgprot_val(pte_pgprot(pte));
4395
	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
4396

4397
	ret = 0;
4398 4399 4400
unlock:
	pte_unmap_unlock(ptep, ptl);
out:
4401
	return ret;
4402 4403 4404 4405 4406 4407 4408
}

int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
			void *buf, int len, int write)
{
	resource_size_t phys_addr;
	unsigned long prot = 0;
K
KOSAKI Motohiro 已提交
4409
	void __iomem *maddr;
4410 4411
	int offset = addr & (PAGE_SIZE-1);

4412
	if (follow_phys(vma, addr, write, &prot, &phys_addr))
4413 4414
		return -EINVAL;

4415
	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
4416 4417 4418
	if (!maddr)
		return -ENOMEM;

4419 4420 4421 4422 4423 4424 4425 4426
	if (write)
		memcpy_toio(maddr + offset, buf, len);
	else
		memcpy_fromio(buf, maddr + offset, len);
	iounmap(maddr);

	return len;
}
4427
EXPORT_SYMBOL_GPL(generic_access_phys);
4428 4429
#endif

4430
/*
4431 4432
 * Access another process' address space as given in mm.  If non-NULL, use the
 * given task for page fault accounting.
4433
 */
4434
int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
4435
		unsigned long addr, void *buf, int len, unsigned int gup_flags)
4436 4437 4438
{
	struct vm_area_struct *vma;
	void *old_buf = buf;
4439
	int write = gup_flags & FOLL_WRITE;
4440 4441

	down_read(&mm->mmap_sem);
S
Simon Arlott 已提交
4442
	/* ignore errors, just check how much was successfully transferred */
4443 4444 4445
	while (len) {
		int bytes, ret, offset;
		void *maddr;
4446
		struct page *page = NULL;
4447

4448
		ret = get_user_pages_remote(tsk, mm, addr, 1,
4449
				gup_flags, &page, &vma, NULL);
4450
		if (ret <= 0) {
4451 4452 4453
#ifndef CONFIG_HAVE_IOREMAP_PROT
			break;
#else
4454 4455 4456 4457 4458
			/*
			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
			 * we can access using slightly different code.
			 */
			vma = find_vma(mm, addr);
4459
			if (!vma || vma->vm_start > addr)
4460 4461 4462 4463 4464 4465 4466
				break;
			if (vma->vm_ops && vma->vm_ops->access)
				ret = vma->vm_ops->access(vma, addr, buf,
							  len, write);
			if (ret <= 0)
				break;
			bytes = ret;
4467
#endif
4468
		} else {
4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483
			bytes = len;
			offset = addr & (PAGE_SIZE-1);
			if (bytes > PAGE_SIZE-offset)
				bytes = PAGE_SIZE-offset;

			maddr = kmap(page);
			if (write) {
				copy_to_user_page(vma, page, addr,
						  maddr + offset, buf, bytes);
				set_page_dirty_lock(page);
			} else {
				copy_from_user_page(vma, page, addr,
						    buf, maddr + offset, bytes);
			}
			kunmap(page);
4484
			put_page(page);
4485 4486 4487 4488 4489 4490 4491 4492 4493
		}
		len -= bytes;
		buf += bytes;
		addr += bytes;
	}
	up_read(&mm->mmap_sem);

	return buf - old_buf;
}
4494

S
Stephen Wilson 已提交
4495
/**
4496
 * access_remote_vm - access another process' address space
S
Stephen Wilson 已提交
4497 4498 4499 4500
 * @mm:		the mm_struct of the target address space
 * @addr:	start address to access
 * @buf:	source or destination buffer
 * @len:	number of bytes to transfer
4501
 * @gup_flags:	flags modifying lookup behaviour
S
Stephen Wilson 已提交
4502 4503 4504 4505
 *
 * The caller must hold a reference on @mm.
 */
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
4506
		void *buf, int len, unsigned int gup_flags)
S
Stephen Wilson 已提交
4507
{
4508
	return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
S
Stephen Wilson 已提交
4509 4510
}

4511 4512 4513 4514 4515 4516
/*
 * Access another process' address space.
 * Source/target buffer must be kernel space,
 * Do not walk the page table directly, use get_user_pages
 */
int access_process_vm(struct task_struct *tsk, unsigned long addr,
4517
		void *buf, int len, unsigned int gup_flags)
4518 4519 4520 4521 4522 4523 4524 4525
{
	struct mm_struct *mm;
	int ret;

	mm = get_task_mm(tsk);
	if (!mm)
		return 0;

4526
	ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
4527

4528 4529 4530 4531
	mmput(mm);

	return ret;
}
4532
EXPORT_SYMBOL_GPL(access_process_vm);
4533

4534 4535 4536 4537 4538 4539 4540 4541
/*
 * Print the name of a VMA.
 */
void print_vma_addr(char *prefix, unsigned long ip)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

4542
	/*
4543
	 * we might be running from an atomic context so we cannot sleep
4544
	 */
4545
	if (!down_read_trylock(&mm->mmap_sem))
4546 4547
		return;

4548 4549 4550
	vma = find_vma(mm, ip);
	if (vma && vma->vm_file) {
		struct file *f = vma->vm_file;
4551
		char *buf = (char *)__get_free_page(GFP_NOWAIT);
4552
		if (buf) {
A
Andy Shevchenko 已提交
4553
			char *p;
4554

M
Miklos Szeredi 已提交
4555
			p = file_path(f, buf, PAGE_SIZE);
4556 4557
			if (IS_ERR(p))
				p = "?";
A
Andy Shevchenko 已提交
4558
			printk("%s%s[%lx+%lx]", prefix, kbasename(p),
4559 4560 4561 4562 4563
					vma->vm_start,
					vma->vm_end - vma->vm_start);
			free_page((unsigned long)buf);
		}
	}
4564
	up_read(&mm->mmap_sem);
4565
}
4566

4567
#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
4568
void __might_fault(const char *file, int line)
4569
{
4570 4571 4572 4573 4574 4575
	/*
	 * Some code (nfs/sunrpc) uses socket ops on kernel memory while
	 * holding the mmap_sem, this is safe because kernel memory doesn't
	 * get paged out, therefore we'll never actually fault, and the
	 * below annotations will generate false positives.
	 */
A
Al Viro 已提交
4576
	if (uaccess_kernel())
4577
		return;
4578
	if (pagefault_disabled())
4579
		return;
4580 4581
	__might_sleep(file, line, 0);
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
4582
	if (current->mm)
4583
		might_lock_read(&current->mm->mmap_sem);
4584
#endif
4585
}
4586
EXPORT_SYMBOL(__might_fault);
4587
#endif
A
Andrea Arcangeli 已提交
4588 4589

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
4590 4591 4592 4593 4594 4595 4596 4597 4598
/*
 * Process all subpages of the specified huge page with the specified
 * operation.  The target subpage will be processed last to keep its
 * cache lines hot.
 */
static inline void process_huge_page(
	unsigned long addr_hint, unsigned int pages_per_huge_page,
	void (*process_subpage)(unsigned long addr, int idx, void *arg),
	void *arg)
A
Andrea Arcangeli 已提交
4599
{
4600 4601 4602
	int i, n, base, l;
	unsigned long addr = addr_hint &
		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
A
Andrea Arcangeli 已提交
4603

4604
	/* Process target subpage last to keep its cache lines hot */
A
Andrea Arcangeli 已提交
4605
	might_sleep();
4606 4607
	n = (addr_hint - addr) / PAGE_SIZE;
	if (2 * n <= pages_per_huge_page) {
4608
		/* If target subpage in first half of huge page */
4609 4610
		base = 0;
		l = n;
4611
		/* Process subpages at the end of huge page */
4612 4613
		for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
			cond_resched();
4614
			process_subpage(addr + i * PAGE_SIZE, i, arg);
4615 4616
		}
	} else {
4617
		/* If target subpage in second half of huge page */
4618 4619
		base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
		l = pages_per_huge_page - n;
4620
		/* Process subpages at the begin of huge page */
4621 4622
		for (i = 0; i < base; i++) {
			cond_resched();
4623
			process_subpage(addr + i * PAGE_SIZE, i, arg);
4624 4625 4626
		}
	}
	/*
4627 4628
	 * Process remaining subpages in left-right-left-right pattern
	 * towards the target subpage
4629 4630 4631 4632 4633 4634
	 */
	for (i = 0; i < l; i++) {
		int left_idx = base + i;
		int right_idx = base + 2 * l - 1 - i;

		cond_resched();
4635
		process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
A
Andrea Arcangeli 已提交
4636
		cond_resched();
4637
		process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
A
Andrea Arcangeli 已提交
4638 4639 4640
	}
}

4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676
static void clear_gigantic_page(struct page *page,
				unsigned long addr,
				unsigned int pages_per_huge_page)
{
	int i;
	struct page *p = page;

	might_sleep();
	for (i = 0; i < pages_per_huge_page;
	     i++, p = mem_map_next(p, page, i)) {
		cond_resched();
		clear_user_highpage(p, addr + i * PAGE_SIZE);
	}
}

static void clear_subpage(unsigned long addr, int idx, void *arg)
{
	struct page *page = arg;

	clear_user_highpage(page + idx, addr);
}

void clear_huge_page(struct page *page,
		     unsigned long addr_hint, unsigned int pages_per_huge_page)
{
	unsigned long addr = addr_hint &
		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);

	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
		clear_gigantic_page(page, addr, pages_per_huge_page);
		return;
	}

	process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
}

A
Andrea Arcangeli 已提交
4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695
static void copy_user_gigantic_page(struct page *dst, struct page *src,
				    unsigned long addr,
				    struct vm_area_struct *vma,
				    unsigned int pages_per_huge_page)
{
	int i;
	struct page *dst_base = dst;
	struct page *src_base = src;

	for (i = 0; i < pages_per_huge_page; ) {
		cond_resched();
		copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);

		i++;
		dst = mem_map_next(dst, dst_base, i);
		src = mem_map_next(src, src_base, i);
	}
}

4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709
struct copy_subpage_arg {
	struct page *dst;
	struct page *src;
	struct vm_area_struct *vma;
};

static void copy_subpage(unsigned long addr, int idx, void *arg)
{
	struct copy_subpage_arg *copy_arg = arg;

	copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
			   addr, copy_arg->vma);
}

A
Andrea Arcangeli 已提交
4710
void copy_user_huge_page(struct page *dst, struct page *src,
4711
			 unsigned long addr_hint, struct vm_area_struct *vma,
A
Andrea Arcangeli 已提交
4712 4713
			 unsigned int pages_per_huge_page)
{
4714 4715 4716 4717 4718 4719 4720
	unsigned long addr = addr_hint &
		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
	struct copy_subpage_arg arg = {
		.dst = dst,
		.src = src,
		.vma = vma,
	};
A
Andrea Arcangeli 已提交
4721 4722 4723 4724 4725 4726 4727

	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
		copy_user_gigantic_page(dst, src, addr, vma,
					pages_per_huge_page);
		return;
	}

4728
	process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
A
Andrea Arcangeli 已提交
4729
}
4730 4731 4732

long copy_huge_page_from_user(struct page *dst_page,
				const void __user *usr_src,
4733 4734
				unsigned int pages_per_huge_page,
				bool allow_pagefault)
4735 4736 4737 4738 4739 4740 4741
{
	void *src = (void *)usr_src;
	void *page_kaddr;
	unsigned long i, rc = 0;
	unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;

	for (i = 0; i < pages_per_huge_page; i++) {
4742 4743 4744 4745
		if (allow_pagefault)
			page_kaddr = kmap(dst_page + i);
		else
			page_kaddr = kmap_atomic(dst_page + i);
4746 4747 4748
		rc = copy_from_user(page_kaddr,
				(const void __user *)(src + i * PAGE_SIZE),
				PAGE_SIZE);
4749 4750 4751 4752
		if (allow_pagefault)
			kunmap(dst_page + i);
		else
			kunmap_atomic(page_kaddr);
4753 4754 4755 4756 4757 4758 4759 4760 4761

		ret_val -= (PAGE_SIZE - rc);
		if (rc)
			break;

		cond_resched();
	}
	return ret_val;
}
A
Andrea Arcangeli 已提交
4762
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
4763

4764
#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
4765 4766 4767 4768 4769 4770 4771 4772 4773

static struct kmem_cache *page_ptl_cachep;

void __init ptlock_cache_init(void)
{
	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
			SLAB_PANIC, NULL);
}

4774
bool ptlock_alloc(struct page *page)
4775 4776 4777
{
	spinlock_t *ptl;

4778
	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
4779 4780
	if (!ptl)
		return false;
4781
	page->ptl = ptl;
4782 4783 4784
	return true;
}

4785
void ptlock_free(struct page *page)
4786
{
4787
	kmem_cache_free(page_ptl_cachep, page->ptl);
4788 4789
}
#endif