pageattr.c 28.3 KB
Newer Older
1 2
/*
 * Copyright 2002 Andi Kleen, SuSE Labs.
L
Linus Torvalds 已提交
3
 * Thanks to Ben LaHaise for precious feedback.
4
 */
L
Linus Torvalds 已提交
5
#include <linux/highmem.h>
I
Ingo Molnar 已提交
6
#include <linux/bootmem.h>
L
Linus Torvalds 已提交
7
#include <linux/module.h>
8
#include <linux/sched.h>
L
Linus Torvalds 已提交
9
#include <linux/slab.h>
10
#include <linux/mm.h>
11
#include <linux/interrupt.h>
12 13
#include <linux/seq_file.h>
#include <linux/debugfs.h>
14

15
#include <asm/e820.h>
L
Linus Torvalds 已提交
16 17
#include <asm/processor.h>
#include <asm/tlbflush.h>
D
Dave Jones 已提交
18
#include <asm/sections.h>
19
#include <asm/setup.h>
20 21
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
T
Thomas Gleixner 已提交
22
#include <asm/proto.h>
23
#include <asm/pat.h>
L
Linus Torvalds 已提交
24

I
Ingo Molnar 已提交
25 26 27
/*
 * The current flushing context - we pass it instead of 5 arguments:
 */
T
Thomas Gleixner 已提交
28
struct cpa_data {
29
	unsigned long	*vaddr;
T
Thomas Gleixner 已提交
30 31
	pgprot_t	mask_set;
	pgprot_t	mask_clr;
32
	int		numpages;
33
	int		flags;
T
Thomas Gleixner 已提交
34
	unsigned long	pfn;
35
	unsigned	force_split : 1;
36
	int		curpage;
T
Thomas Gleixner 已提交
37 38
};

39 40 41 42 43 44 45 46
/*
 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
 * entries change the page attribute in parallel to some other cpu
 * splitting a large page entry along with changing the attribute.
 */
static DEFINE_SPINLOCK(cpa_lock);

47 48 49
#define CPA_FLUSHTLB 1
#define CPA_ARRAY 2

50
#ifdef CONFIG_PROC_FS
51 52
static unsigned long direct_pages_count[PG_LEVEL_NUM];

53
void update_page_count(int level, unsigned long pages)
54 55
{
	unsigned long flags;
56

57 58 59 60
	/* Protect against CPA */
	spin_lock_irqsave(&pgd_lock, flags);
	direct_pages_count[level] += pages;
	spin_unlock_irqrestore(&pgd_lock, flags);
61 62 63 64 65 66 67 68
}

static void split_page_count(int level)
{
	direct_pages_count[level]--;
	direct_pages_count[level - 1] += PTRS_PER_PTE;
}

69
void arch_report_meminfo(struct seq_file *m)
70
{
71
	seq_printf(m, "DirectMap4k:    %8lu kB\n",
H
Hugh Dickins 已提交
72 73
			direct_pages_count[PG_LEVEL_4K] << 2);
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
74
	seq_printf(m, "DirectMap2M:    %8lu kB\n",
H
Hugh Dickins 已提交
75 76
			direct_pages_count[PG_LEVEL_2M] << 11);
#else
77
	seq_printf(m, "DirectMap4M:    %8lu kB\n",
H
Hugh Dickins 已提交
78 79
			direct_pages_count[PG_LEVEL_2M] << 12);
#endif
80
#ifdef CONFIG_X86_64
H
Hugh Dickins 已提交
81
	if (direct_gbpages)
82
		seq_printf(m, "DirectMap1G:    %8lu kB\n",
H
Hugh Dickins 已提交
83
			direct_pages_count[PG_LEVEL_1G] << 20);
84 85
#endif
}
86 87 88
#else
static inline void split_page_count(int level) { }
#endif
89

T
Thomas Gleixner 已提交
90 91 92 93 94 95 96 97 98
#ifdef CONFIG_X86_64

static inline unsigned long highmap_start_pfn(void)
{
	return __pa(_text) >> PAGE_SHIFT;
}

static inline unsigned long highmap_end_pfn(void)
{
99
	return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
T
Thomas Gleixner 已提交
100 101 102 103
}

#endif

104 105 106 107 108 109
#ifdef CONFIG_DEBUG_PAGEALLOC
# define debug_pagealloc 1
#else
# define debug_pagealloc 0
#endif

110 111
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
I
Ingo Molnar 已提交
112
{
113 114 115
	return addr >= start && addr < end;
}

T
Thomas Gleixner 已提交
116 117 118
/*
 * Flushing functions
 */
119 120 121 122 123 124 125 126 127

/**
 * clflush_cache_range - flush a cache range with clflush
 * @addr:	virtual start address
 * @size:	number of bytes to flush
 *
 * clflush is an unordered instruction which needs fencing with mfence
 * to avoid ordering issues.
 */
I
Ingo Molnar 已提交
128
void clflush_cache_range(void *vaddr, unsigned int size)
T
Thomas Gleixner 已提交
129
{
I
Ingo Molnar 已提交
130
	void *vend = vaddr + size - 1;
T
Thomas Gleixner 已提交
131

132
	mb();
I
Ingo Molnar 已提交
133 134 135 136 137 138 139 140

	for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
		clflush(vaddr);
	/*
	 * Flush any possible final partial cacheline:
	 */
	clflush(vend);

141
	mb();
T
Thomas Gleixner 已提交
142 143
}

144
static void __cpa_flush_all(void *arg)
T
Thomas Gleixner 已提交
145
{
146 147
	unsigned long cache = (unsigned long)arg;

T
Thomas Gleixner 已提交
148 149 150 151 152 153
	/*
	 * Flush all to work around Errata in early athlons regarding
	 * large page flushing.
	 */
	__flush_tlb_all();

154
	if (cache && boot_cpu_data.x86_model >= 4)
T
Thomas Gleixner 已提交
155 156 157
		wbinvd();
}

158
static void cpa_flush_all(unsigned long cache)
T
Thomas Gleixner 已提交
159 160 161
{
	BUG_ON(irqs_disabled());

162
	on_each_cpu(__cpa_flush_all, (void *) cache, 1);
T
Thomas Gleixner 已提交
163 164
}

165 166 167 168 169 170 171 172 173 174
static void __cpa_flush_range(void *arg)
{
	/*
	 * We could optimize that further and do individual per page
	 * tlb invalidates for a low number of pages. Caveat: we must
	 * flush the high aliases on 64bit as well.
	 */
	__flush_tlb_all();
}

175
static void cpa_flush_range(unsigned long start, int numpages, int cache)
176
{
I
Ingo Molnar 已提交
177 178 179
	unsigned int i, level;
	unsigned long addr;

180
	BUG_ON(irqs_disabled());
I
Ingo Molnar 已提交
181
	WARN_ON(PAGE_ALIGN(start) != start);
182

183
	on_each_cpu(__cpa_flush_range, NULL, 1);
184

185 186 187
	if (!cache)
		return;

T
Thomas Gleixner 已提交
188 189 190 191 192 193
	/*
	 * We only need to flush on one CPU,
	 * clflush is a MESI-coherent instruction that
	 * will cause all other CPUs to flush the same
	 * cachelines:
	 */
I
Ingo Molnar 已提交
194 195 196 197 198 199
	for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
		pte_t *pte = lookup_address(addr, &level);

		/*
		 * Only flush present addresses:
		 */
200
		if (pte && (pte_val(*pte) & _PAGE_PRESENT))
I
Ingo Molnar 已提交
201 202
			clflush_cache_range((void *) addr, PAGE_SIZE);
	}
203 204
}

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
static void cpa_flush_array(unsigned long *start, int numpages, int cache)
{
	unsigned int i, level;
	unsigned long *addr;

	BUG_ON(irqs_disabled());

	on_each_cpu(__cpa_flush_range, NULL, 1);

	if (!cache)
		return;

	/* 4M threshold */
	if (numpages >= 1024) {
		if (boot_cpu_data.x86_model >= 4)
			wbinvd();
		return;
	}
	/*
	 * We only need to flush on one CPU,
	 * clflush is a MESI-coherent instruction that
	 * will cause all other CPUs to flush the same
	 * cachelines:
	 */
	for (i = 0, addr = start; i < numpages; i++, addr++) {
		pte_t *pte = lookup_address(*addr, &level);

		/*
		 * Only flush present addresses:
		 */
		if (pte && (pte_val(*pte) & _PAGE_PRESENT))
			clflush_cache_range((void *) *addr, PAGE_SIZE);
	}
}

240 241 242 243 244 245
/*
 * Certain areas of memory on x86 require very specific protection flags,
 * for example the BIOS area or kernel text. Callers don't always get this
 * right (again, ioremap() on BIOS memory is not uncommon) so this function
 * checks and fixes these known static required protection bits.
 */
T
Thomas Gleixner 已提交
246 247
static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
				   unsigned long pfn)
248 249 250
{
	pgprot_t forbidden = __pgprot(0);

I
Ingo Molnar 已提交
251
	/*
252 253
	 * The BIOS area between 640k and 1Mb needs to be executable for
	 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
I
Ingo Molnar 已提交
254
	 */
T
Thomas Gleixner 已提交
255
	if (within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
256 257 258 259
		pgprot_val(forbidden) |= _PAGE_NX;

	/*
	 * The kernel text needs to be executable for obvious reasons
T
Thomas Gleixner 已提交
260 261
	 * Does not cover __inittext since that is gone later on. On
	 * 64bit we do not enforce !NX on the low mapping
262 263 264
	 */
	if (within(address, (unsigned long)_text, (unsigned long)_etext))
		pgprot_val(forbidden) |= _PAGE_NX;
265 266

	/*
T
Thomas Gleixner 已提交
267 268
	 * The .rodata section needs to be read-only. Using the pfn
	 * catches all aliases.
269
	 */
T
Thomas Gleixner 已提交
270 271
	if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT,
		   __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))
272
		pgprot_val(forbidden) |= _PAGE_RW;
273 274

	prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
I
Ingo Molnar 已提交
275 276 277 278

	return prot;
}

T
Thomas Gleixner 已提交
279 280 281 282 283 284 285 286
/*
 * Lookup the page table entry for a virtual address. Return a pointer
 * to the entry and the level of the mapping.
 *
 * Note: We return pud and pmd either when the entry is marked large
 * or when the present bit is not set. Otherwise we would return a
 * pointer to a nonexisting mapping.
 */
287
pte_t *lookup_address(unsigned long address, unsigned int *level)
288
{
L
Linus Torvalds 已提交
289 290 291
	pgd_t *pgd = pgd_offset_k(address);
	pud_t *pud;
	pmd_t *pmd;
292

T
Thomas Gleixner 已提交
293 294
	*level = PG_LEVEL_NONE;

L
Linus Torvalds 已提交
295 296
	if (pgd_none(*pgd))
		return NULL;
I
Ingo Molnar 已提交
297

L
Linus Torvalds 已提交
298 299 300
	pud = pud_offset(pgd, address);
	if (pud_none(*pud))
		return NULL;
301 302 303 304 305

	*level = PG_LEVEL_1G;
	if (pud_large(*pud) || !pud_present(*pud))
		return (pte_t *)pud;

L
Linus Torvalds 已提交
306 307 308
	pmd = pmd_offset(pud, address);
	if (pmd_none(*pmd))
		return NULL;
T
Thomas Gleixner 已提交
309 310

	*level = PG_LEVEL_2M;
T
Thomas Gleixner 已提交
311
	if (pmd_large(*pmd) || !pmd_present(*pmd))
L
Linus Torvalds 已提交
312 313
		return (pte_t *)pmd;

T
Thomas Gleixner 已提交
314
	*level = PG_LEVEL_4K;
I
Ingo Molnar 已提交
315

316 317
	return pte_offset_kernel(pmd, address);
}
318
EXPORT_SYMBOL_GPL(lookup_address);
319

I
Ingo Molnar 已提交
320 321 322
/*
 * Set the new pmd in all the pgds we know about:
 */
I
Ingo Molnar 已提交
323
static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
324 325 326
{
	/* change init_mm */
	set_pte_atomic(kpte, pte);
327
#ifdef CONFIG_X86_32
328
	if (!SHARED_KERNEL_PMD) {
329 330
		struct page *page;

331
		list_for_each_entry(page, &pgd_list, lru) {
332 333 334 335 336 337 338 339 340
			pgd_t *pgd;
			pud_t *pud;
			pmd_t *pmd;

			pgd = (pgd_t *)page_address(page) + pgd_index(address);
			pud = pud_offset(pgd, address);
			pmd = pmd_offset(pud, address);
			set_pte_atomic((pte_t *)pmd, pte);
		}
L
Linus Torvalds 已提交
341
	}
342
#endif
L
Linus Torvalds 已提交
343 344
}

I
Ingo Molnar 已提交
345 346 347
static int
try_preserve_large_page(pte_t *kpte, unsigned long address,
			struct cpa_data *cpa)
348
{
T
Thomas Gleixner 已提交
349
	unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn;
350 351
	pte_t new_pte, old_pte, *tmp;
	pgprot_t old_prot, new_prot;
352
	int i, do_split = 1;
353
	unsigned int level;
354

355 356 357
	if (cpa->force_split)
		return 1;

358 359 360 361 362 363 364 365 366 367 368
	spin_lock_irqsave(&pgd_lock, flags);
	/*
	 * Check for races, another CPU might have split this page
	 * up already:
	 */
	tmp = lookup_address(address, &level);
	if (tmp != kpte)
		goto out_unlock;

	switch (level) {
	case PG_LEVEL_2M:
369 370
		psize = PMD_PAGE_SIZE;
		pmask = PMD_PAGE_MASK;
371
		break;
372
#ifdef CONFIG_X86_64
373
	case PG_LEVEL_1G:
374 375
		psize = PUD_PAGE_SIZE;
		pmask = PUD_PAGE_MASK;
376 377
		break;
#endif
378
	default:
I
Ingo Molnar 已提交
379
		do_split = -EINVAL;
380 381 382 383 384 385 386 387 388
		goto out_unlock;
	}

	/*
	 * Calculate the number of pages, which fit into this large
	 * page starting at address:
	 */
	nextpage_addr = (address + psize) & pmask;
	numpages = (nextpage_addr - address) >> PAGE_SHIFT;
389 390
	if (numpages < cpa->numpages)
		cpa->numpages = numpages;
391 392 393 394 395 396 397 398 399

	/*
	 * We are safe now. Check whether the new pgprot is the same:
	 */
	old_pte = *kpte;
	old_prot = new_prot = pte_pgprot(old_pte);

	pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
	pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
T
Thomas Gleixner 已提交
400 401 402 403 404 405 406 407 408

	/*
	 * old_pte points to the large page base address. So we need
	 * to add the offset of the virtual address:
	 */
	pfn = pte_pfn(old_pte) + ((address & (psize - 1)) >> PAGE_SHIFT);
	cpa->pfn = pfn;

	new_prot = static_protections(new_prot, address, pfn);
409

410 411 412 413 414 415
	/*
	 * We need to check the full range, whether
	 * static_protection() requires a different pgprot for one of
	 * the pages in the range we try to preserve:
	 */
	addr = address + PAGE_SIZE;
T
Thomas Gleixner 已提交
416
	pfn++;
417
	for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE, pfn++) {
T
Thomas Gleixner 已提交
418
		pgprot_t chk_prot = static_protections(new_prot, addr, pfn);
419 420 421 422 423

		if (pgprot_val(chk_prot) != pgprot_val(new_prot))
			goto out_unlock;
	}

424 425 426 427 428
	/*
	 * If there are no changes, return. maxpages has been updated
	 * above:
	 */
	if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
I
Ingo Molnar 已提交
429
		do_split = 0;
430 431 432 433 434 435 436 437 438 439 440
		goto out_unlock;
	}

	/*
	 * We need to change the attributes. Check, whether we can
	 * change the large page in one go. We request a split, when
	 * the address is not aligned and the number of pages is
	 * smaller than the number of pages in the large page. Note
	 * that we limited the number of possible pages already to
	 * the number of pages in the large page.
	 */
441
	if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
442 443 444 445 446 447
		/*
		 * The address is aligned and the number of pages
		 * covers the full page.
		 */
		new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
		__set_pmd_pte(kpte, address, new_pte);
448
		cpa->flags |= CPA_FLUSHTLB;
I
Ingo Molnar 已提交
449
		do_split = 0;
450 451 452 453
	}

out_unlock:
	spin_unlock_irqrestore(&pgd_lock, flags);
I
Ingo Molnar 已提交
454

I
Ingo Molnar 已提交
455
	return do_split;
456 457
}

458
static int split_large_page(pte_t *kpte, unsigned long address)
459
{
T
Thomas Gleixner 已提交
460
	unsigned long flags, pfn, pfninc = 1;
I
Ingo Molnar 已提交
461
	unsigned int i, level;
462
	pte_t *pbase, *tmp;
I
Ingo Molnar 已提交
463
	pgprot_t ref_prot;
464 465 466 467 468 469 470
	struct page *base;

	if (!debug_pagealloc)
		spin_unlock(&cpa_lock);
	base = alloc_pages(GFP_KERNEL, 0);
	if (!debug_pagealloc)
		spin_lock(&cpa_lock);
S
Suresh Siddha 已提交
471 472
	if (!base)
		return -ENOMEM;
473

T
Thomas Gleixner 已提交
474
	spin_lock_irqsave(&pgd_lock, flags);
475 476 477 478 479
	/*
	 * Check for races, another CPU might have split this page
	 * up for us already:
	 */
	tmp = lookup_address(address, &level);
I
Ingo Molnar 已提交
480
	if (tmp != kpte)
481 482 483
		goto out_unlock;

	pbase = (pte_t *)page_address(base);
484
	paravirt_alloc_pte(&init_mm, page_to_pfn(base));
T
Thomas Gleixner 已提交
485
	ref_prot = pte_pgprot(pte_clrhuge(*kpte));
486 487 488 489 490 491 492
	/*
	 * If we ever want to utilize the PAT bit, we need to
	 * update this function to make sure it's converted from
	 * bit 12 to bit 7 when we cross from the 2MB level to
	 * the 4K level:
	 */
	WARN_ON_ONCE(pgprot_val(ref_prot) & _PAGE_PAT_LARGE);
493

494 495 496 497 498 499 500
#ifdef CONFIG_X86_64
	if (level == PG_LEVEL_1G) {
		pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
		pgprot_val(ref_prot) |= _PAGE_PSE;
	}
#endif

501 502 503 504
	/*
	 * Get the target pfn from the original entry:
	 */
	pfn = pte_pfn(*kpte);
505
	for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
506
		set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
507

508
	if (address >= (unsigned long)__va(0) &&
509 510 511 512 513
		address < (unsigned long)__va(max_low_pfn_mapped << PAGE_SHIFT))
		split_page_count(level);

#ifdef CONFIG_X86_64
	if (address >= (unsigned long)__va(1UL<<32) &&
514 515
		address < (unsigned long)__va(max_pfn_mapped << PAGE_SHIFT))
		split_page_count(level);
516
#endif
517

518
	/*
519
	 * Install the new, split up pagetable.
520
	 *
521 522 523
	 * We use the standard kernel pagetable protections for the new
	 * pagetable protections, the actual ptes set above control the
	 * primary protection behavior:
524
	 */
525
	__set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
526 527 528 529 530 531 532 533 534 535 536

	/*
	 * Intel Atom errata AAH41 workaround.
	 *
	 * The real fix should be in hw or in a microcode update, but
	 * we also probabilistically try to reduce the window of having
	 * a large TLB mixed with 4K TLBs while instruction fetches are
	 * going on.
	 */
	__flush_tlb_all();

537 538 539
	base = NULL;

out_unlock:
T
Thomas Gleixner 已提交
540 541 542 543
	/*
	 * If we dropped out via the lookup_address check under
	 * pgd_lock then stick the page back into the pool:
	 */
S
Suresh Siddha 已提交
544 545
	if (base)
		__free_page(base);
I
Ingo Molnar 已提交
546
	spin_unlock_irqrestore(&pgd_lock, flags);
547 548 549 550

	return 0;
}

551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
			       int primary)
{
	/*
	 * Ignore all non primary paths.
	 */
	if (!primary)
		return 0;

	/*
	 * Ignore the NULL PTE for kernel identity mapping, as it is expected
	 * to have holes.
	 * Also set numpages to '1' indicating that we processed cpa req for
	 * one virtual address page and its pfn. TBD: numpages can be set based
	 * on the initial value and the level returned by lookup_address().
	 */
	if (within(vaddr, PAGE_OFFSET,
		   PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
		cpa->numpages = 1;
		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
		return 0;
	} else {
		WARN(1, KERN_WARNING "CPA: called for zero pte. "
			"vaddr = %lx cpa->vaddr = %lx\n", vaddr,
			*cpa->vaddr);

		return -EFAULT;
	}
}

T
Thomas Gleixner 已提交
581
static int __change_page_attr(struct cpa_data *cpa, int primary)
582
{
583
	unsigned long address;
584 585
	int do_split, err;
	unsigned int level;
T
Thomas Gleixner 已提交
586
	pte_t *kpte, old_pte;
L
Linus Torvalds 已提交
587

588 589 590 591
	if (cpa->flags & CPA_ARRAY)
		address = cpa->vaddr[cpa->curpage];
	else
		address = *cpa->vaddr;
592
repeat:
593
	kpte = lookup_address(address, &level);
L
Linus Torvalds 已提交
594
	if (!kpte)
595
		return __cpa_process_fault(cpa, address, primary);
T
Thomas Gleixner 已提交
596 597

	old_pte = *kpte;
598 599
	if (!pte_val(old_pte))
		return __cpa_process_fault(cpa, address, primary);
600

T
Thomas Gleixner 已提交
601
	if (level == PG_LEVEL_4K) {
T
Thomas Gleixner 已提交
602
		pte_t new_pte;
603
		pgprot_t new_prot = pte_pgprot(old_pte);
T
Thomas Gleixner 已提交
604
		unsigned long pfn = pte_pfn(old_pte);
I
Ingo Molnar 已提交
605

T
Thomas Gleixner 已提交
606 607
		pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
		pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
I
Ingo Molnar 已提交
608

T
Thomas Gleixner 已提交
609
		new_prot = static_protections(new_prot, address, pfn);
I
Ingo Molnar 已提交
610

611 612 613 614 615
		/*
		 * We need to keep the pfn from the existing PTE,
		 * after all we're only going to change it's attributes
		 * not the memory it points to
		 */
T
Thomas Gleixner 已提交
616 617
		new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
		cpa->pfn = pfn;
618 619 620 621 622
		/*
		 * Do we really change anything ?
		 */
		if (pte_val(old_pte) != pte_val(new_pte)) {
			set_pte_atomic(kpte, new_pte);
623
			cpa->flags |= CPA_FLUSHTLB;
624
		}
625
		cpa->numpages = 1;
626
		return 0;
L
Linus Torvalds 已提交
627
	}
628 629 630 631 632

	/*
	 * Check, whether we can keep the large page intact
	 * and just change the pte:
	 */
I
Ingo Molnar 已提交
633
	do_split = try_preserve_large_page(kpte, address, cpa);
634 635
	/*
	 * When the range fits into the existing large page,
636
	 * return. cp->numpages and cpa->tlbflush have been updated in
637 638
	 * try_large_page:
	 */
I
Ingo Molnar 已提交
639 640
	if (do_split <= 0)
		return do_split;
641 642 643 644

	/*
	 * We have to split the large page:
	 */
I
Ingo Molnar 已提交
645 646
	err = split_large_page(kpte, address);
	if (!err) {
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
		/*
	 	 * Do a global flush tlb after splitting the large page
	 	 * and before we do the actual change page attribute in the PTE.
	 	 *
	 	 * With out this, we violate the TLB application note, that says
	 	 * "The TLBs may contain both ordinary and large-page
		 *  translations for a 4-KByte range of linear addresses. This
		 *  may occur if software modifies the paging structures so that
		 *  the page size used for the address range changes. If the two
		 *  translations differ with respect to page frame or attributes
		 *  (e.g., permissions), processor behavior is undefined and may
		 *  be implementation-specific."
	 	 *
	 	 * We do this global tlb flush inside the cpa_lock, so that we
		 * don't allow any other cpu, with stale tlb entries change the
		 * page attribute in parallel, that also falls into the
		 * just split large page entry.
	 	 */
		flush_tlb_all();
I
Ingo Molnar 已提交
666 667
		goto repeat;
	}
I
Ingo Molnar 已提交
668

I
Ingo Molnar 已提交
669
	return err;
670
}
L
Linus Torvalds 已提交
671

T
Thomas Gleixner 已提交
672 673 674
static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);

static int cpa_process_alias(struct cpa_data *cpa)
L
Linus Torvalds 已提交
675
{
T
Thomas Gleixner 已提交
676
	struct cpa_data alias_cpa;
677
	int ret = 0;
678
	unsigned long temp_cpa_vaddr, vaddr;
679

Y
Yinghai Lu 已提交
680
	if (cpa->pfn >= max_pfn_mapped)
T
Thomas Gleixner 已提交
681
		return 0;
682

683
#ifdef CONFIG_X86_64
Y
Yinghai Lu 已提交
684
	if (cpa->pfn >= max_low_pfn_mapped && cpa->pfn < (1UL<<(32-PAGE_SHIFT)))
685 686
		return 0;
#endif
687 688 689 690
	/*
	 * No need to redo, when the primary call touched the direct
	 * mapping already:
	 */
691 692 693 694 695 696
	if (cpa->flags & CPA_ARRAY)
		vaddr = cpa->vaddr[cpa->curpage];
	else
		vaddr = *cpa->vaddr;

	if (!(within(vaddr, PAGE_OFFSET,
697
		    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
698

699
		alias_cpa = *cpa;
700 701 702 703
		temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
		alias_cpa.vaddr = &temp_cpa_vaddr;
		alias_cpa.flags &= ~CPA_ARRAY;

704 705 706

		ret = __change_page_attr_set_clr(&alias_cpa, 0);
	}
707 708

#ifdef CONFIG_X86_64
T
Thomas Gleixner 已提交
709 710
	if (ret)
		return ret;
711 712 713 714
	/*
	 * No need to redo, when the primary call touched the high
	 * mapping already:
	 */
715
	if (within(vaddr, (unsigned long) _text, _brk_end))
716 717
		return 0;

A
Arjan van de Ven 已提交
718
	/*
719 720
	 * If the physical address is inside the kernel map, we need
	 * to touch the high mapped kernel as well:
A
Arjan van de Ven 已提交
721
	 */
T
Thomas Gleixner 已提交
722 723
	if (!within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn()))
		return 0;
724

T
Thomas Gleixner 已提交
725
	alias_cpa = *cpa;
726 727 728
	temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base;
	alias_cpa.vaddr = &temp_cpa_vaddr;
	alias_cpa.flags &= ~CPA_ARRAY;
T
Thomas Gleixner 已提交
729 730 731 732 733

	/*
	 * The high mapping range is imprecise, so ignore the return value.
	 */
	__change_page_attr_set_clr(&alias_cpa, 0);
A
Arjan van de Ven 已提交
734
#endif
T
Thomas Gleixner 已提交
735
	return ret;
L
Linus Torvalds 已提交
736 737
}

T
Thomas Gleixner 已提交
738
static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
739
{
740
	int ret, numpages = cpa->numpages;
741

742 743 744 745 746
	while (numpages) {
		/*
		 * Store the remaining nr of pages for the large page
		 * preservation check.
		 */
747
		cpa->numpages = numpages;
748 749 750
		/* for array changes, we can't use large page */
		if (cpa->flags & CPA_ARRAY)
			cpa->numpages = 1;
T
Thomas Gleixner 已提交
751

752 753
		if (!debug_pagealloc)
			spin_lock(&cpa_lock);
T
Thomas Gleixner 已提交
754
		ret = __change_page_attr(cpa, checkalias);
755 756
		if (!debug_pagealloc)
			spin_unlock(&cpa_lock);
757 758 759
		if (ret)
			return ret;

T
Thomas Gleixner 已提交
760 761 762 763 764 765
		if (checkalias) {
			ret = cpa_process_alias(cpa);
			if (ret)
				return ret;
		}

766 767 768 769 770
		/*
		 * Adjust the number of pages with the result of the
		 * CPA operation. Either a large page has been
		 * preserved or a single page update happened.
		 */
771 772
		BUG_ON(cpa->numpages > numpages);
		numpages -= cpa->numpages;
773 774 775 776 777
		if (cpa->flags & CPA_ARRAY)
			cpa->curpage++;
		else
			*cpa->vaddr += cpa->numpages * PAGE_SIZE;

778
	}
779 780 781
	return 0;
}

782 783 784 785 786 787
static inline int cache_attr(pgprot_t attr)
{
	return pgprot_val(attr) &
		(_PAGE_PAT | _PAGE_PAT_LARGE | _PAGE_PWT | _PAGE_PCD);
}

788
static int change_page_attr_set_clr(unsigned long *addr, int numpages,
789
				    pgprot_t mask_set, pgprot_t mask_clr,
790
				    int force_split, int in_flag)
791
{
T
Thomas Gleixner 已提交
792
	struct cpa_data cpa;
793
	int ret, cache, checkalias;
794 795 796 797 798 799 800

	/*
	 * Check, if we are requested to change a not supported
	 * feature:
	 */
	mask_set = canon_pgprot(mask_set);
	mask_clr = canon_pgprot(mask_clr);
801
	if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
802 803
		return 0;

804
	/* Ensure we are PAGE_SIZE aligned */
805
	if (!(in_flag & CPA_ARRAY)) {
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
		if (*addr & ~PAGE_MASK) {
			*addr &= PAGE_MASK;
			/*
			 * People should not be passing in unaligned addresses:
			 */
			WARN_ON_ONCE(1);
		}
	} else {
		int i;
		for (i = 0; i < numpages; i++) {
			if (addr[i] & ~PAGE_MASK) {
				addr[i] &= PAGE_MASK;
				WARN_ON_ONCE(1);
			}
		}
821 822
	}

823 824 825
	/* Must avoid aliasing mappings in the highmem code */
	kmap_flush_unused();

N
Nick Piggin 已提交
826 827
	vm_unmap_aliases();

828 829 830 831 832 833 834
	/*
	 * If we're called with lazy mmu updates enabled, the
	 * in-memory pte state may be stale.  Flush pending updates to
	 * bring them up to date.
	 */
	arch_flush_lazy_mmu_mode();

T
Thomas Gleixner 已提交
835 836 837 838
	cpa.vaddr = addr;
	cpa.numpages = numpages;
	cpa.mask_set = mask_set;
	cpa.mask_clr = mask_clr;
839 840
	cpa.flags = 0;
	cpa.curpage = 0;
841
	cpa.force_split = force_split;
T
Thomas Gleixner 已提交
842

843
	if (in_flag & CPA_ARRAY)
844 845
		cpa.flags |= CPA_ARRAY;

846 847 848 849
	/* No alias checking for _NX bit modifications */
	checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;

	ret = __change_page_attr_set_clr(&cpa, checkalias);
850

851 852 853
	/*
	 * Check whether we really changed something:
	 */
854
	if (!(cpa.flags & CPA_FLUSHTLB))
855
		goto out;
856

857 858 859 860 861 862
	/*
	 * No need to flush, when we did not set any of the caching
	 * attributes:
	 */
	cache = cache_attr(mask_set);

863 864 865
	/*
	 * On success we use clflush, when the CPU supports it to
	 * avoid the wbindv. If the CPU does not support it and in the
866
	 * error case we fall back to cpa_flush_all (which uses
867 868
	 * wbindv):
	 */
869 870 871 872 873 874
	if (!ret && cpu_has_clflush) {
		if (cpa.flags & CPA_ARRAY)
			cpa_flush_array(addr, numpages, cache);
		else
			cpa_flush_range(*addr, numpages, cache);
	} else
875
		cpa_flush_all(cache);
876

877 878 879 880 881 882 883
	/*
	 * If we've been called with lazy mmu updates enabled, then
	 * make sure that everything gets flushed out before we
	 * return.
	 */
	arch_flush_lazy_mmu_mode();

884
out:
885 886 887
	return ret;
}

888 889
static inline int change_page_attr_set(unsigned long *addr, int numpages,
				       pgprot_t mask, int array)
890
{
891
	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
892
		(array ? CPA_ARRAY : 0));
893 894
}

895 896
static inline int change_page_attr_clear(unsigned long *addr, int numpages,
					 pgprot_t mask, int array)
897
{
898
	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
899
		(array ? CPA_ARRAY : 0));
900 901
}

902
int _set_memory_uc(unsigned long addr, int numpages)
903
{
904 905 906
	/*
	 * for now UC MINUS. see comments in ioremap_nocache()
	 */
907 908
	return change_page_attr_set(&addr, numpages,
				    __pgprot(_PAGE_CACHE_UC_MINUS), 0);
909
}
910 911 912

int set_memory_uc(unsigned long addr, int numpages)
{
913 914 915
	/*
	 * for now UC MINUS. see comments in ioremap_nocache()
	 */
916
	if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
917
			    _PAGE_CACHE_UC_MINUS, NULL))
918 919 920 921
		return -EINVAL;

	return _set_memory_uc(addr, numpages);
}
922 923
EXPORT_SYMBOL(set_memory_uc);

924 925
int set_memory_array_uc(unsigned long *addr, int addrinarray)
{
926 927
	unsigned long start;
	unsigned long end;
928 929 930 931 932
	int i;
	/*
	 * for now UC MINUS. see comments in ioremap_nocache()
	 */
	for (i = 0; i < addrinarray; i++) {
933 934 935 936 937 938 939
		start = __pa(addr[i]);
		for (end = start + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
			if (end != __pa(addr[i + 1]))
				break;
			i++;
		}
		if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL))
940 941 942 943 944 945
			goto out;
	}

	return change_page_attr_set(addr, addrinarray,
				    __pgprot(_PAGE_CACHE_UC_MINUS), 1);
out:
946 947 948 949 950
	for (i = 0; i < addrinarray; i++) {
		unsigned long tmp = __pa(addr[i]);

		if (tmp == start)
			break;
951
		for (end = tmp + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
952 953 954 955 956 957
			if (end != __pa(addr[i + 1]))
				break;
			i++;
		}
		free_memtype(tmp, end);
	}
958 959 960 961
	return -EINVAL;
}
EXPORT_SYMBOL(set_memory_array_uc);

962 963
int _set_memory_wc(unsigned long addr, int numpages)
{
964 965
	return change_page_attr_set(&addr, numpages,
				    __pgprot(_PAGE_CACHE_WC), 0);
966 967 968 969
}

int set_memory_wc(unsigned long addr, int numpages)
{
970
	if (!pat_enabled)
971 972
		return set_memory_uc(addr, numpages);

973
	if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
974 975 976 977 978 979 980
		_PAGE_CACHE_WC, NULL))
		return -EINVAL;

	return _set_memory_wc(addr, numpages);
}
EXPORT_SYMBOL(set_memory_wc);

981
int _set_memory_wb(unsigned long addr, int numpages)
982
{
983 984
	return change_page_attr_clear(&addr, numpages,
				      __pgprot(_PAGE_CACHE_MASK), 0);
985
}
986 987 988

int set_memory_wb(unsigned long addr, int numpages)
{
989
	free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
990 991 992

	return _set_memory_wb(addr, numpages);
}
993 994
EXPORT_SYMBOL(set_memory_wb);

995 996 997 998
int set_memory_array_wb(unsigned long *addr, int addrinarray)
{
	int i;

999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	for (i = 0; i < addrinarray; i++) {
		unsigned long start = __pa(addr[i]);
		unsigned long end;

		for (end = start + PAGE_SIZE; i < addrinarray - 1; end += PAGE_SIZE) {
			if (end != __pa(addr[i + 1]))
				break;
			i++;
		}
		free_memtype(start, end);
	}
1010 1011 1012 1013 1014
	return change_page_attr_clear(addr, addrinarray,
				      __pgprot(_PAGE_CACHE_MASK), 1);
}
EXPORT_SYMBOL(set_memory_array_wb);

1015 1016
int set_memory_x(unsigned long addr, int numpages)
{
1017
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
1018 1019 1020 1021 1022
}
EXPORT_SYMBOL(set_memory_x);

int set_memory_nx(unsigned long addr, int numpages)
{
1023
	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
1024 1025 1026 1027 1028
}
EXPORT_SYMBOL(set_memory_nx);

int set_memory_ro(unsigned long addr, int numpages)
{
1029
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
1030
}
1031
EXPORT_SYMBOL_GPL(set_memory_ro);
1032 1033 1034

int set_memory_rw(unsigned long addr, int numpages)
{
1035
	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
1036
}
1037
EXPORT_SYMBOL_GPL(set_memory_rw);
I
Ingo Molnar 已提交
1038 1039 1040

int set_memory_np(unsigned long addr, int numpages)
{
1041
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
I
Ingo Molnar 已提交
1042
}
1043

1044 1045
int set_memory_4k(unsigned long addr, int numpages)
{
1046 1047
	return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
					__pgprot(0), 1, 0);
1048 1049
}

1050 1051 1052 1053
int set_pages_uc(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1054
	return set_memory_uc(addr, numpages);
1055 1056 1057 1058 1059 1060 1061
}
EXPORT_SYMBOL(set_pages_uc);

int set_pages_wb(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1062
	return set_memory_wb(addr, numpages);
1063 1064 1065 1066 1067 1068 1069
}
EXPORT_SYMBOL(set_pages_wb);

int set_pages_x(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1070
	return set_memory_x(addr, numpages);
1071 1072 1073 1074 1075 1076 1077
}
EXPORT_SYMBOL(set_pages_x);

int set_pages_nx(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1078
	return set_memory_nx(addr, numpages);
1079 1080 1081 1082 1083 1084 1085
}
EXPORT_SYMBOL(set_pages_nx);

int set_pages_ro(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1086
	return set_memory_ro(addr, numpages);
1087 1088 1089 1090 1091
}

int set_pages_rw(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);
1092

T
Thomas Gleixner 已提交
1093
	return set_memory_rw(addr, numpages);
I
Ingo Molnar 已提交
1094 1095
}

L
Linus Torvalds 已提交
1096
#ifdef CONFIG_DEBUG_PAGEALLOC
I
Ingo Molnar 已提交
1097 1098 1099

static int __set_pages_p(struct page *page, int numpages)
{
1100 1101
	unsigned long tempaddr = (unsigned long) page_address(page);
	struct cpa_data cpa = { .vaddr = &tempaddr,
T
Thomas Gleixner 已提交
1102 1103
				.numpages = numpages,
				.mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1104 1105
				.mask_clr = __pgprot(0),
				.flags = 0};
1106

1107 1108 1109 1110 1111 1112 1113
	/*
	 * No alias checking needed for setting present flag. otherwise,
	 * we may need to break large pages for 64-bit kernel text
	 * mappings (this adds to complexity if we want to do this from
	 * atomic context especially). Let's keep it simple!
	 */
	return __change_page_attr_set_clr(&cpa, 0);
I
Ingo Molnar 已提交
1114 1115 1116 1117
}

static int __set_pages_np(struct page *page, int numpages)
{
1118 1119
	unsigned long tempaddr = (unsigned long) page_address(page);
	struct cpa_data cpa = { .vaddr = &tempaddr,
T
Thomas Gleixner 已提交
1120 1121
				.numpages = numpages,
				.mask_set = __pgprot(0),
1122 1123
				.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
				.flags = 0};
1124

1125 1126 1127 1128 1129 1130 1131
	/*
	 * No alias checking needed for setting not present flag. otherwise,
	 * we may need to break large pages for 64-bit kernel text
	 * mappings (this adds to complexity if we want to do this from
	 * atomic context especially). Let's keep it simple!
	 */
	return __change_page_attr_set_clr(&cpa, 0);
I
Ingo Molnar 已提交
1132 1133
}

L
Linus Torvalds 已提交
1134 1135 1136 1137
void kernel_map_pages(struct page *page, int numpages, int enable)
{
	if (PageHighMem(page))
		return;
1138
	if (!enable) {
1139 1140
		debug_check_no_locks_freed(page_address(page),
					   numpages * PAGE_SIZE);
1141
	}
1142

1143 1144 1145 1146 1147 1148
	/*
	 * If page allocator is not up yet then do not call c_p_a():
	 */
	if (!debug_pagealloc_enabled)
		return;

1149
	/*
I
Ingo Molnar 已提交
1150
	 * The return value is ignored as the calls cannot fail.
1151 1152
	 * Large pages for identity mappings are not used at boot time
	 * and hence no memory allocations during large page split.
L
Linus Torvalds 已提交
1153
	 */
I
Ingo Molnar 已提交
1154 1155 1156 1157
	if (enable)
		__set_pages_p(page, numpages);
	else
		__set_pages_np(page, numpages);
1158 1159

	/*
1160 1161
	 * We should perform an IPI and flush all tlbs,
	 * but that can deadlock->flush only current cpu:
L
Linus Torvalds 已提交
1162 1163
	 */
	__flush_tlb_all();
1164 1165
}

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
#ifdef CONFIG_HIBERNATION

bool kernel_page_present(struct page *page)
{
	unsigned int level;
	pte_t *pte;

	if (PageHighMem(page))
		return false;

	pte = lookup_address((unsigned long)page_address(page), &level);
	return (pte_val(*pte) & _PAGE_PRESENT);
}

#endif /* CONFIG_HIBERNATION */

#endif /* CONFIG_DEBUG_PAGEALLOC */
1183 1184 1185 1186 1187 1188 1189 1190

/*
 * The testcases use internal knowledge of the implementation that shouldn't
 * be exposed to the rest of the kernel. Include these directly here.
 */
#ifdef CONFIG_CPA_DEBUG
#include "pageattr-test.c"
#endif