pageattr.c 50.5 KB
Newer Older
1 2
/*
 * Copyright 2002 Andi Kleen, SuSE Labs.
L
Linus Torvalds 已提交
3
 * Thanks to Ben LaHaise for precious feedback.
4
 */
L
Linus Torvalds 已提交
5
#include <linux/highmem.h>
I
Ingo Molnar 已提交
6
#include <linux/bootmem.h>
7 8
#include <linux/sched.h>
#include <linux/mm.h>
9
#include <linux/interrupt.h>
10 11
#include <linux/seq_file.h>
#include <linux/debugfs.h>
12
#include <linux/pfn.h>
13
#include <linux/percpu.h>
14
#include <linux/gfp.h>
15
#include <linux/pci.h>
16
#include <linux/vmalloc.h>
17

18
#include <asm/e820/api.h>
L
Linus Torvalds 已提交
19 20
#include <asm/processor.h>
#include <asm/tlbflush.h>
D
Dave Jones 已提交
21
#include <asm/sections.h>
22
#include <asm/setup.h>
23
#include <linux/uaccess.h>
24
#include <asm/pgalloc.h>
T
Thomas Gleixner 已提交
25
#include <asm/proto.h>
26
#include <asm/pat.h>
L
Laura Abbott 已提交
27
#include <asm/set_memory.h>
L
Linus Torvalds 已提交
28

I
Ingo Molnar 已提交
29 30 31
/*
 * The current flushing context - we pass it instead of 5 arguments:
 */
T
Thomas Gleixner 已提交
32
struct cpa_data {
33
	unsigned long	*vaddr;
34
	pgd_t		*pgd;
T
Thomas Gleixner 已提交
35 36
	pgprot_t	mask_set;
	pgprot_t	mask_clr;
37
	unsigned long	numpages;
38
	int		flags;
T
Thomas Gleixner 已提交
39
	unsigned long	pfn;
40
	unsigned	force_split : 1;
41
	int		curpage;
42
	struct page	**pages;
T
Thomas Gleixner 已提交
43 44
};

45 46 47 48 49 50 51 52
/*
 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
 * entries change the page attribute in parallel to some other cpu
 * splitting a large page entry along with changing the attribute.
 */
static DEFINE_SPINLOCK(cpa_lock);

53 54
#define CPA_FLUSHTLB 1
#define CPA_ARRAY 2
55
#define CPA_PAGES_ARRAY 4
56

57
#ifdef CONFIG_PROC_FS
58 59
static unsigned long direct_pages_count[PG_LEVEL_NUM];

60
void update_page_count(int level, unsigned long pages)
61 62
{
	/* Protect against CPA */
A
Andrea Arcangeli 已提交
63
	spin_lock(&pgd_lock);
64
	direct_pages_count[level] += pages;
A
Andrea Arcangeli 已提交
65
	spin_unlock(&pgd_lock);
66 67 68 69
}

static void split_page_count(int level)
{
70 71 72
	if (direct_pages_count[level] == 0)
		return;

73 74 75 76
	direct_pages_count[level]--;
	direct_pages_count[level - 1] += PTRS_PER_PTE;
}

77
void arch_report_meminfo(struct seq_file *m)
78
{
79
	seq_printf(m, "DirectMap4k:    %8lu kB\n",
H
Hugh Dickins 已提交
80 81
			direct_pages_count[PG_LEVEL_4K] << 2);
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
82
	seq_printf(m, "DirectMap2M:    %8lu kB\n",
H
Hugh Dickins 已提交
83 84
			direct_pages_count[PG_LEVEL_2M] << 11);
#else
85
	seq_printf(m, "DirectMap4M:    %8lu kB\n",
H
Hugh Dickins 已提交
86 87 88
			direct_pages_count[PG_LEVEL_2M] << 12);
#endif
	if (direct_gbpages)
89
		seq_printf(m, "DirectMap1G:    %8lu kB\n",
H
Hugh Dickins 已提交
90
			direct_pages_count[PG_LEVEL_1G] << 20);
91
}
92 93 94
#else
static inline void split_page_count(int level) { }
#endif
95

T
Thomas Gleixner 已提交
96 97 98 99
#ifdef CONFIG_X86_64

static inline unsigned long highmap_start_pfn(void)
{
100
	return __pa_symbol(_text) >> PAGE_SHIFT;
T
Thomas Gleixner 已提交
101 102 103 104
}

static inline unsigned long highmap_end_pfn(void)
{
105 106
	/* Do not reference physical address outside the kernel. */
	return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
T
Thomas Gleixner 已提交
107 108 109 110
}

#endif

111 112
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
I
Ingo Molnar 已提交
113
{
114 115 116
	return addr >= start && addr < end;
}

117 118 119 120 121 122
static inline int
within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr <= end;
}

T
Thomas Gleixner 已提交
123 124 125
/*
 * Flushing functions
 */
126 127 128

/**
 * clflush_cache_range - flush a cache range with clflush
129
 * @vaddr:	virtual start address
130 131
 * @size:	number of bytes to flush
 *
132 133
 * clflushopt is an unordered instruction which needs fencing with mfence or
 * sfence to avoid ordering issues.
134
 */
I
Ingo Molnar 已提交
135
void clflush_cache_range(void *vaddr, unsigned int size)
T
Thomas Gleixner 已提交
136
{
137 138
	const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
	void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
139
	void *vend = vaddr + size;
140 141 142

	if (p >= vend)
		return;
T
Thomas Gleixner 已提交
143

144
	mb();
I
Ingo Molnar 已提交
145

146
	for (; p < vend; p += clflush_size)
147
		clflushopt(p);
I
Ingo Molnar 已提交
148

149
	mb();
T
Thomas Gleixner 已提交
150
}
151
EXPORT_SYMBOL_GPL(clflush_cache_range);
T
Thomas Gleixner 已提交
152

153 154 155 156 157 158
void arch_invalidate_pmem(void *addr, size_t size)
{
	clflush_cache_range(addr, size);
}
EXPORT_SYMBOL_GPL(arch_invalidate_pmem);

159
static void __cpa_flush_all(void *arg)
T
Thomas Gleixner 已提交
160
{
161 162
	unsigned long cache = (unsigned long)arg;

T
Thomas Gleixner 已提交
163 164 165 166 167 168
	/*
	 * Flush all to work around Errata in early athlons regarding
	 * large page flushing.
	 */
	__flush_tlb_all();

169
	if (cache && boot_cpu_data.x86 >= 4)
T
Thomas Gleixner 已提交
170 171 172
		wbinvd();
}

173
static void cpa_flush_all(unsigned long cache)
T
Thomas Gleixner 已提交
174 175 176
{
	BUG_ON(irqs_disabled());

177
	on_each_cpu(__cpa_flush_all, (void *) cache, 1);
T
Thomas Gleixner 已提交
178 179
}

180 181 182 183 184 185 186 187 188 189
static void __cpa_flush_range(void *arg)
{
	/*
	 * We could optimize that further and do individual per page
	 * tlb invalidates for a low number of pages. Caveat: we must
	 * flush the high aliases on 64bit as well.
	 */
	__flush_tlb_all();
}

190
static void cpa_flush_range(unsigned long start, int numpages, int cache)
191
{
I
Ingo Molnar 已提交
192 193 194
	unsigned int i, level;
	unsigned long addr;

195
	BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
I
Ingo Molnar 已提交
196
	WARN_ON(PAGE_ALIGN(start) != start);
197

198
	on_each_cpu(__cpa_flush_range, NULL, 1);
199

200 201 202
	if (!cache)
		return;

T
Thomas Gleixner 已提交
203 204 205 206 207 208
	/*
	 * We only need to flush on one CPU,
	 * clflush is a MESI-coherent instruction that
	 * will cause all other CPUs to flush the same
	 * cachelines:
	 */
I
Ingo Molnar 已提交
209 210 211 212 213 214
	for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
		pte_t *pte = lookup_address(addr, &level);

		/*
		 * Only flush present addresses:
		 */
215
		if (pte && (pte_val(*pte) & _PAGE_PRESENT))
I
Ingo Molnar 已提交
216 217
			clflush_cache_range((void *) addr, PAGE_SIZE);
	}
218 219
}

220 221
static void cpa_flush_array(unsigned long *start, int numpages, int cache,
			    int in_flags, struct page **pages)
222 223
{
	unsigned int i, level;
224 225 226 227 228 229 230 231 232 233 234 235
#ifdef CONFIG_PREEMPT
	/*
	 * Avoid wbinvd() because it causes latencies on all CPUs,
	 * regardless of any CPU isolation that may be in effect.
	 *
	 * This should be extended for CAT enabled systems independent of
	 * PREEMPT because wbinvd() does not respect the CAT partitions and
	 * this is exposed to unpriviledged users through the graphics
	 * subsystem.
	 */
	unsigned long do_wbinvd = 0;
#else
236
	unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
237
#endif
238 239 240

	BUG_ON(irqs_disabled());

241
	on_each_cpu(__cpa_flush_all, (void *) do_wbinvd, 1);
242

243
	if (!cache || do_wbinvd)
244 245 246 247 248 249 250 251
		return;

	/*
	 * We only need to flush on one CPU,
	 * clflush is a MESI-coherent instruction that
	 * will cause all other CPUs to flush the same
	 * cachelines:
	 */
252 253 254 255 256 257 258 259 260 261
	for (i = 0; i < numpages; i++) {
		unsigned long addr;
		pte_t *pte;

		if (in_flags & CPA_PAGES_ARRAY)
			addr = (unsigned long)page_address(pages[i]);
		else
			addr = start[i];

		pte = lookup_address(addr, &level);
262 263 264 265 266

		/*
		 * Only flush present addresses:
		 */
		if (pte && (pte_val(*pte) & _PAGE_PRESENT))
267
			clflush_cache_range((void *)addr, PAGE_SIZE);
268 269 270
	}
}

271 272 273 274 275 276
/*
 * Certain areas of memory on x86 require very specific protection flags,
 * for example the BIOS area or kernel text. Callers don't always get this
 * right (again, ioremap() on BIOS memory is not uncommon) so this function
 * checks and fixes these known static required protection bits.
 */
T
Thomas Gleixner 已提交
277 278
static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
				   unsigned long pfn)
279 280 281
{
	pgprot_t forbidden = __pgprot(0);

I
Ingo Molnar 已提交
282
	/*
283 284
	 * The BIOS area between 640k and 1Mb needs to be executable for
	 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
I
Ingo Molnar 已提交
285
	 */
286 287
#ifdef CONFIG_PCI_BIOS
	if (pcibios_enabled && within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
288
		pgprot_val(forbidden) |= _PAGE_NX;
289
#endif
290 291 292

	/*
	 * The kernel text needs to be executable for obvious reasons
T
Thomas Gleixner 已提交
293 294
	 * Does not cover __inittext since that is gone later on. On
	 * 64bit we do not enforce !NX on the low mapping
295 296 297
	 */
	if (within(address, (unsigned long)_text, (unsigned long)_etext))
		pgprot_val(forbidden) |= _PAGE_NX;
298 299

	/*
T
Thomas Gleixner 已提交
300 301
	 * The .rodata section needs to be read-only. Using the pfn
	 * catches all aliases.
302
	 */
303 304
	if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
		   __pa_symbol(__end_rodata) >> PAGE_SHIFT))
305
		pgprot_val(forbidden) |= _PAGE_RW;
306

307
#if defined(CONFIG_X86_64)
308
	/*
309 310 311 312
	 * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
	 * kernel text mappings for the large page aligned text, rodata sections
	 * will be always read-only. For the kernel identity mappings covering
	 * the holes caused by this alignment can be anything that user asks.
313 314 315 316
	 *
	 * This will preserve the large page mappings for kernel text/data
	 * at no extra cost.
	 */
317 318
	if (kernel_set_to_readonly &&
	    within(address, (unsigned long)_text,
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
		   (unsigned long)__end_rodata_hpage_align)) {
		unsigned int level;

		/*
		 * Don't enforce the !RW mapping for the kernel text mapping,
		 * if the current mapping is already using small page mapping.
		 * No need to work hard to preserve large page mappings in this
		 * case.
		 *
		 * This also fixes the Linux Xen paravirt guest boot failure
		 * (because of unexpected read-only mappings for kernel identity
		 * mappings). In this paravirt guest case, the kernel text
		 * mapping and the kernel identity mapping share the same
		 * page-table pages. Thus we can't really use different
		 * protections for the kernel text and identity mappings. Also,
		 * these shared mappings are made of small page mappings.
		 * Thus this don't enforce !RW mapping for small page kernel
		 * text mapping logic will help Linux Xen parvirt guest boot
L
Lucas De Marchi 已提交
337
		 * as well.
338 339 340 341
		 */
		if (lookup_address(address, &level) && (level != PG_LEVEL_4K))
			pgprot_val(forbidden) |= _PAGE_RW;
	}
342 343
#endif

344
	prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
I
Ingo Molnar 已提交
345 346 347 348

	return prot;
}

349 350 351 352 353 354
/*
 * Lookup the page table entry for a virtual address in a specific pgd.
 * Return a pointer to the entry and the level of the mapping.
 */
pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
			     unsigned int *level)
355
{
356
	p4d_t *p4d;
L
Linus Torvalds 已提交
357 358
	pud_t *pud;
	pmd_t *pmd;
359

T
Thomas Gleixner 已提交
360 361
	*level = PG_LEVEL_NONE;

L
Linus Torvalds 已提交
362 363
	if (pgd_none(*pgd))
		return NULL;
I
Ingo Molnar 已提交
364

365 366 367 368 369 370 371 372 373
	p4d = p4d_offset(pgd, address);
	if (p4d_none(*p4d))
		return NULL;

	*level = PG_LEVEL_512G;
	if (p4d_large(*p4d) || !p4d_present(*p4d))
		return (pte_t *)p4d;

	pud = pud_offset(p4d, address);
L
Linus Torvalds 已提交
374 375
	if (pud_none(*pud))
		return NULL;
376 377 378 379 380

	*level = PG_LEVEL_1G;
	if (pud_large(*pud) || !pud_present(*pud))
		return (pte_t *)pud;

L
Linus Torvalds 已提交
381 382 383
	pmd = pmd_offset(pud, address);
	if (pmd_none(*pmd))
		return NULL;
T
Thomas Gleixner 已提交
384 385

	*level = PG_LEVEL_2M;
T
Thomas Gleixner 已提交
386
	if (pmd_large(*pmd) || !pmd_present(*pmd))
L
Linus Torvalds 已提交
387 388
		return (pte_t *)pmd;

T
Thomas Gleixner 已提交
389
	*level = PG_LEVEL_4K;
I
Ingo Molnar 已提交
390

391 392
	return pte_offset_kernel(pmd, address);
}
393 394 395 396 397 398 399 400 401 402 403

/*
 * Lookup the page table entry for a virtual address. Return a pointer
 * to the entry and the level of the mapping.
 *
 * Note: We return pud and pmd either when the entry is marked large
 * or when the present bit is not set. Otherwise we would return a
 * pointer to a nonexisting mapping.
 */
pte_t *lookup_address(unsigned long address, unsigned int *level)
{
404
        return lookup_address_in_pgd(pgd_offset_k(address), address, level);
405
}
406
EXPORT_SYMBOL_GPL(lookup_address);
407

408 409 410 411
static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
				  unsigned int *level)
{
        if (cpa->pgd)
412
		return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
413 414 415 416 417
					       address, level);

        return lookup_address(address, level);
}

418 419 420 421 422 423 424
/*
 * Lookup the PMD entry for a virtual address. Return a pointer to the entry
 * or NULL if not present.
 */
pmd_t *lookup_pmd_address(unsigned long address)
{
	pgd_t *pgd;
425
	p4d_t *p4d;
426 427 428 429 430 431
	pud_t *pud;

	pgd = pgd_offset_k(address);
	if (pgd_none(*pgd))
		return NULL;

432 433 434 435 436
	p4d = p4d_offset(pgd, address);
	if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
		return NULL;

	pud = pud_offset(p4d, address);
437 438 439 440 441 442
	if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
		return NULL;

	return pmd_offset(pud, address);
}

443 444 445 446 447 448 449 450 451 452 453 454 455 456
/*
 * This is necessary because __pa() does not work on some
 * kinds of memory, like vmalloc() or the alloc_remap()
 * areas on 32-bit NUMA systems.  The percpu areas can
 * end up in this kind of memory, for instance.
 *
 * This could be optimized, but it is only intended to be
 * used at inititalization time, and keeping it
 * unoptimized should increase the testing coverage for
 * the more obscure platforms.
 */
phys_addr_t slow_virt_to_phys(void *__virt_addr)
{
	unsigned long virt_addr = (unsigned long)__virt_addr;
457 458
	phys_addr_t phys_addr;
	unsigned long offset;
459 460 461 462 463
	enum pg_level level;
	pte_t *pte;

	pte = lookup_address(virt_addr, &level);
	BUG_ON(!pte);
464

465 466 467 468 469
	/*
	 * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
	 * before being left-shifted PAGE_SHIFT bits -- this trick is to
	 * make 32-PAE kernel work correctly.
	 */
470 471
	switch (level) {
	case PG_LEVEL_1G:
472
		phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
473 474 475
		offset = virt_addr & ~PUD_PAGE_MASK;
		break;
	case PG_LEVEL_2M:
476
		phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
477 478 479
		offset = virt_addr & ~PMD_PAGE_MASK;
		break;
	default:
480
		phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
481 482 483 484
		offset = virt_addr & ~PAGE_MASK;
	}

	return (phys_addr_t)(phys_addr | offset);
485 486 487
}
EXPORT_SYMBOL_GPL(slow_virt_to_phys);

I
Ingo Molnar 已提交
488 489 490
/*
 * Set the new pmd in all the pgds we know about:
 */
I
Ingo Molnar 已提交
491
static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
492 493 494
{
	/* change init_mm */
	set_pte_atomic(kpte, pte);
495
#ifdef CONFIG_X86_32
496
	if (!SHARED_KERNEL_PMD) {
497 498
		struct page *page;

499
		list_for_each_entry(page, &pgd_list, lru) {
500
			pgd_t *pgd;
501
			p4d_t *p4d;
502 503 504 505
			pud_t *pud;
			pmd_t *pmd;

			pgd = (pgd_t *)page_address(page) + pgd_index(address);
506 507
			p4d = p4d_offset(pgd, address);
			pud = pud_offset(p4d, address);
508 509 510
			pmd = pmd_offset(pud, address);
			set_pte_atomic((pte_t *)pmd, pte);
		}
L
Linus Torvalds 已提交
511
	}
512
#endif
L
Linus Torvalds 已提交
513 514
}

I
Ingo Molnar 已提交
515 516 517
static int
try_preserve_large_page(pte_t *kpte, unsigned long address,
			struct cpa_data *cpa)
518
{
519
	unsigned long nextpage_addr, numpages, pmask, psize, addr, pfn, old_pfn;
520
	pte_t new_pte, old_pte, *tmp;
521
	pgprot_t old_prot, new_prot, req_prot;
522
	int i, do_split = 1;
523
	enum pg_level level;
524

525 526 527
	if (cpa->force_split)
		return 1;

A
Andrea Arcangeli 已提交
528
	spin_lock(&pgd_lock);
529 530 531 532
	/*
	 * Check for races, another CPU might have split this page
	 * up already:
	 */
533
	tmp = _lookup_address_cpa(cpa, address, &level);
534 535 536 537 538
	if (tmp != kpte)
		goto out_unlock;

	switch (level) {
	case PG_LEVEL_2M:
539 540 541
		old_prot = pmd_pgprot(*(pmd_t *)kpte);
		old_pfn = pmd_pfn(*(pmd_t *)kpte);
		break;
542
	case PG_LEVEL_1G:
543 544
		old_prot = pud_pgprot(*(pud_t *)kpte);
		old_pfn = pud_pfn(*(pud_t *)kpte);
545
		break;
546
	default:
I
Ingo Molnar 已提交
547
		do_split = -EINVAL;
548 549 550
		goto out_unlock;
	}

551 552 553
	psize = page_level_size(level);
	pmask = page_level_mask(level);

554 555 556 557 558 559
	/*
	 * Calculate the number of pages, which fit into this large
	 * page starting at address:
	 */
	nextpage_addr = (address + psize) & pmask;
	numpages = (nextpage_addr - address) >> PAGE_SHIFT;
560 561
	if (numpages < cpa->numpages)
		cpa->numpages = numpages;
562 563 564

	/*
	 * We are safe now. Check whether the new pgprot is the same:
565 566
	 * Convert protection attributes to 4k-format, as cpa->mask* are set
	 * up accordingly.
567 568
	 */
	old_pte = *kpte;
569
	req_prot = pgprot_large_2_4k(old_prot);
570

571 572
	pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
	pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
T
Thomas Gleixner 已提交
573

574 575 576 577 578 579 580
	/*
	 * req_prot is in format of 4k pages. It must be converted to large
	 * page format: the caching mode includes the PAT bit located at
	 * different bit positions in the two formats.
	 */
	req_prot = pgprot_4k_2_large(req_prot);

581 582 583 584 585 586
	/*
	 * Set the PSE and GLOBAL flags only if the PRESENT flag is
	 * set otherwise pmd_present/pmd_huge will return true even on
	 * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
	 * for the ancient hardware that doesn't support it.
	 */
587 588
	if (pgprot_val(req_prot) & _PAGE_PRESENT)
		pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
589
	else
590
		pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
591

592
	req_prot = canon_pgprot(req_prot);
593

T
Thomas Gleixner 已提交
594
	/*
595
	 * old_pfn points to the large page base pfn. So we need
T
Thomas Gleixner 已提交
596 597
	 * to add the offset of the virtual address:
	 */
598
	pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
T
Thomas Gleixner 已提交
599 600
	cpa->pfn = pfn;

601
	new_prot = static_protections(req_prot, address, pfn);
602

603 604 605 606 607
	/*
	 * We need to check the full range, whether
	 * static_protection() requires a different pgprot for one of
	 * the pages in the range we try to preserve:
	 */
608
	addr = address & pmask;
609
	pfn = old_pfn;
610 611
	for (i = 0; i < (psize >> PAGE_SHIFT); i++, addr += PAGE_SIZE, pfn++) {
		pgprot_t chk_prot = static_protections(req_prot, addr, pfn);
612 613 614 615 616

		if (pgprot_val(chk_prot) != pgprot_val(new_prot))
			goto out_unlock;
	}

617 618 619 620 621
	/*
	 * If there are no changes, return. maxpages has been updated
	 * above:
	 */
	if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
I
Ingo Molnar 已提交
622
		do_split = 0;
623 624 625 626 627 628 629 630 631 632 633
		goto out_unlock;
	}

	/*
	 * We need to change the attributes. Check, whether we can
	 * change the large page in one go. We request a split, when
	 * the address is not aligned and the number of pages is
	 * smaller than the number of pages in the large page. Note
	 * that we limited the number of possible pages already to
	 * the number of pages in the large page.
	 */
634
	if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) {
635 636 637 638
		/*
		 * The address is aligned and the number of pages
		 * covers the full page.
		 */
639
		new_pte = pfn_pte(old_pfn, new_prot);
640
		__set_pmd_pte(kpte, address, new_pte);
641
		cpa->flags |= CPA_FLUSHTLB;
I
Ingo Molnar 已提交
642
		do_split = 0;
643 644 645
	}

out_unlock:
A
Andrea Arcangeli 已提交
646
	spin_unlock(&pgd_lock);
I
Ingo Molnar 已提交
647

I
Ingo Molnar 已提交
648
	return do_split;
649 650
}

651
static int
652 653
__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
		   struct page *base)
654
{
655
	pte_t *pbase = (pte_t *)page_address(base);
656
	unsigned long ref_pfn, pfn, pfninc = 1;
I
Ingo Molnar 已提交
657
	unsigned int i, level;
658
	pte_t *tmp;
I
Ingo Molnar 已提交
659
	pgprot_t ref_prot;
660

A
Andrea Arcangeli 已提交
661
	spin_lock(&pgd_lock);
662 663 664 665
	/*
	 * Check for races, another CPU might have split this page
	 * up for us already:
	 */
666
	tmp = _lookup_address_cpa(cpa, address, &level);
667 668 669 670
	if (tmp != kpte) {
		spin_unlock(&pgd_lock);
		return 1;
	}
671

672
	paravirt_alloc_pte(&init_mm, page_to_pfn(base));
673

674 675 676 677
	switch (level) {
	case PG_LEVEL_2M:
		ref_prot = pmd_pgprot(*(pmd_t *)kpte);
		/* clear PSE and promote PAT bit to correct position */
678
		ref_prot = pgprot_large_2_4k(ref_prot);
679 680
		ref_pfn = pmd_pfn(*(pmd_t *)kpte);
		break;
681

682 683 684
	case PG_LEVEL_1G:
		ref_prot = pud_pgprot(*(pud_t *)kpte);
		ref_pfn = pud_pfn(*(pud_t *)kpte);
685
		pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
686

687
		/*
688
		 * Clear the PSE flags if the PRESENT flag is not set
689 690 691
		 * otherwise pmd_present/pmd_huge will return true
		 * even on a non present pmd.
		 */
692
		if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
693
			pgprot_val(ref_prot) &= ~_PAGE_PSE;
694 695 696 697 698
		break;

	default:
		spin_unlock(&pgd_lock);
		return 1;
699 700
	}

701 702 703 704 705 706 707 708 709 710 711
	/*
	 * Set the GLOBAL flags only if the PRESENT flag is set
	 * otherwise pmd/pte_present will return true even on a non
	 * present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL
	 * for the ancient hardware that doesn't support it.
	 */
	if (pgprot_val(ref_prot) & _PAGE_PRESENT)
		pgprot_val(ref_prot) |= _PAGE_GLOBAL;
	else
		pgprot_val(ref_prot) &= ~_PAGE_GLOBAL;

712 713 714
	/*
	 * Get the target pfn from the original entry:
	 */
715
	pfn = ref_pfn;
716
	for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
717
		set_pte(&pbase[i], pfn_pte(pfn, canon_pgprot(ref_prot)));
718

719 720 721 722 723 724
	if (virt_addr_valid(address)) {
		unsigned long pfn = PFN_DOWN(__pa(address));

		if (pfn_range_is_mapped(pfn, pfn + 1))
			split_page_count(level);
	}
725

726
	/*
727
	 * Install the new, split up pagetable.
728
	 *
729 730 731
	 * We use the standard kernel pagetable protections for the new
	 * pagetable protections, the actual ptes set above control the
	 * primary protection behavior:
732
	 */
733
	__set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
734 735 736 737 738 739 740 741 742 743

	/*
	 * Intel Atom errata AAH41 workaround.
	 *
	 * The real fix should be in hw or in a microcode update, but
	 * we also probabilistically try to reduce the window of having
	 * a large TLB mixed with 4K TLBs while instruction fetches are
	 * going on.
	 */
	__flush_tlb_all();
744
	spin_unlock(&pgd_lock);
745

746 747
	return 0;
}
748

749 750
static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
			    unsigned long address)
751 752 753
{
	struct page *base;

754
	if (!debug_pagealloc_enabled())
755 756
		spin_unlock(&cpa_lock);
	base = alloc_pages(GFP_KERNEL | __GFP_NOTRACK, 0);
757
	if (!debug_pagealloc_enabled())
758 759 760 761
		spin_lock(&cpa_lock);
	if (!base)
		return -ENOMEM;

762
	if (__split_large_page(cpa, kpte, address, base))
S
Suresh Siddha 已提交
763
		__free_page(base);
764 765 766 767

	return 0;
}

768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
static bool try_to_free_pte_page(pte_t *pte)
{
	int i;

	for (i = 0; i < PTRS_PER_PTE; i++)
		if (!pte_none(pte[i]))
			return false;

	free_page((unsigned long)pte);
	return true;
}

static bool try_to_free_pmd_page(pmd_t *pmd)
{
	int i;

	for (i = 0; i < PTRS_PER_PMD; i++)
		if (!pmd_none(pmd[i]))
			return false;

	free_page((unsigned long)pmd);
	return true;
}

static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
{
	pte_t *pte = pte_offset_kernel(pmd, start);

	while (start < end) {
		set_pte(pte, __pte(0));

		start += PAGE_SIZE;
		pte++;
	}

	if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
		pmd_clear(pmd);
		return true;
	}
	return false;
}

static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
			      unsigned long start, unsigned long end)
{
	if (unmap_pte_range(pmd, start, end))
		if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
			pud_clear(pud);
}

static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
{
	pmd_t *pmd = pmd_offset(pud, start);

	/*
	 * Not on a 2MB page boundary?
	 */
	if (start & (PMD_SIZE - 1)) {
		unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
		unsigned long pre_end = min_t(unsigned long, end, next_page);

		__unmap_pmd_range(pud, pmd, start, pre_end);

		start = pre_end;
		pmd++;
	}

	/*
	 * Try to unmap in 2M chunks.
	 */
	while (end - start >= PMD_SIZE) {
		if (pmd_large(*pmd))
			pmd_clear(pmd);
		else
			__unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);

		start += PMD_SIZE;
		pmd++;
	}

	/*
	 * 4K leftovers?
	 */
	if (start < end)
		return __unmap_pmd_range(pud, pmd, start, end);

	/*
	 * Try again to free the PMD page if haven't succeeded above.
	 */
	if (!pud_none(*pud))
		if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
			pud_clear(pud);
}
861

862
static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
863
{
864
	pud_t *pud = pud_offset(p4d, start);
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904

	/*
	 * Not on a GB page boundary?
	 */
	if (start & (PUD_SIZE - 1)) {
		unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
		unsigned long pre_end	= min_t(unsigned long, end, next_page);

		unmap_pmd_range(pud, start, pre_end);

		start = pre_end;
		pud++;
	}

	/*
	 * Try to unmap in 1G chunks?
	 */
	while (end - start >= PUD_SIZE) {

		if (pud_large(*pud))
			pud_clear(pud);
		else
			unmap_pmd_range(pud, start, start + PUD_SIZE);

		start += PUD_SIZE;
		pud++;
	}

	/*
	 * 2M leftovers?
	 */
	if (start < end)
		unmap_pmd_range(pud, start, end);

	/*
	 * No need to try to free the PUD page because we'll free it in
	 * populate_pgd's error path
	 */
}

905 906 907 908 909 910 911 912 913 914
static int alloc_pte_page(pmd_t *pmd)
{
	pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
	if (!pte)
		return -1;

	set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
	return 0;
}

915 916 917 918 919 920 921 922 923 924
static int alloc_pmd_page(pud_t *pud)
{
	pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
	if (!pmd)
		return -1;

	set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
	return 0;
}

925 926 927 928 929 930 931 932
static void populate_pte(struct cpa_data *cpa,
			 unsigned long start, unsigned long end,
			 unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
{
	pte_t *pte;

	pte = pte_offset_kernel(pmd, start);

933 934 935 936 937 938 939 940 941 942 943
	/*
	 * Set the GLOBAL flags only if the PRESENT flag is
	 * set otherwise pte_present will return true even on
	 * a non present pte. The canon_pgprot will clear
	 * _PAGE_GLOBAL for the ancient hardware that doesn't
	 * support it.
	 */
	if (pgprot_val(pgprot) & _PAGE_PRESENT)
		pgprot_val(pgprot) |= _PAGE_GLOBAL;
	else
		pgprot_val(pgprot) &= ~_PAGE_GLOBAL;
944

945
	pgprot = canon_pgprot(pgprot);
946 947

	while (num_pages-- && start < end) {
948
		set_pte(pte, pfn_pte(cpa->pfn, pgprot));
949 950

		start	 += PAGE_SIZE;
951
		cpa->pfn++;
952 953 954
		pte++;
	}
}
955

956 957 958
static long populate_pmd(struct cpa_data *cpa,
			 unsigned long start, unsigned long end,
			 unsigned num_pages, pud_t *pud, pgprot_t pgprot)
959
{
960
	long cur_pages = 0;
961
	pmd_t *pmd;
962
	pgprot_t pmd_pgprot;
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993

	/*
	 * Not on a 2M boundary?
	 */
	if (start & (PMD_SIZE - 1)) {
		unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
		unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;

		pre_end   = min_t(unsigned long, pre_end, next_page);
		cur_pages = (pre_end - start) >> PAGE_SHIFT;
		cur_pages = min_t(unsigned int, num_pages, cur_pages);

		/*
		 * Need a PTE page?
		 */
		pmd = pmd_offset(pud, start);
		if (pmd_none(*pmd))
			if (alloc_pte_page(pmd))
				return -1;

		populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);

		start = pre_end;
	}

	/*
	 * We mapped them all?
	 */
	if (num_pages == cur_pages)
		return cur_pages;

994 995
	pmd_pgprot = pgprot_4k_2_large(pgprot);

996 997 998 999 1000 1001 1002 1003 1004 1005 1006
	while (end - start >= PMD_SIZE) {

		/*
		 * We cannot use a 1G page so allocate a PMD page if needed.
		 */
		if (pud_none(*pud))
			if (alloc_pmd_page(pud))
				return -1;

		pmd = pmd_offset(pud, start);

1007
		set_pmd(pmd, __pmd(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
1008
				   massage_pgprot(pmd_pgprot)));
1009 1010

		start	  += PMD_SIZE;
1011
		cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
		cur_pages += PMD_SIZE >> PAGE_SHIFT;
	}

	/*
	 * Map trailing 4K pages.
	 */
	if (start < end) {
		pmd = pmd_offset(pud, start);
		if (pmd_none(*pmd))
			if (alloc_pte_page(pmd))
				return -1;

		populate_pte(cpa, start, end, num_pages - cur_pages,
			     pmd, pgprot);
	}
	return num_pages;
}
1029

1030 1031
static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
			pgprot_t pgprot)
1032 1033 1034
{
	pud_t *pud;
	unsigned long end;
1035
	long cur_pages = 0;
1036
	pgprot_t pud_pgprot;
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051

	end = start + (cpa->numpages << PAGE_SHIFT);

	/*
	 * Not on a Gb page boundary? => map everything up to it with
	 * smaller pages.
	 */
	if (start & (PUD_SIZE - 1)) {
		unsigned long pre_end;
		unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;

		pre_end   = min_t(unsigned long, end, next_page);
		cur_pages = (pre_end - start) >> PAGE_SHIFT;
		cur_pages = min_t(int, (int)cpa->numpages, cur_pages);

1052
		pud = pud_offset(p4d, start);
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072

		/*
		 * Need a PMD page?
		 */
		if (pud_none(*pud))
			if (alloc_pmd_page(pud))
				return -1;

		cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
					 pud, pgprot);
		if (cur_pages < 0)
			return cur_pages;

		start = pre_end;
	}

	/* We mapped them all? */
	if (cpa->numpages == cur_pages)
		return cur_pages;

1073
	pud = pud_offset(p4d, start);
1074
	pud_pgprot = pgprot_4k_2_large(pgprot);
1075 1076 1077 1078

	/*
	 * Map everything starting from the Gb boundary, possibly with 1G pages
	 */
1079
	while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
1080
		set_pud(pud, __pud(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
1081
				   massage_pgprot(pud_pgprot)));
1082 1083

		start	  += PUD_SIZE;
1084
		cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1085 1086 1087 1088 1089 1090
		cur_pages += PUD_SIZE >> PAGE_SHIFT;
		pud++;
	}

	/* Map trailing leftover */
	if (start < end) {
1091
		long tmp;
1092

1093
		pud = pud_offset(p4d, start);
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
		if (pud_none(*pud))
			if (alloc_pmd_page(pud))
				return -1;

		tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
				   pud, pgprot);
		if (tmp < 0)
			return cur_pages;

		cur_pages += tmp;
	}
	return cur_pages;
}
1107 1108 1109 1110 1111 1112 1113 1114 1115

/*
 * Restrictions for kernel page table do not necessarily apply when mapping in
 * an alternate PGD.
 */
static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
{
	pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
	pud_t *pud = NULL;	/* shut up gcc */
1116
	p4d_t *p4d;
1117
	pgd_t *pgd_entry;
1118
	long ret;
1119 1120 1121

	pgd_entry = cpa->pgd + pgd_index(addr);

1122 1123 1124 1125 1126 1127 1128 1129
	if (pgd_none(*pgd_entry)) {
		p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
		if (!p4d)
			return -1;

		set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
	}

1130 1131 1132
	/*
	 * Allocate a PUD page and hand it down for mapping.
	 */
1133 1134
	p4d = p4d_offset(pgd_entry, addr);
	if (p4d_none(*p4d)) {
1135 1136 1137
		pud = (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
		if (!pud)
			return -1;
1138

1139
		set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
1140 1141 1142 1143 1144
	}

	pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
	pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);

1145
	ret = populate_pud(cpa, addr, p4d, pgprot);
1146
	if (ret < 0) {
1147 1148 1149 1150 1151
		/*
		 * Leave the PUD page in place in case some other CPU or thread
		 * already found it, but remove any useless entries we just
		 * added to it.
		 */
1152
		unmap_pud_range(p4d, addr,
1153
				addr + (cpa->numpages << PAGE_SHIFT));
1154
		return ret;
1155
	}
1156

1157 1158 1159 1160
	cpa->numpages = ret;
	return 0;
}

1161 1162 1163
static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
			       int primary)
{
1164 1165 1166 1167 1168 1169
	if (cpa->pgd) {
		/*
		 * Right now, we only execute this code path when mapping
		 * the EFI virtual memory map regions, no other users
		 * provide a ->pgd value. This may change in the future.
		 */
1170
		return populate_pgd(cpa, vaddr);
1171
	}
1172

1173 1174 1175
	/*
	 * Ignore all non primary paths.
	 */
1176 1177
	if (!primary) {
		cpa->numpages = 1;
1178
		return 0;
1179
	}
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201

	/*
	 * Ignore the NULL PTE for kernel identity mapping, as it is expected
	 * to have holes.
	 * Also set numpages to '1' indicating that we processed cpa req for
	 * one virtual address page and its pfn. TBD: numpages can be set based
	 * on the initial value and the level returned by lookup_address().
	 */
	if (within(vaddr, PAGE_OFFSET,
		   PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
		cpa->numpages = 1;
		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
		return 0;
	} else {
		WARN(1, KERN_WARNING "CPA: called for zero pte. "
			"vaddr = %lx cpa->vaddr = %lx\n", vaddr,
			*cpa->vaddr);

		return -EFAULT;
	}
}

T
Thomas Gleixner 已提交
1202
static int __change_page_attr(struct cpa_data *cpa, int primary)
1203
{
1204
	unsigned long address;
1205 1206
	int do_split, err;
	unsigned int level;
T
Thomas Gleixner 已提交
1207
	pte_t *kpte, old_pte;
L
Linus Torvalds 已提交
1208

1209 1210 1211 1212 1213 1214
	if (cpa->flags & CPA_PAGES_ARRAY) {
		struct page *page = cpa->pages[cpa->curpage];
		if (unlikely(PageHighMem(page)))
			return 0;
		address = (unsigned long)page_address(page);
	} else if (cpa->flags & CPA_ARRAY)
1215 1216 1217
		address = cpa->vaddr[cpa->curpage];
	else
		address = *cpa->vaddr;
1218
repeat:
1219
	kpte = _lookup_address_cpa(cpa, address, &level);
L
Linus Torvalds 已提交
1220
	if (!kpte)
1221
		return __cpa_process_fault(cpa, address, primary);
T
Thomas Gleixner 已提交
1222 1223

	old_pte = *kpte;
1224
	if (pte_none(old_pte))
1225
		return __cpa_process_fault(cpa, address, primary);
1226

T
Thomas Gleixner 已提交
1227
	if (level == PG_LEVEL_4K) {
T
Thomas Gleixner 已提交
1228
		pte_t new_pte;
1229
		pgprot_t new_prot = pte_pgprot(old_pte);
T
Thomas Gleixner 已提交
1230
		unsigned long pfn = pte_pfn(old_pte);
I
Ingo Molnar 已提交
1231

T
Thomas Gleixner 已提交
1232 1233
		pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
		pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
I
Ingo Molnar 已提交
1234

T
Thomas Gleixner 已提交
1235
		new_prot = static_protections(new_prot, address, pfn);
I
Ingo Molnar 已提交
1236

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
		/*
		 * Set the GLOBAL flags only if the PRESENT flag is
		 * set otherwise pte_present will return true even on
		 * a non present pte. The canon_pgprot will clear
		 * _PAGE_GLOBAL for the ancient hardware that doesn't
		 * support it.
		 */
		if (pgprot_val(new_prot) & _PAGE_PRESENT)
			pgprot_val(new_prot) |= _PAGE_GLOBAL;
		else
			pgprot_val(new_prot) &= ~_PAGE_GLOBAL;

1249 1250 1251 1252 1253
		/*
		 * We need to keep the pfn from the existing PTE,
		 * after all we're only going to change it's attributes
		 * not the memory it points to
		 */
T
Thomas Gleixner 已提交
1254 1255
		new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
		cpa->pfn = pfn;
1256 1257 1258 1259 1260
		/*
		 * Do we really change anything ?
		 */
		if (pte_val(old_pte) != pte_val(new_pte)) {
			set_pte_atomic(kpte, new_pte);
1261
			cpa->flags |= CPA_FLUSHTLB;
1262
		}
1263
		cpa->numpages = 1;
1264
		return 0;
L
Linus Torvalds 已提交
1265
	}
1266 1267 1268 1269 1270

	/*
	 * Check, whether we can keep the large page intact
	 * and just change the pte:
	 */
I
Ingo Molnar 已提交
1271
	do_split = try_preserve_large_page(kpte, address, cpa);
1272 1273
	/*
	 * When the range fits into the existing large page,
1274
	 * return. cp->numpages and cpa->tlbflush have been updated in
1275 1276
	 * try_large_page:
	 */
I
Ingo Molnar 已提交
1277 1278
	if (do_split <= 0)
		return do_split;
1279 1280 1281 1282

	/*
	 * We have to split the large page:
	 */
1283
	err = split_large_page(cpa, kpte, address);
I
Ingo Molnar 已提交
1284
	if (!err) {
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
		/*
	 	 * Do a global flush tlb after splitting the large page
	 	 * and before we do the actual change page attribute in the PTE.
	 	 *
	 	 * With out this, we violate the TLB application note, that says
	 	 * "The TLBs may contain both ordinary and large-page
		 *  translations for a 4-KByte range of linear addresses. This
		 *  may occur if software modifies the paging structures so that
		 *  the page size used for the address range changes. If the two
		 *  translations differ with respect to page frame or attributes
		 *  (e.g., permissions), processor behavior is undefined and may
		 *  be implementation-specific."
	 	 *
	 	 * We do this global tlb flush inside the cpa_lock, so that we
		 * don't allow any other cpu, with stale tlb entries change the
		 * page attribute in parallel, that also falls into the
		 * just split large page entry.
	 	 */
		flush_tlb_all();
I
Ingo Molnar 已提交
1304 1305
		goto repeat;
	}
I
Ingo Molnar 已提交
1306

I
Ingo Molnar 已提交
1307
	return err;
1308
}
L
Linus Torvalds 已提交
1309

T
Thomas Gleixner 已提交
1310 1311 1312
static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);

static int cpa_process_alias(struct cpa_data *cpa)
L
Linus Torvalds 已提交
1313
{
T
Thomas Gleixner 已提交
1314
	struct cpa_data alias_cpa;
T
Tejun Heo 已提交
1315
	unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1316
	unsigned long vaddr;
T
Tejun Heo 已提交
1317
	int ret;
1318

1319
	if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
T
Thomas Gleixner 已提交
1320
		return 0;
1321

1322 1323 1324 1325
	/*
	 * No need to redo, when the primary call touched the direct
	 * mapping already:
	 */
1326 1327 1328 1329 1330 1331
	if (cpa->flags & CPA_PAGES_ARRAY) {
		struct page *page = cpa->pages[cpa->curpage];
		if (unlikely(PageHighMem(page)))
			return 0;
		vaddr = (unsigned long)page_address(page);
	} else if (cpa->flags & CPA_ARRAY)
1332 1333 1334 1335 1336
		vaddr = cpa->vaddr[cpa->curpage];
	else
		vaddr = *cpa->vaddr;

	if (!(within(vaddr, PAGE_OFFSET,
1337
		    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
1338

1339
		alias_cpa = *cpa;
T
Tejun Heo 已提交
1340
		alias_cpa.vaddr = &laddr;
1341
		alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1342

1343
		ret = __change_page_attr_set_clr(&alias_cpa, 0);
T
Tejun Heo 已提交
1344 1345
		if (ret)
			return ret;
1346
	}
1347 1348

#ifdef CONFIG_X86_64
A
Arjan van de Ven 已提交
1349
	/*
T
Tejun Heo 已提交
1350 1351
	 * If the primary call didn't touch the high mapping already
	 * and the physical address is inside the kernel map, we need
1352
	 * to touch the high mapped kernel as well:
A
Arjan van de Ven 已提交
1353
	 */
T
Tejun Heo 已提交
1354
	if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1355 1356
	    within_inclusive(cpa->pfn, highmap_start_pfn(),
			     highmap_end_pfn())) {
T
Tejun Heo 已提交
1357 1358 1359 1360 1361
		unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
					       __START_KERNEL_map - phys_base;
		alias_cpa = *cpa;
		alias_cpa.vaddr = &temp_cpa_vaddr;
		alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
T
Thomas Gleixner 已提交
1362

T
Tejun Heo 已提交
1363 1364 1365 1366 1367 1368
		/*
		 * The high mapping range is imprecise, so ignore the
		 * return value.
		 */
		__change_page_attr_set_clr(&alias_cpa, 0);
	}
A
Arjan van de Ven 已提交
1369
#endif
T
Tejun Heo 已提交
1370 1371

	return 0;
L
Linus Torvalds 已提交
1372 1373
}

T
Thomas Gleixner 已提交
1374
static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
1375
{
1376 1377
	unsigned long numpages = cpa->numpages;
	int ret;
1378

1379 1380 1381 1382 1383
	while (numpages) {
		/*
		 * Store the remaining nr of pages for the large page
		 * preservation check.
		 */
1384
		cpa->numpages = numpages;
1385
		/* for array changes, we can't use large page */
1386
		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1387
			cpa->numpages = 1;
T
Thomas Gleixner 已提交
1388

1389
		if (!debug_pagealloc_enabled())
1390
			spin_lock(&cpa_lock);
T
Thomas Gleixner 已提交
1391
		ret = __change_page_attr(cpa, checkalias);
1392
		if (!debug_pagealloc_enabled())
1393
			spin_unlock(&cpa_lock);
1394 1395 1396
		if (ret)
			return ret;

T
Thomas Gleixner 已提交
1397 1398 1399 1400 1401 1402
		if (checkalias) {
			ret = cpa_process_alias(cpa);
			if (ret)
				return ret;
		}

1403 1404 1405 1406 1407
		/*
		 * Adjust the number of pages with the result of the
		 * CPA operation. Either a large page has been
		 * preserved or a single page update happened.
		 */
1408
		BUG_ON(cpa->numpages > numpages || !cpa->numpages);
1409
		numpages -= cpa->numpages;
1410
		if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
1411 1412 1413 1414
			cpa->curpage++;
		else
			*cpa->vaddr += cpa->numpages * PAGE_SIZE;

1415
	}
1416 1417 1418
	return 0;
}

1419
static int change_page_attr_set_clr(unsigned long *addr, int numpages,
1420
				    pgprot_t mask_set, pgprot_t mask_clr,
1421 1422
				    int force_split, int in_flag,
				    struct page **pages)
1423
{
T
Thomas Gleixner 已提交
1424
	struct cpa_data cpa;
1425
	int ret, cache, checkalias;
1426
	unsigned long baddr = 0;
1427

1428 1429
	memset(&cpa, 0, sizeof(cpa));

1430 1431 1432 1433 1434 1435
	/*
	 * Check, if we are requested to change a not supported
	 * feature:
	 */
	mask_set = canon_pgprot(mask_set);
	mask_clr = canon_pgprot(mask_clr);
1436
	if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
1437 1438
		return 0;

1439
	/* Ensure we are PAGE_SIZE aligned */
1440
	if (in_flag & CPA_ARRAY) {
1441 1442 1443 1444 1445 1446 1447
		int i;
		for (i = 0; i < numpages; i++) {
			if (addr[i] & ~PAGE_MASK) {
				addr[i] &= PAGE_MASK;
				WARN_ON_ONCE(1);
			}
		}
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	} else if (!(in_flag & CPA_PAGES_ARRAY)) {
		/*
		 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
		 * No need to cehck in that case
		 */
		if (*addr & ~PAGE_MASK) {
			*addr &= PAGE_MASK;
			/*
			 * People should not be passing in unaligned addresses:
			 */
			WARN_ON_ONCE(1);
		}
1460 1461 1462 1463 1464
		/*
		 * Save address for cache flush. *addr is modified in the call
		 * to __change_page_attr_set_clr() below.
		 */
		baddr = *addr;
1465 1466
	}

1467 1468 1469
	/* Must avoid aliasing mappings in the highmem code */
	kmap_flush_unused();

N
Nick Piggin 已提交
1470 1471
	vm_unmap_aliases();

T
Thomas Gleixner 已提交
1472
	cpa.vaddr = addr;
1473
	cpa.pages = pages;
T
Thomas Gleixner 已提交
1474 1475 1476
	cpa.numpages = numpages;
	cpa.mask_set = mask_set;
	cpa.mask_clr = mask_clr;
1477 1478
	cpa.flags = 0;
	cpa.curpage = 0;
1479
	cpa.force_split = force_split;
T
Thomas Gleixner 已提交
1480

1481 1482
	if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
		cpa.flags |= in_flag;
1483

1484 1485 1486 1487
	/* No alias checking for _NX bit modifications */
	checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;

	ret = __change_page_attr_set_clr(&cpa, checkalias);
1488

1489 1490 1491
	/*
	 * Check whether we really changed something:
	 */
1492
	if (!(cpa.flags & CPA_FLUSHTLB))
1493
		goto out;
1494

1495 1496 1497 1498
	/*
	 * No need to flush, when we did not set any of the caching
	 * attributes:
	 */
1499
	cache = !!pgprot2cachemode(mask_set);
1500

1501
	/*
1502 1503
	 * On success we use CLFLUSH, when the CPU supports it to
	 * avoid the WBINVD. If the CPU does not support it and in the
1504
	 * error case we fall back to cpa_flush_all (which uses
1505
	 * WBINVD):
1506
	 */
1507
	if (!ret && boot_cpu_has(X86_FEATURE_CLFLUSH)) {
1508 1509 1510 1511
		if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
			cpa_flush_array(addr, numpages, cache,
					cpa.flags, pages);
		} else
1512
			cpa_flush_range(baddr, numpages, cache);
1513
	} else
1514
		cpa_flush_all(cache);
1515

1516
out:
1517 1518 1519
	return ret;
}

1520 1521
static inline int change_page_attr_set(unsigned long *addr, int numpages,
				       pgprot_t mask, int array)
1522
{
1523
	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
1524
		(array ? CPA_ARRAY : 0), NULL);
1525 1526
}

1527 1528
static inline int change_page_attr_clear(unsigned long *addr, int numpages,
					 pgprot_t mask, int array)
1529
{
1530
	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
1531
		(array ? CPA_ARRAY : 0), NULL);
1532 1533
}

1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547
static inline int cpa_set_pages_array(struct page **pages, int numpages,
				       pgprot_t mask)
{
	return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
		CPA_PAGES_ARRAY, pages);
}

static inline int cpa_clear_pages_array(struct page **pages, int numpages,
					 pgprot_t mask)
{
	return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
		CPA_PAGES_ARRAY, pages);
}

1548
int _set_memory_uc(unsigned long addr, int numpages)
1549
{
1550 1551
	/*
	 * for now UC MINUS. see comments in ioremap_nocache()
1552 1553 1554
	 * If you really need strong UC use ioremap_uc(), but note
	 * that you cannot override IO areas with set_memory_*() as
	 * these helpers cannot work with IO memory.
1555
	 */
1556
	return change_page_attr_set(&addr, numpages,
1557 1558
				    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
				    0);
1559
}
1560 1561 1562

int set_memory_uc(unsigned long addr, int numpages)
{
1563 1564
	int ret;

1565 1566 1567
	/*
	 * for now UC MINUS. see comments in ioremap_nocache()
	 */
1568
	ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1569
			      _PAGE_CACHE_MODE_UC_MINUS, NULL);
1570 1571 1572 1573 1574 1575 1576 1577
	if (ret)
		goto out_err;

	ret = _set_memory_uc(addr, numpages);
	if (ret)
		goto out_free;

	return 0;
1578

1579 1580 1581 1582
out_free:
	free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
out_err:
	return ret;
1583
}
1584 1585
EXPORT_SYMBOL(set_memory_uc);

1586
static int _set_memory_array(unsigned long *addr, int addrinarray,
1587
		enum page_cache_mode new_type)
1588
{
1589
	enum page_cache_mode set_type;
1590 1591 1592
	int i, j;
	int ret;

1593
	for (i = 0; i < addrinarray; i++) {
1594
		ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
1595
					new_type, NULL);
1596 1597
		if (ret)
			goto out_free;
1598 1599
	}

1600 1601 1602 1603
	/* If WC, set to UC- first and then WC */
	set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
				_PAGE_CACHE_MODE_UC_MINUS : new_type;

1604
	ret = change_page_attr_set(addr, addrinarray,
1605
				   cachemode2pgprot(set_type), 1);
1606

1607
	if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1608
		ret = change_page_attr_set_clr(addr, addrinarray,
1609 1610
					       cachemode2pgprot(
						_PAGE_CACHE_MODE_WC),
1611 1612
					       __pgprot(_PAGE_CACHE_MASK),
					       0, CPA_ARRAY, NULL);
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
	if (ret)
		goto out_free;

	return 0;

out_free:
	for (j = 0; j < i; j++)
		free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);

	return ret;
1623
}
1624 1625 1626

int set_memory_array_uc(unsigned long *addr, int addrinarray)
{
1627
	return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
1628
}
1629 1630
EXPORT_SYMBOL(set_memory_array_uc);

1631 1632
int set_memory_array_wc(unsigned long *addr, int addrinarray)
{
1633
	return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC);
1634 1635 1636
}
EXPORT_SYMBOL(set_memory_array_wc);

1637 1638 1639 1640 1641 1642
int set_memory_array_wt(unsigned long *addr, int addrinarray)
{
	return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WT);
}
EXPORT_SYMBOL_GPL(set_memory_array_wt);

1643 1644
int _set_memory_wc(unsigned long addr, int numpages)
{
1645
	int ret;
1646 1647
	unsigned long addr_copy = addr;

1648
	ret = change_page_attr_set(&addr, numpages,
1649 1650
				   cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
				   0);
1651
	if (!ret) {
1652
		ret = change_page_attr_set_clr(&addr_copy, numpages,
1653 1654
					       cachemode2pgprot(
						_PAGE_CACHE_MODE_WC),
1655 1656
					       __pgprot(_PAGE_CACHE_MASK),
					       0, 0, NULL);
1657 1658
	}
	return ret;
1659 1660 1661 1662
}

int set_memory_wc(unsigned long addr, int numpages)
{
1663 1664 1665
	int ret;

	ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1666
		_PAGE_CACHE_MODE_WC, NULL);
1667
	if (ret)
1668
		return ret;
1669

1670 1671
	ret = _set_memory_wc(addr, numpages);
	if (ret)
1672
		free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1673 1674

	return ret;
1675 1676 1677
}
EXPORT_SYMBOL(set_memory_wc);

1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
int _set_memory_wt(unsigned long addr, int numpages)
{
	return change_page_attr_set(&addr, numpages,
				    cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
}

int set_memory_wt(unsigned long addr, int numpages)
{
	int ret;

	ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
			      _PAGE_CACHE_MODE_WT, NULL);
	if (ret)
		return ret;

	ret = _set_memory_wt(addr, numpages);
	if (ret)
		free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);

	return ret;
}
EXPORT_SYMBOL_GPL(set_memory_wt);

1701
int _set_memory_wb(unsigned long addr, int numpages)
1702
{
1703
	/* WB cache mode is hard wired to all cache attribute bits being 0 */
1704 1705
	return change_page_attr_clear(&addr, numpages,
				      __pgprot(_PAGE_CACHE_MASK), 0);
1706
}
1707 1708 1709

int set_memory_wb(unsigned long addr, int numpages)
{
1710 1711 1712 1713 1714 1715
	int ret;

	ret = _set_memory_wb(addr, numpages);
	if (ret)
		return ret;

1716
	free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1717
	return 0;
1718
}
1719 1720
EXPORT_SYMBOL(set_memory_wb);

1721 1722 1723
int set_memory_array_wb(unsigned long *addr, int addrinarray)
{
	int i;
1724 1725
	int ret;

1726
	/* WB cache mode is hard wired to all cache attribute bits being 0 */
1727 1728
	ret = change_page_attr_clear(addr, addrinarray,
				      __pgprot(_PAGE_CACHE_MASK), 1);
1729 1730
	if (ret)
		return ret;
1731

1732 1733
	for (i = 0; i < addrinarray; i++)
		free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
1734

1735
	return 0;
1736 1737 1738
}
EXPORT_SYMBOL(set_memory_array_wb);

1739 1740
int set_memory_x(unsigned long addr, int numpages)
{
1741 1742 1743
	if (!(__supported_pte_mask & _PAGE_NX))
		return 0;

1744
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
1745 1746 1747 1748 1749
}
EXPORT_SYMBOL(set_memory_x);

int set_memory_nx(unsigned long addr, int numpages)
{
1750 1751 1752
	if (!(__supported_pte_mask & _PAGE_NX))
		return 0;

1753
	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
1754 1755 1756 1757 1758
}
EXPORT_SYMBOL(set_memory_nx);

int set_memory_ro(unsigned long addr, int numpages)
{
1759
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
1760 1761 1762 1763
}

int set_memory_rw(unsigned long addr, int numpages)
{
1764
	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
1765
}
I
Ingo Molnar 已提交
1766 1767 1768

int set_memory_np(unsigned long addr, int numpages)
{
1769
	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
I
Ingo Molnar 已提交
1770
}
1771

1772 1773
int set_memory_4k(unsigned long addr, int numpages)
{
1774
	return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1775
					__pgprot(0), 1, 0, NULL);
1776 1777
}

1778 1779 1780 1781 1782 1783
static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
{
	struct cpa_data cpa;
	unsigned long start;
	int ret;

1784 1785
	/* Nothing to do if memory encryption is not active */
	if (!mem_encrypt_active())
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
		return 0;

	/* Should not be working on unaligned addresses */
	if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
		addr &= PAGE_MASK;

	start = addr;

	memset(&cpa, 0, sizeof(cpa));
	cpa.vaddr = &addr;
	cpa.numpages = numpages;
	cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
	cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
	cpa.pgd = init_mm.pgd;

	/* Must avoid aliasing mappings in the highmem code */
	kmap_flush_unused();
	vm_unmap_aliases();

	/*
	 * Before changing the encryption attribute, we need to flush caches.
	 */
	if (static_cpu_has(X86_FEATURE_CLFLUSH))
		cpa_flush_range(start, numpages, 1);
	else
		cpa_flush_all(1);

	ret = __change_page_attr_set_clr(&cpa, 1);

	/*
	 * After changing the encryption attribute, we need to flush TLBs
	 * again in case any speculative TLB caching occurred (but no need
	 * to flush caches again).  We could just use cpa_flush_all(), but
	 * in case TLB flushing gets optimized in the cpa_flush_range()
	 * path use the same logic as above.
	 */
	if (static_cpu_has(X86_FEATURE_CLFLUSH))
		cpa_flush_range(start, numpages, 0);
	else
		cpa_flush_all(0);

	return ret;
}

int set_memory_encrypted(unsigned long addr, int numpages)
{
	return __set_memory_enc_dec(addr, numpages, true);
}
1834
EXPORT_SYMBOL_GPL(set_memory_encrypted);
1835 1836 1837 1838 1839

int set_memory_decrypted(unsigned long addr, int numpages)
{
	return __set_memory_enc_dec(addr, numpages, false);
}
1840
EXPORT_SYMBOL_GPL(set_memory_decrypted);
1841

1842 1843 1844 1845
int set_pages_uc(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1846
	return set_memory_uc(addr, numpages);
1847 1848 1849
}
EXPORT_SYMBOL(set_pages_uc);

1850
static int _set_pages_array(struct page **pages, int addrinarray,
1851
		enum page_cache_mode new_type)
1852 1853 1854
{
	unsigned long start;
	unsigned long end;
1855
	enum page_cache_mode set_type;
1856 1857
	int i;
	int free_idx;
1858
	int ret;
1859 1860

	for (i = 0; i < addrinarray; i++) {
1861 1862 1863
		if (PageHighMem(pages[i]))
			continue;
		start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1864
		end = start + PAGE_SIZE;
1865
		if (reserve_memtype(start, end, new_type, NULL))
1866 1867 1868
			goto err_out;
	}

1869 1870 1871 1872
	/* If WC, set to UC- first and then WC */
	set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
				_PAGE_CACHE_MODE_UC_MINUS : new_type;

1873
	ret = cpa_set_pages_array(pages, addrinarray,
1874
				  cachemode2pgprot(set_type));
1875
	if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1876
		ret = change_page_attr_set_clr(NULL, addrinarray,
1877 1878
					       cachemode2pgprot(
						_PAGE_CACHE_MODE_WC),
1879 1880 1881 1882 1883
					       __pgprot(_PAGE_CACHE_MASK),
					       0, CPA_PAGES_ARRAY, pages);
	if (ret)
		goto err_out;
	return 0; /* Success */
1884 1885 1886
err_out:
	free_idx = i;
	for (i = 0; i < free_idx; i++) {
1887 1888 1889
		if (PageHighMem(pages[i]))
			continue;
		start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1890 1891 1892 1893 1894
		end = start + PAGE_SIZE;
		free_memtype(start, end);
	}
	return -EINVAL;
}
1895 1896 1897

int set_pages_array_uc(struct page **pages, int addrinarray)
{
1898
	return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
1899
}
1900 1901
EXPORT_SYMBOL(set_pages_array_uc);

1902 1903
int set_pages_array_wc(struct page **pages, int addrinarray)
{
1904
	return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC);
1905 1906 1907
}
EXPORT_SYMBOL(set_pages_array_wc);

1908 1909 1910 1911 1912 1913
int set_pages_array_wt(struct page **pages, int addrinarray)
{
	return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WT);
}
EXPORT_SYMBOL_GPL(set_pages_array_wt);

1914 1915 1916 1917
int set_pages_wb(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1918
	return set_memory_wb(addr, numpages);
1919 1920 1921
}
EXPORT_SYMBOL(set_pages_wb);

1922 1923 1924 1925 1926 1927 1928
int set_pages_array_wb(struct page **pages, int addrinarray)
{
	int retval;
	unsigned long start;
	unsigned long end;
	int i;

1929
	/* WB cache mode is hard wired to all cache attribute bits being 0 */
1930 1931
	retval = cpa_clear_pages_array(pages, addrinarray,
			__pgprot(_PAGE_CACHE_MASK));
1932 1933
	if (retval)
		return retval;
1934 1935

	for (i = 0; i < addrinarray; i++) {
1936 1937 1938
		if (PageHighMem(pages[i]))
			continue;
		start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1939 1940 1941 1942
		end = start + PAGE_SIZE;
		free_memtype(start, end);
	}

1943
	return 0;
1944 1945 1946
}
EXPORT_SYMBOL(set_pages_array_wb);

1947 1948 1949 1950
int set_pages_x(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1951
	return set_memory_x(addr, numpages);
1952 1953 1954 1955 1956 1957 1958
}
EXPORT_SYMBOL(set_pages_x);

int set_pages_nx(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1959
	return set_memory_nx(addr, numpages);
1960 1961 1962 1963 1964 1965 1966
}
EXPORT_SYMBOL(set_pages_nx);

int set_pages_ro(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

T
Thomas Gleixner 已提交
1967
	return set_memory_ro(addr, numpages);
1968 1969 1970 1971 1972
}

int set_pages_rw(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);
1973

T
Thomas Gleixner 已提交
1974
	return set_memory_rw(addr, numpages);
I
Ingo Molnar 已提交
1975 1976
}

L
Linus Torvalds 已提交
1977
#ifdef CONFIG_DEBUG_PAGEALLOC
I
Ingo Molnar 已提交
1978 1979 1980

static int __set_pages_p(struct page *page, int numpages)
{
1981 1982
	unsigned long tempaddr = (unsigned long) page_address(page);
	struct cpa_data cpa = { .vaddr = &tempaddr,
1983
				.pgd = NULL,
T
Thomas Gleixner 已提交
1984 1985
				.numpages = numpages,
				.mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1986 1987
				.mask_clr = __pgprot(0),
				.flags = 0};
1988

1989 1990 1991 1992 1993 1994 1995
	/*
	 * No alias checking needed for setting present flag. otherwise,
	 * we may need to break large pages for 64-bit kernel text
	 * mappings (this adds to complexity if we want to do this from
	 * atomic context especially). Let's keep it simple!
	 */
	return __change_page_attr_set_clr(&cpa, 0);
I
Ingo Molnar 已提交
1996 1997 1998 1999
}

static int __set_pages_np(struct page *page, int numpages)
{
2000 2001
	unsigned long tempaddr = (unsigned long) page_address(page);
	struct cpa_data cpa = { .vaddr = &tempaddr,
2002
				.pgd = NULL,
T
Thomas Gleixner 已提交
2003 2004
				.numpages = numpages,
				.mask_set = __pgprot(0),
2005 2006
				.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
				.flags = 0};
2007

2008 2009 2010 2011 2012 2013 2014
	/*
	 * No alias checking needed for setting not present flag. otherwise,
	 * we may need to break large pages for 64-bit kernel text
	 * mappings (this adds to complexity if we want to do this from
	 * atomic context especially). Let's keep it simple!
	 */
	return __change_page_attr_set_clr(&cpa, 0);
I
Ingo Molnar 已提交
2015 2016
}

2017
void __kernel_map_pages(struct page *page, int numpages, int enable)
L
Linus Torvalds 已提交
2018 2019 2020
{
	if (PageHighMem(page))
		return;
2021
	if (!enable) {
2022 2023
		debug_check_no_locks_freed(page_address(page),
					   numpages * PAGE_SIZE);
2024
	}
2025

2026
	/*
I
Ingo Molnar 已提交
2027
	 * The return value is ignored as the calls cannot fail.
2028 2029
	 * Large pages for identity mappings are not used at boot time
	 * and hence no memory allocations during large page split.
L
Linus Torvalds 已提交
2030
	 */
I
Ingo Molnar 已提交
2031 2032 2033 2034
	if (enable)
		__set_pages_p(page, numpages);
	else
		__set_pages_np(page, numpages);
2035 2036

	/*
2037 2038
	 * We should perform an IPI and flush all tlbs,
	 * but that can deadlock->flush only current cpu:
L
Linus Torvalds 已提交
2039 2040
	 */
	__flush_tlb_all();
2041 2042

	arch_flush_lazy_mmu_mode();
2043 2044
}

2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061
#ifdef CONFIG_HIBERNATION

bool kernel_page_present(struct page *page)
{
	unsigned int level;
	pte_t *pte;

	if (PageHighMem(page))
		return false;

	pte = lookup_address((unsigned long)page_address(page), &level);
	return (pte_val(*pte) & _PAGE_PRESENT);
}

#endif /* CONFIG_HIBERNATION */

#endif /* CONFIG_DEBUG_PAGEALLOC */
2062

2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
			    unsigned numpages, unsigned long page_flags)
{
	int retval = -EINVAL;

	struct cpa_data cpa = {
		.vaddr = &address,
		.pfn = pfn,
		.pgd = pgd,
		.numpages = numpages,
		.mask_set = __pgprot(0),
		.mask_clr = __pgprot(0),
		.flags = 0,
	};

	if (!(__supported_pte_mask & _PAGE_NX))
		goto out;

	if (!(page_flags & _PAGE_NX))
		cpa.mask_clr = __pgprot(_PAGE_NX);

2084 2085 2086
	if (!(page_flags & _PAGE_RW))
		cpa.mask_clr = __pgprot(_PAGE_RW);

2087 2088 2089
	if (!(page_flags & _PAGE_ENC))
		cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);

2090 2091 2092 2093 2094 2095 2096 2097 2098
	cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);

	retval = __change_page_attr_set_clr(&cpa, 0);
	__flush_tlb_all();

out:
	return retval;
}

2099 2100 2101 2102 2103 2104 2105
/*
 * The testcases use internal knowledge of the implementation that shouldn't
 * be exposed to the rest of the kernel. Include these directly here.
 */
#ifdef CONFIG_CPA_DEBUG
#include "pageattr-test.c"
#endif