mmu.c 24.9 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Based on arch/arm/mm/mmu.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/cache.h>
C
Catalin Marinas 已提交
21 22 23 24
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
25 26
#include <linux/ioport.h>
#include <linux/kexec.h>
27
#include <linux/libfdt.h>
C
Catalin Marinas 已提交
28 29 30 31
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/memblock.h>
#include <linux/fs.h>
32
#include <linux/io.h>
33
#include <linux/mm.h>
34
#include <linux/vmalloc.h>
C
Catalin Marinas 已提交
35

36
#include <asm/barrier.h>
C
Catalin Marinas 已提交
37
#include <asm/cputype.h>
38
#include <asm/fixmap.h>
39
#include <asm/kasan.h>
40
#include <asm/kernel-pgtable.h>
C
Catalin Marinas 已提交
41 42 43 44
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
45
#include <asm/memblock.h>
C
Catalin Marinas 已提交
46
#include <asm/mmu_context.h>
47
#include <asm/ptdump.h>
C
Catalin Marinas 已提交
48

49
#define NO_BLOCK_MAPPINGS	BIT(0)
50
#define NO_CONT_MAPPINGS	BIT(1)
51

52
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
53
u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
54

55
u64 kimage_voffset __ro_after_init;
56 57
EXPORT_SYMBOL(kimage_voffset);

C
Catalin Marinas 已提交
58 59 60 61
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
62
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
C
Catalin Marinas 已提交
63 64
EXPORT_SYMBOL(empty_zero_page);

65 66 67 68
static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;

C
Catalin Marinas 已提交
69 70 71 72 73 74 75 76 77 78 79
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (!pfn_valid(pfn))
		return pgprot_noncached(vma_prot);
	else if (file->f_flags & O_SYNC)
		return pgprot_writecombine(vma_prot);
	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

80
static phys_addr_t __init early_pgtable_alloc(void)
C
Catalin Marinas 已提交
81
{
82 83 84
	phys_addr_t phys;
	void *ptr;

85
	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
86 87 88 89 90 91 92 93

	/*
	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
	 * any level of table.
	 */
	ptr = pte_set_fixmap(phys);

94 95
	memset(ptr, 0, PAGE_SIZE);

96 97 98 99 100 101 102
	/*
	 * Implicit barriers also ensure the zeroed page is visible to the page
	 * table walker
	 */
	pte_clear_fixmap();

	return phys;
C
Catalin Marinas 已提交
103 104
}

105 106 107 108 109 110 111 112
static bool pgattr_change_is_safe(u64 old, u64 new)
{
	/*
	 * The following mapping attributes may be updated in live
	 * kernel mappings without the need for break-before-make.
	 */
	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;

113 114 115 116 117 118 119 120
	/* creating or taking down mappings is always safe */
	if (old == 0 || new == 0)
		return true;

	/* live contiguous mappings may not be manipulated at all */
	if ((old | new) & PTE_CONT)
		return false;

121 122 123 124
	/* Transitioning from Global to Non-Global is safe */
	if (((old ^ new) == PTE_NG) && (new & PTE_NG))
		return true;

125
	return ((old ^ new) & ~mask) == 0;
126 127
}

128 129
static void init_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
		     phys_addr_t phys, pgprot_t prot)
C
Catalin Marinas 已提交
130 131 132
{
	pte_t *pte;

133
	pte = pte_set_fixmap_offset(pmd, addr);
C
Catalin Marinas 已提交
134
	do {
135 136
		pte_t old_pte = *pte;

137
		set_pte(pte, pfn_pte(__phys_to_pfn(phys), prot));
138 139 140 141 142 143 144

		/*
		 * After the PTE entry has been populated once, we
		 * only allow updates to the permission attributes.
		 */
		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));

145
		phys += PAGE_SIZE;
146
	} while (pte++, addr += PAGE_SIZE, addr != end);
147 148

	pte_clear_fixmap();
C
Catalin Marinas 已提交
149 150
}

151 152 153 154 155
static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr,
				unsigned long end, phys_addr_t phys,
				pgprot_t prot,
				phys_addr_t (*pgtable_alloc)(void),
				int flags)
C
Catalin Marinas 已提交
156 157 158
{
	unsigned long next;

159 160 161
	BUG_ON(pmd_sect(*pmd));
	if (pmd_none(*pmd)) {
		phys_addr_t pte_phys;
162
		BUG_ON(!pgtable_alloc);
163 164
		pte_phys = pgtable_alloc();
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
C
Catalin Marinas 已提交
165
	}
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
	BUG_ON(pmd_bad(*pmd));

	do {
		pgprot_t __prot = prot;

		next = pte_cont_addr_end(addr, end);

		/* use a contiguous mapping if the range is suitably aligned */
		if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
		    (flags & NO_CONT_MAPPINGS) == 0)
			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);

		init_pte(pmd, addr, next, phys, __prot);

		phys += next - addr;
	} while (addr = next, addr != end);
}

static void init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
		     phys_addr_t phys, pgprot_t prot,
		     phys_addr_t (*pgtable_alloc)(void), int flags)
{
	unsigned long next;
	pmd_t *pmd;
C
Catalin Marinas 已提交
190

191
	pmd = pmd_set_fixmap_offset(pud, addr);
C
Catalin Marinas 已提交
192
	do {
193 194
		pmd_t old_pmd = *pmd;

C
Catalin Marinas 已提交
195
		next = pmd_addr_end(addr, end);
196

C
Catalin Marinas 已提交
197
		/* try section mapping first */
198
		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
199
		    (flags & NO_BLOCK_MAPPINGS) == 0) {
200
			pmd_set_huge(pmd, phys, prot);
201

202
			/*
203 204
			 * After the PMD entry has been populated once, we
			 * only allow updates to the permission attributes.
205
			 */
206 207
			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
						      pmd_val(*pmd)));
208
		} else {
209 210
			alloc_init_cont_pte(pmd, addr, next, phys, prot,
					    pgtable_alloc, flags);
211 212 213

			BUG_ON(pmd_val(old_pmd) != 0 &&
			       pmd_val(old_pmd) != pmd_val(*pmd));
214
		}
C
Catalin Marinas 已提交
215 216
		phys += next - addr;
	} while (pmd++, addr = next, addr != end);
217 218

	pmd_clear_fixmap();
C
Catalin Marinas 已提交
219 220
}

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr,
				unsigned long end, phys_addr_t phys,
				pgprot_t prot,
				phys_addr_t (*pgtable_alloc)(void), int flags)
{
	unsigned long next;

	/*
	 * Check for initial section mappings in the pgd/pud.
	 */
	BUG_ON(pud_sect(*pud));
	if (pud_none(*pud)) {
		phys_addr_t pmd_phys;
		BUG_ON(!pgtable_alloc);
		pmd_phys = pgtable_alloc();
		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
	}
	BUG_ON(pud_bad(*pud));

	do {
		pgprot_t __prot = prot;

		next = pmd_cont_addr_end(addr, end);

		/* use a contiguous mapping if the range is suitably aligned */
		if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
		    (flags & NO_CONT_MAPPINGS) == 0)
			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);

		init_pmd(pud, addr, next, phys, __prot, pgtable_alloc, flags);

		phys += next - addr;
	} while (addr = next, addr != end);
}

256 257 258 259 260 261 262 263 264 265 266 267
static inline bool use_1G_block(unsigned long addr, unsigned long next,
			unsigned long phys)
{
	if (PAGE_SHIFT != 12)
		return false;

	if (((addr | next | phys) & ~PUD_MASK) != 0)
		return false;

	return true;
}

268
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
269
				  phys_addr_t phys, pgprot_t prot,
270
				  phys_addr_t (*pgtable_alloc)(void),
271
				  int flags)
C
Catalin Marinas 已提交
272
{
273
	pud_t *pud;
C
Catalin Marinas 已提交
274 275
	unsigned long next;

276
	if (pgd_none(*pgd)) {
277 278 279
		phys_addr_t pud_phys;
		BUG_ON(!pgtable_alloc);
		pud_phys = pgtable_alloc();
280
		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
281 282 283
	}
	BUG_ON(pgd_bad(*pgd));

284
	pud = pud_set_fixmap_offset(pgd, addr);
C
Catalin Marinas 已提交
285
	do {
286 287
		pud_t old_pud = *pud;

C
Catalin Marinas 已提交
288
		next = pud_addr_end(addr, end);
289 290 291 292

		/*
		 * For 4K granule only, attempt to put down a 1GB block
		 */
293 294
		if (use_1G_block(addr, next, phys) &&
		    (flags & NO_BLOCK_MAPPINGS) == 0) {
295
			pud_set_huge(pud, phys, prot);
296 297

			/*
298 299
			 * After the PUD entry has been populated once, we
			 * only allow updates to the permission attributes.
300
			 */
301 302
			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
						      pud_val(*pud)));
303
		} else {
304 305
			alloc_init_cont_pmd(pud, addr, next, phys, prot,
					    pgtable_alloc, flags);
306 307 308

			BUG_ON(pud_val(old_pud) != 0 &&
			       pud_val(old_pud) != pud_val(*pud));
309
		}
C
Catalin Marinas 已提交
310 311
		phys += next - addr;
	} while (pud++, addr = next, addr != end);
312 313

	pud_clear_fixmap();
C
Catalin Marinas 已提交
314 315
}

316 317 318 319
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
				 unsigned long virt, phys_addr_t size,
				 pgprot_t prot,
				 phys_addr_t (*pgtable_alloc)(void),
320
				 int flags)
C
Catalin Marinas 已提交
321 322
{
	unsigned long addr, length, end, next;
323
	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
C
Catalin Marinas 已提交
324

325 326 327 328 329 330 331
	/*
	 * If the virtual and physical address don't have the same offset
	 * within a page, we cannot map the region as the caller expects.
	 */
	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
		return;

332
	phys &= PAGE_MASK;
C
Catalin Marinas 已提交
333 334 335 336 337 338
	addr = virt & PAGE_MASK;
	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));

	end = addr + length;
	do {
		next = pgd_addr_end(addr, end);
339
		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
340
			       flags);
C
Catalin Marinas 已提交
341 342 343 344
		phys += next - addr;
	} while (pgd++, addr = next, addr != end);
}

345
static phys_addr_t pgd_pgtable_alloc(void)
346
{
347
	void *ptr = (void *)__get_free_page(PGALLOC_GFP);
348 349
	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
		BUG();
350 351 352

	/* Ensure the zeroed page is visible to the page table walker */
	dsb(ishst);
353
	return __pa(ptr);
354 355
}

356 357 358 359 360 361
/*
 * This function can only be used to modify existing table entries,
 * without allocating new levels of table. Note that this permits the
 * creation of new section or page entries.
 */
static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
362
				  phys_addr_t size, pgprot_t prot)
363 364 365 366 367 368
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}
369 370
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
			     NO_CONT_MAPPINGS);
371 372
}

373 374
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
			       unsigned long virt, phys_addr_t size,
375
			       pgprot_t prot, bool page_mappings_only)
376
{
377 378
	int flags = 0;

379 380
	BUG_ON(mm == &init_mm);

381
	if (page_mappings_only)
382
		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
383

384
	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
385
			     pgd_pgtable_alloc, flags);
386 387
}

388 389
static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
				phys_addr_t size, pgprot_t prot)
390 391
{
	if (virt < VMALLOC_START) {
392
		pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
393 394 395 396
			&phys, virt);
		return;
	}

397 398
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
			     NO_CONT_MAPPINGS);
399 400 401

	/* flush the TLBs after updating live kernel mappings */
	flush_tlb_kernel_range(virt, virt + size);
402 403
}

404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
static void __init __map_memblock(pgd_t *pgd, phys_addr_t start,
				  phys_addr_t end, pgprot_t prot, int flags)
{
	__create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start,
			     prot, early_pgtable_alloc, flags);
}

void __init mark_linear_text_alias_ro(void)
{
	/*
	 * Remove the write permissions from the linear alias of .text/.rodata
	 */
	update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
			    (unsigned long)__init_begin - (unsigned long)_text,
			    PAGE_KERNEL_RO);
}

static void __init map_mem(pgd_t *pgd)
422
{
423 424
	phys_addr_t kernel_start = __pa_symbol(_text);
	phys_addr_t kernel_end = __pa_symbol(__init_begin);
425
	struct memblock_region *reg;
426 427 428
	int flags = 0;

	if (debug_pagealloc_enabled())
429
		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
430

431
	/*
432 433
	 * Take care not to create a writable alias for the
	 * read-only text and rodata sections of the kernel image.
434 435
	 * So temporarily mark them as NOMAP to skip mappings in
	 * the following for-loop
436
	 */
437 438 439 440 441 442
	memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
#ifdef CONFIG_KEXEC_CORE
	if (crashk_res.end)
		memblock_mark_nomap(crashk_res.start,
				    resource_size(&crashk_res));
#endif
443

444 445 446 447
	/* map all the memory banks */
	for_each_memblock(memory, reg) {
		phys_addr_t start = reg->base;
		phys_addr_t end = start + reg->size;
448

449 450 451 452 453 454 455
		if (start >= end)
			break;
		if (memblock_is_nomap(reg))
			continue;

		__map_memblock(pgd, start, end, PAGE_KERNEL, flags);
	}
456 457

	/*
458 459 460 461 462 463
	 * Map the linear alias of the [_text, __init_begin) interval
	 * as non-executable now, and remove the write permission in
	 * mark_linear_text_alias_ro() below (which will be called after
	 * alternative patching has completed). This makes the contents
	 * of the region accessible to subsystems such as hibernate,
	 * but protects it from inadvertent modification or execution.
464 465
	 * Note that contiguous mappings cannot be remapped in this way,
	 * so we should avoid them here.
466
	 */
467 468 469
	__map_memblock(pgd, kernel_start, kernel_end,
		       PAGE_KERNEL, NO_CONT_MAPPINGS);
	memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
470

471
#ifdef CONFIG_KEXEC_CORE
472
	/*
473 474 475
	 * Use page-level mappings here so that we can shrink the region
	 * in page granularity and put back unused memory to buddy system
	 * through /sys/kernel/kexec_crash_size interface.
476
	 */
477 478 479 480 481 482
	if (crashk_res.end) {
		__map_memblock(pgd, crashk_res.start, crashk_res.end + 1,
			       PAGE_KERNEL,
			       NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
		memblock_clear_nomap(crashk_res.start,
				     resource_size(&crashk_res));
C
Catalin Marinas 已提交
483
	}
484
#endif
C
Catalin Marinas 已提交
485 486
}

487 488
void mark_rodata_ro(void)
{
J
Jeremy Linton 已提交
489
	unsigned long section_size;
490

J
Jeremy Linton 已提交
491
	/*
492 493
	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
	 * to cover NOTES and EXCEPTION_TABLE.
J
Jeremy Linton 已提交
494
	 */
495
	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
496
	update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
J
Jeremy Linton 已提交
497
			    section_size, PAGE_KERNEL_RO);
498

499
	debug_checkwx();
500 501
}

502
static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
503
				      pgprot_t prot, struct vm_struct *vma,
504
				      int flags, unsigned long vm_flags)
505
{
506
	phys_addr_t pa_start = __pa_symbol(va_start);
507 508 509 510 511 512
	unsigned long size = va_end - va_start;

	BUG_ON(!PAGE_ALIGNED(pa_start));
	BUG_ON(!PAGE_ALIGNED(size));

	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
513
			     early_pgtable_alloc, flags);
514

515 516 517
	if (!(vm_flags & VM_NO_GUARD))
		size += PAGE_SIZE;

518 519 520
	vma->addr	= va_start;
	vma->phys_addr	= pa_start;
	vma->size	= size;
521
	vma->flags	= VM_MAP | vm_flags;
522 523 524
	vma->caller	= __builtin_return_address(0);

	vm_area_add_early(vma);
525 526
}

527 528 529 530 531 532
static int __init parse_rodata(char *arg)
{
	return strtobool(arg, &rodata_enabled);
}
early_param("rodata", parse_rodata);

533 534 535 536 537 538 539 540 541 542 543 544 545 546
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
static int __init map_entry_trampoline(void)
{
	pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
	phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);

	/* The trampoline is always mapped and can therefore be global */
	pgprot_val(prot) &= ~PTE_NG;

	/* Map only the text into the trampoline page table */
	memset(tramp_pg_dir, 0, PGD_SIZE);
	__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
			     prot, pgd_pgtable_alloc, 0);

547
	/* Map both the text and data into the kernel page table */
548
	__set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
549 550 551 552 553 554 555 556
	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
		extern char __entry_tramp_data_start[];

		__set_fixmap(FIX_ENTRY_TRAMP_DATA,
			     __pa_symbol(__entry_tramp_data_start),
			     PAGE_KERNEL_RO);
	}

557 558 559 560 561
	return 0;
}
core_initcall(map_entry_trampoline);
#endif

562 563 564 565 566
/*
 * Create fine-grained mappings for the kernel.
 */
static void __init map_kernel(pgd_t *pgd)
{
567 568
	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
				vmlinux_initdata, vmlinux_data;
569

570 571 572 573 574 575 576
	/*
	 * External debuggers may need to write directly to the text
	 * mapping to install SW breakpoints. Allow this (only) when
	 * explicitly requested with rodata=off.
	 */
	pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;

577 578 579 580
	/*
	 * Only rodata will be remapped with different permissions later on,
	 * all other segments are allowed to use contiguous mappings.
	 */
581 582
	map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text, 0,
			   VM_NO_GUARD);
583
	map_kernel_segment(pgd, __start_rodata, __inittext_begin, PAGE_KERNEL,
584
			   &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
585
	map_kernel_segment(pgd, __inittext_begin, __inittext_end, text_prot,
586
			   &vmlinux_inittext, 0, VM_NO_GUARD);
587
	map_kernel_segment(pgd, __initdata_begin, __initdata_end, PAGE_KERNEL,
588 589
			   &vmlinux_initdata, 0, VM_NO_GUARD);
	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
590

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
		/*
		 * The fixmap falls in a separate pgd to the kernel, and doesn't
		 * live in the carveout for the swapper_pg_dir. We can simply
		 * re-use the existing dir for the fixmap.
		 */
		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
			*pgd_offset_k(FIXADDR_START));
	} else if (CONFIG_PGTABLE_LEVELS > 3) {
		/*
		 * The fixmap shares its top level pgd entry with the kernel
		 * mapping. This can really only occur when we are running
		 * with 16k/4 levels, so we can simply reuse the pud level
		 * entry instead.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
607 608
		pud_populate(&init_mm, pud_set_fixmap_offset(pgd, FIXADDR_START),
			     lm_alias(bm_pmd));
609 610 611 612
		pud_clear_fixmap();
	} else {
		BUG();
	}
613 614 615 616

	kasan_copy_shadow(pgd);
}

C
Catalin Marinas 已提交
617 618 619 620 621 622
/*
 * paging_init() sets up the page tables, initialises the zone memory
 * maps and sets up the zero page.
 */
void __init paging_init(void)
{
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
	phys_addr_t pgd_phys = early_pgtable_alloc();
	pgd_t *pgd = pgd_set_fixmap(pgd_phys);

	map_kernel(pgd);
	map_mem(pgd);

	/*
	 * We want to reuse the original swapper_pg_dir so we don't have to
	 * communicate the new address to non-coherent secondaries in
	 * secondary_entry, and so cpu_switch_mm can generate the address with
	 * adrp+add rather than a load from some global variable.
	 *
	 * To do this we need to go via a temporary pgd.
	 */
	cpu_replace_ttbr1(__va(pgd_phys));
638
	memcpy(swapper_pg_dir, pgd, PGD_SIZE);
639
	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
640 641 642 643 644 645 646 647

	pgd_clear_fixmap();
	memblock_free(pgd_phys, PAGE_SIZE);

	/*
	 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
	 * allocated with it.
	 */
648
	memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
649 650
		      __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir)
		      - PAGE_SIZE);
C
Catalin Marinas 已提交
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
}

/*
 * Check whether a kernel address is valid (derived from arch/x86/).
 */
int kern_addr_valid(unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	if ((((long)addr) >> VA_BITS) != -1UL)
		return 0;

	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0;

674 675 676
	if (pud_sect(*pud))
		return pfn_valid(pud_pfn(*pud));

C
Catalin Marinas 已提交
677 678 679 680
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;

681 682 683
	if (pmd_sect(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

C
Catalin Marinas 已提交
684 685 686 687 688 689 690
	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;

	return pfn_valid(pte_pfn(*pte));
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
691
#if !ARM64_SWAPPER_USES_SECTION_MAPS
692 693
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
		struct vmem_altmap *altmap)
C
Catalin Marinas 已提交
694
{
695
	return vmemmap_populate_basepages(start, end, node);
C
Catalin Marinas 已提交
696
}
697
#else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
698 699
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
		struct vmem_altmap *altmap)
C
Catalin Marinas 已提交
700
{
701
	unsigned long addr = start;
C
Catalin Marinas 已提交
702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	do {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(*pmd)) {
			void *p = NULL;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
			if (!p)
				return -ENOMEM;

726
			pmd_set_huge(pmd, __pa(p), __pgprot(PROT_SECT_NORMAL));
C
Catalin Marinas 已提交
727 728 729 730 731 732 733
		} else
			vmemmap_verify((pte_t *)pmd, node, addr, next);
	} while (addr = next, addr != end);

	return 0;
}
#endif	/* CONFIG_ARM64_64K_PAGES */
734 735
void vmemmap_free(unsigned long start, unsigned long end,
		struct vmem_altmap *altmap)
736 737
{
}
C
Catalin Marinas 已提交
738
#endif	/* CONFIG_SPARSEMEM_VMEMMAP */
739 740 741 742 743 744 745

static inline pud_t * fixmap_pud(unsigned long addr)
{
	pgd_t *pgd = pgd_offset_k(addr);

	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));

746
	return pud_offset_kimg(pgd, addr);
747 748 749 750 751 752 753 754
}

static inline pmd_t * fixmap_pmd(unsigned long addr)
{
	pud_t *pud = fixmap_pud(addr);

	BUG_ON(pud_none(*pud) || pud_bad(*pud));

755
	return pmd_offset_kimg(pud, addr);
756 757 758 759
}

static inline pte_t * fixmap_pte(unsigned long addr)
{
760
	return &bm_pte[pte_index(addr)];
761 762
}

763 764 765 766 767 768
/*
 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
 * directly on kernel symbols (bm_p*d). This function is called too early to use
 * lm_alias so __p*d_populate functions must be used to populate with the
 * physical address from __pa_symbol.
 */
769 770 771 772 773 774 775 776
void __init early_fixmap_init(void)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	unsigned long addr = FIXADDR_START;

	pgd = pgd_offset_k(addr);
777
	if (CONFIG_PGTABLE_LEVELS > 3 &&
778
	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) {
779 780 781 782 783 784 785 786
		/*
		 * We only end up here if the kernel mapping and the fixmap
		 * share the top level pgd entry, which should only happen on
		 * 16k/4 levels configurations.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		pud = pud_offset_kimg(pgd, addr);
	} else {
787 788
		if (pgd_none(*pgd))
			__pgd_populate(pgd, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
789 790
		pud = fixmap_pud(addr);
	}
791 792
	if (pud_none(*pud))
		__pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
793
	pmd = fixmap_pmd(addr);
794
	__pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
795 796 797

	/*
	 * The boot-ioremap range spans multiple pmds, for which
798
	 * we are not prepared:
799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
	 */
	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));

	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		pr_warn("pmd %p != %p, %p\n",
			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
			fix_to_virt(FIX_BTMAP_BEGIN));
		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
			fix_to_virt(FIX_BTMAP_END));

		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
	}
}

819 820 821 822
/*
 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
 * ever need to use IPIs for TLB broadcasting, then we're in trouble here.
 */
823 824 825 826 827 828
void __set_fixmap(enum fixed_addresses idx,
			       phys_addr_t phys, pgprot_t flags)
{
	unsigned long addr = __fix_to_virt(idx);
	pte_t *pte;

829
	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
830 831 832 833 834 835 836 837 838 839

	pte = fixmap_pte(addr);

	if (pgprot_val(flags)) {
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
	} else {
		pte_clear(&init_mm, addr, pte);
		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
	}
}
840

841
void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
842 843
{
	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
844
	int offset;
845 846 847 848 849
	void *dt_virt;

	/*
	 * Check whether the physical FDT address is set and meets the minimum
	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
850 851 852
	 * at least 8 bytes so that we can always access the magic and size
	 * fields of the FDT header after mapping the first chunk, double check
	 * here if that is indeed the case.
853 854 855 856 857 858 859 860
	 */
	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
		return NULL;

	/*
	 * Make sure that the FDT region can be mapped without the need to
	 * allocate additional translation table pages, so that it is safe
861
	 * to call create_mapping_noalloc() this early.
862 863 864 865 866 867 868 869
	 *
	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
	 * be in the same PMD as the rest of the fixmap.
	 * On 4k pages, we'll use section mappings for the FDT so we only
	 * have to be in the same PUD.
	 */
	BUILD_BUG_ON(dt_virt_base % SZ_2M);

870 871
	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
872

873
	offset = dt_phys % SWAPPER_BLOCK_SIZE;
874 875 876
	dt_virt = (void *)dt_virt_base + offset;

	/* map the first chunk so we can read the size from the header */
877 878
	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
879

880
	if (fdt_magic(dt_virt) != FDT_MAGIC)
881 882
		return NULL;

883 884
	*size = fdt_totalsize(dt_virt);
	if (*size > MAX_FDT_SIZE)
885 886
		return NULL;

887
	if (offset + *size > SWAPPER_BLOCK_SIZE)
888
		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
889
			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
890

891 892
	return dt_virt;
}
893

894 895 896 897 898 899 900 901 902 903
void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
{
	void *dt_virt;
	int size;

	dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO);
	if (!dt_virt)
		return NULL;

	memblock_reserve(dt_phys, size);
904 905
	return dt_virt;
}
906 907 908 909 910 911 912 913 914 915 916 917 918 919

int __init arch_ioremap_pud_supported(void)
{
	/* only 4k granule supports level 1 block mappings */
	return IS_ENABLED(CONFIG_ARM64_4K_PAGES);
}

int __init arch_ioremap_pmd_supported(void)
{
	return 1;
}

int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
{
920 921
	pgprot_t sect_prot = __pgprot(PUD_TYPE_SECT |
					pgprot_val(mk_sect_prot(prot)));
922
	BUG_ON(phys & ~PUD_MASK);
923
	set_pud(pud, pfn_pud(__phys_to_pfn(phys), sect_prot));
924 925 926 927 928
	return 1;
}

int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
{
929 930
	pgprot_t sect_prot = __pgprot(PMD_TYPE_SECT |
					pgprot_val(mk_sect_prot(prot)));
931
	BUG_ON(phys & ~PMD_MASK);
932
	set_pmd(pmd, pfn_pmd(__phys_to_pfn(phys), sect_prot));
933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
	return 1;
}

int pud_clear_huge(pud_t *pud)
{
	if (!pud_sect(*pud))
		return 0;
	pud_clear(pud);
	return 1;
}

int pmd_clear_huge(pmd_t *pmd)
{
	if (!pmd_sect(*pmd))
		return 0;
	pmd_clear(pmd);
	return 1;
}