mmu.c 20.2 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Based on arch/arm/mm/mmu.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/cache.h>
C
Catalin Marinas 已提交
21 22 23 24
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
25
#include <linux/libfdt.h>
C
Catalin Marinas 已提交
26 27 28 29
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/memblock.h>
#include <linux/fs.h>
30
#include <linux/io.h>
C
Catalin Marinas 已提交
31

32
#include <asm/barrier.h>
C
Catalin Marinas 已提交
33
#include <asm/cputype.h>
34
#include <asm/fixmap.h>
35
#include <asm/kasan.h>
36
#include <asm/kernel-pgtable.h>
C
Catalin Marinas 已提交
37 38 39 40
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
41
#include <asm/memblock.h>
C
Catalin Marinas 已提交
42 43
#include <asm/mmu_context.h>

44 45
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);

46
u64 kimage_voffset __ro_after_init;
47 48
EXPORT_SYMBOL(kimage_voffset);

C
Catalin Marinas 已提交
49 50 51 52
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
53
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
C
Catalin Marinas 已提交
54 55
EXPORT_SYMBOL(empty_zero_page);

56 57 58 59
static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;

C
Catalin Marinas 已提交
60 61 62 63 64 65 66 67 68 69 70
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (!pfn_valid(pfn))
		return pgprot_noncached(vma_prot);
	else if (file->f_flags & O_SYNC)
		return pgprot_writecombine(vma_prot);
	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

71
static phys_addr_t __init early_pgtable_alloc(void)
C
Catalin Marinas 已提交
72
{
73 74 75
	phys_addr_t phys;
	void *ptr;

76
	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
77 78 79 80 81 82 83 84

	/*
	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
	 * any level of table.
	 */
	ptr = pte_set_fixmap(phys);

85 86
	memset(ptr, 0, PAGE_SIZE);

87 88 89 90 91 92 93
	/*
	 * Implicit barriers also ensure the zeroed page is visible to the page
	 * table walker
	 */
	pte_clear_fixmap();

	return phys;
C
Catalin Marinas 已提交
94 95
}

96 97 98 99 100 101 102 103 104 105 106
static bool pgattr_change_is_safe(u64 old, u64 new)
{
	/*
	 * The following mapping attributes may be updated in live
	 * kernel mappings without the need for break-before-make.
	 */
	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;

	return old  == 0 || new  == 0 || ((old ^ new) & ~mask) == 0;
}

107
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
108
				  unsigned long end, unsigned long pfn,
109
				  pgprot_t prot,
110
				  phys_addr_t (*pgtable_alloc)(void))
C
Catalin Marinas 已提交
111 112 113
{
	pte_t *pte;

114 115
	BUG_ON(pmd_sect(*pmd));
	if (pmd_none(*pmd)) {
116 117 118
		phys_addr_t pte_phys;
		BUG_ON(!pgtable_alloc);
		pte_phys = pgtable_alloc();
119 120 121
		pte = pte_set_fixmap(pte_phys);
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
		pte_clear_fixmap();
C
Catalin Marinas 已提交
122
	}
123
	BUG_ON(pmd_bad(*pmd));
C
Catalin Marinas 已提交
124

125
	pte = pte_set_fixmap_offset(pmd, addr);
C
Catalin Marinas 已提交
126
	do {
127 128
		pte_t old_pte = *pte;

129 130
		set_pte(pte, pfn_pte(pfn, prot));
		pfn++;
131 132 133 134 135 136 137

		/*
		 * After the PTE entry has been populated once, we
		 * only allow updates to the permission attributes.
		 */
		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));

138
	} while (pte++, addr += PAGE_SIZE, addr != end);
139 140

	pte_clear_fixmap();
C
Catalin Marinas 已提交
141 142
}

143
static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
144
				  phys_addr_t phys, pgprot_t prot,
145 146
				  phys_addr_t (*pgtable_alloc)(void),
				  bool allow_block_mappings)
C
Catalin Marinas 已提交
147 148 149 150 151 152 153
{
	pmd_t *pmd;
	unsigned long next;

	/*
	 * Check for initial section mappings in the pgd/pud and remove them.
	 */
154 155
	BUG_ON(pud_sect(*pud));
	if (pud_none(*pud)) {
156 157 158
		phys_addr_t pmd_phys;
		BUG_ON(!pgtable_alloc);
		pmd_phys = pgtable_alloc();
159 160 161
		pmd = pmd_set_fixmap(pmd_phys);
		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
		pmd_clear_fixmap();
C
Catalin Marinas 已提交
162
	}
163
	BUG_ON(pud_bad(*pud));
C
Catalin Marinas 已提交
164

165
	pmd = pmd_set_fixmap_offset(pud, addr);
C
Catalin Marinas 已提交
166
	do {
167 168
		pmd_t old_pmd = *pmd;

C
Catalin Marinas 已提交
169
		next = pmd_addr_end(addr, end);
170

C
Catalin Marinas 已提交
171
		/* try section mapping first */
172
		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
173
		      allow_block_mappings) {
174
			pmd_set_huge(pmd, phys, prot);
175

176
			/*
177 178
			 * After the PMD entry has been populated once, we
			 * only allow updates to the permission attributes.
179
			 */
180 181
			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
						      pmd_val(*pmd)));
182
		} else {
183
			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
184
				       prot, pgtable_alloc);
185 186 187

			BUG_ON(pmd_val(old_pmd) != 0 &&
			       pmd_val(old_pmd) != pmd_val(*pmd));
188
		}
C
Catalin Marinas 已提交
189 190
		phys += next - addr;
	} while (pmd++, addr = next, addr != end);
191 192

	pmd_clear_fixmap();
C
Catalin Marinas 已提交
193 194
}

195 196 197 198 199 200 201 202 203 204 205 206
static inline bool use_1G_block(unsigned long addr, unsigned long next,
			unsigned long phys)
{
	if (PAGE_SHIFT != 12)
		return false;

	if (((addr | next | phys) & ~PUD_MASK) != 0)
		return false;

	return true;
}

207
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
208
				  phys_addr_t phys, pgprot_t prot,
209 210
				  phys_addr_t (*pgtable_alloc)(void),
				  bool allow_block_mappings)
C
Catalin Marinas 已提交
211
{
212
	pud_t *pud;
C
Catalin Marinas 已提交
213 214
	unsigned long next;

215
	if (pgd_none(*pgd)) {
216 217 218
		phys_addr_t pud_phys;
		BUG_ON(!pgtable_alloc);
		pud_phys = pgtable_alloc();
219
		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
220 221 222
	}
	BUG_ON(pgd_bad(*pgd));

223
	pud = pud_set_fixmap_offset(pgd, addr);
C
Catalin Marinas 已提交
224
	do {
225 226
		pud_t old_pud = *pud;

C
Catalin Marinas 已提交
227
		next = pud_addr_end(addr, end);
228 229 230 231

		/*
		 * For 4K granule only, attempt to put down a 1GB block
		 */
232
		if (use_1G_block(addr, next, phys) && allow_block_mappings) {
233
			pud_set_huge(pud, phys, prot);
234 235

			/*
236 237
			 * After the PUD entry has been populated once, we
			 * only allow updates to the permission attributes.
238
			 */
239 240
			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
						      pud_val(*pud)));
241
		} else {
242
			alloc_init_pmd(pud, addr, next, phys, prot,
243
				       pgtable_alloc, allow_block_mappings);
244 245 246

			BUG_ON(pud_val(old_pud) != 0 &&
			       pud_val(old_pud) != pud_val(*pud));
247
		}
C
Catalin Marinas 已提交
248 249
		phys += next - addr;
	} while (pud++, addr = next, addr != end);
250 251

	pud_clear_fixmap();
C
Catalin Marinas 已提交
252 253
}

254 255 256 257 258
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
				 unsigned long virt, phys_addr_t size,
				 pgprot_t prot,
				 phys_addr_t (*pgtable_alloc)(void),
				 bool allow_block_mappings)
C
Catalin Marinas 已提交
259 260
{
	unsigned long addr, length, end, next;
261
	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
C
Catalin Marinas 已提交
262

263 264 265 266 267 268 269
	/*
	 * If the virtual and physical address don't have the same offset
	 * within a page, we cannot map the region as the caller expects.
	 */
	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
		return;

270
	phys &= PAGE_MASK;
C
Catalin Marinas 已提交
271 272 273 274 275 276
	addr = virt & PAGE_MASK;
	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));

	end = addr + length;
	do {
		next = pgd_addr_end(addr, end);
277 278
		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
			       allow_block_mappings);
C
Catalin Marinas 已提交
279 280 281 282
		phys += next - addr;
	} while (pgd++, addr = next, addr != end);
}

283
static phys_addr_t pgd_pgtable_alloc(void)
284
{
285
	void *ptr = (void *)__get_free_page(PGALLOC_GFP);
286 287
	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
		BUG();
288 289 290

	/* Ensure the zeroed page is visible to the page table walker */
	dsb(ishst);
291
	return __pa(ptr);
292 293
}

294 295 296 297 298 299
/*
 * This function can only be used to modify existing table entries,
 * without allocating new levels of table. Note that this permits the
 * creation of new section or page entries.
 */
static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
300
				  phys_addr_t size, pgprot_t prot)
301 302 303 304 305 306
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}
307
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true);
308 309
}

310 311
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
			       unsigned long virt, phys_addr_t size,
312
			       pgprot_t prot, bool allow_block_mappings)
313
{
314 315
	BUG_ON(mm == &init_mm);

316
	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
317
			     pgd_pgtable_alloc, allow_block_mappings);
318 319
}

320 321 322 323 324 325 326 327 328
static void create_mapping_late(phys_addr_t phys, unsigned long virt,
				  phys_addr_t size, pgprot_t prot)
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}

329
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot,
330
			     NULL, !debug_pagealloc_enabled());
331 332
}

333
static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
334
{
335
	unsigned long kernel_start = __pa(_text);
336
	unsigned long kernel_end = __pa(__init_begin);
337

338
	/*
339 340
	 * Take care not to create a writable alias for the
	 * read-only text and rodata sections of the kernel image.
341
	 */
342

343
	/* No overlap with the kernel text/rodata */
344 345 346
	if (end < kernel_start || start >= kernel_end) {
		__create_pgd_mapping(pgd, start, __phys_to_virt(start),
				     end - start, PAGE_KERNEL,
347 348
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
349
		return;
350 351
	}

352
	/*
353
	 * This block overlaps the kernel text/rodata mappings.
354
	 * Map the portion(s) which don't overlap.
355 356 357 358 359
	 */
	if (start < kernel_start)
		__create_pgd_mapping(pgd, start,
				     __phys_to_virt(start),
				     kernel_start - start, PAGE_KERNEL,
360 361
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
362 363 364 365
	if (kernel_end < end)
		__create_pgd_mapping(pgd, kernel_end,
				     __phys_to_virt(kernel_end),
				     end - kernel_end, PAGE_KERNEL,
366 367
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
368 369

	/*
370
	 * Map the linear alias of the [_text, __init_begin) interval as
371 372 373 374 375 376
	 * read-only/non-executable. This makes the contents of the
	 * region accessible to subsystems such as hibernate, but
	 * protects it from inadvertent modification or execution.
	 */
	__create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
			     kernel_end - kernel_start, PAGE_KERNEL_RO,
377
			     early_pgtable_alloc, !debug_pagealloc_enabled());
378 379
}

380
static void __init map_mem(pgd_t *pgd)
C
Catalin Marinas 已提交
381 382
{
	struct memblock_region *reg;
383

C
Catalin Marinas 已提交
384 385 386 387 388 389 390
	/* map all the memory banks */
	for_each_memblock(memory, reg) {
		phys_addr_t start = reg->base;
		phys_addr_t end = start + reg->size;

		if (start >= end)
			break;
391 392
		if (memblock_is_nomap(reg))
			continue;
C
Catalin Marinas 已提交
393

394
		__map_memblock(pgd, start, end);
C
Catalin Marinas 已提交
395 396 397
	}
}

398 399
void mark_rodata_ro(void)
{
J
Jeremy Linton 已提交
400
	unsigned long section_size;
401

402
	section_size = (unsigned long)_etext - (unsigned long)_text;
403
	create_mapping_late(__pa(_text), (unsigned long)_text,
J
Jeremy Linton 已提交
404 405
			    section_size, PAGE_KERNEL_ROX);
	/*
406 407
	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
	 * to cover NOTES and EXCEPTION_TABLE.
J
Jeremy Linton 已提交
408
	 */
409
	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
J
Jeremy Linton 已提交
410 411
	create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata,
			    section_size, PAGE_KERNEL_RO);
412 413 414

	/* flush the TLBs after updating live kernel mappings */
	flush_tlb_all();
415 416
}

417 418
static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
				      pgprot_t prot, struct vm_struct *vma)
419 420 421 422 423 424 425 426
{
	phys_addr_t pa_start = __pa(va_start);
	unsigned long size = va_end - va_start;

	BUG_ON(!PAGE_ALIGNED(pa_start));
	BUG_ON(!PAGE_ALIGNED(size));

	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
427
			     early_pgtable_alloc, !debug_pagealloc_enabled());
428 429 430 431 432 433 434 435

	vma->addr	= va_start;
	vma->phys_addr	= pa_start;
	vma->size	= size;
	vma->flags	= VM_MAP;
	vma->caller	= __builtin_return_address(0);

	vm_area_add_early(vma);
436 437 438 439 440 441 442
}

/*
 * Create fine-grained mappings for the kernel.
 */
static void __init map_kernel(pgd_t *pgd)
{
J
Jeremy Linton 已提交
443
	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
444

445 446
	map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text);
	map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata);
447 448 449
	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
			   &vmlinux_init);
	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
450

451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
		/*
		 * The fixmap falls in a separate pgd to the kernel, and doesn't
		 * live in the carveout for the swapper_pg_dir. We can simply
		 * re-use the existing dir for the fixmap.
		 */
		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
			*pgd_offset_k(FIXADDR_START));
	} else if (CONFIG_PGTABLE_LEVELS > 3) {
		/*
		 * The fixmap shares its top level pgd entry with the kernel
		 * mapping. This can really only occur when we are running
		 * with 16k/4 levels, so we can simply reuse the pud level
		 * entry instead.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START),
			__pud(__pa(bm_pmd) | PUD_TYPE_TABLE));
		pud_clear_fixmap();
	} else {
		BUG();
	}
473 474 475 476

	kasan_copy_shadow(pgd);
}

C
Catalin Marinas 已提交
477 478 479 480 481 482
/*
 * paging_init() sets up the page tables, initialises the zone memory
 * maps and sets up the zero page.
 */
void __init paging_init(void)
{
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
	phys_addr_t pgd_phys = early_pgtable_alloc();
	pgd_t *pgd = pgd_set_fixmap(pgd_phys);

	map_kernel(pgd);
	map_mem(pgd);

	/*
	 * We want to reuse the original swapper_pg_dir so we don't have to
	 * communicate the new address to non-coherent secondaries in
	 * secondary_entry, and so cpu_switch_mm can generate the address with
	 * adrp+add rather than a load from some global variable.
	 *
	 * To do this we need to go via a temporary pgd.
	 */
	cpu_replace_ttbr1(__va(pgd_phys));
	memcpy(swapper_pg_dir, pgd, PAGE_SIZE);
	cpu_replace_ttbr1(swapper_pg_dir);

	pgd_clear_fixmap();
	memblock_free(pgd_phys, PAGE_SIZE);

	/*
	 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
	 * allocated with it.
	 */
	memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE,
		      SWAPPER_DIR_SIZE - PAGE_SIZE);
C
Catalin Marinas 已提交
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
}

/*
 * Check whether a kernel address is valid (derived from arch/x86/).
 */
int kern_addr_valid(unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	if ((((long)addr) >> VA_BITS) != -1UL)
		return 0;

	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0;

533 534 535
	if (pud_sect(*pud))
		return pfn_valid(pud_pfn(*pud));

C
Catalin Marinas 已提交
536 537 538 539
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;

540 541 542
	if (pmd_sect(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

C
Catalin Marinas 已提交
543 544 545 546 547 548 549
	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;

	return pfn_valid(pte_pfn(*pte));
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
550
#if !ARM64_SWAPPER_USES_SECTION_MAPS
551
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
552
{
553
	return vmemmap_populate_basepages(start, end, node);
C
Catalin Marinas 已提交
554
}
555
#else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
556
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
557
{
558
	unsigned long addr = start;
C
Catalin Marinas 已提交
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	do {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(*pmd)) {
			void *p = NULL;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
			if (!p)
				return -ENOMEM;

583
			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
C
Catalin Marinas 已提交
584 585 586 587 588 589 590
		} else
			vmemmap_verify((pte_t *)pmd, node, addr, next);
	} while (addr = next, addr != end);

	return 0;
}
#endif	/* CONFIG_ARM64_64K_PAGES */
591
void vmemmap_free(unsigned long start, unsigned long end)
592 593
{
}
C
Catalin Marinas 已提交
594
#endif	/* CONFIG_SPARSEMEM_VMEMMAP */
595 596 597 598 599 600 601

static inline pud_t * fixmap_pud(unsigned long addr)
{
	pgd_t *pgd = pgd_offset_k(addr);

	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));

602
	return pud_offset_kimg(pgd, addr);
603 604 605 606 607 608 609 610
}

static inline pmd_t * fixmap_pmd(unsigned long addr)
{
	pud_t *pud = fixmap_pud(addr);

	BUG_ON(pud_none(*pud) || pud_bad(*pud));

611
	return pmd_offset_kimg(pud, addr);
612 613 614 615
}

static inline pte_t * fixmap_pte(unsigned long addr)
{
616
	return &bm_pte[pte_index(addr)];
617 618 619 620 621 622 623 624 625 626
}

void __init early_fixmap_init(void)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	unsigned long addr = FIXADDR_START;

	pgd = pgd_offset_k(addr);
627 628
	if (CONFIG_PGTABLE_LEVELS > 3 &&
	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) {
629 630 631 632 633 634 635 636 637 638 639
		/*
		 * We only end up here if the kernel mapping and the fixmap
		 * share the top level pgd entry, which should only happen on
		 * 16k/4 levels configurations.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		pud = pud_offset_kimg(pgd, addr);
	} else {
		pgd_populate(&init_mm, pgd, bm_pud);
		pud = fixmap_pud(addr);
	}
640
	pud_populate(&init_mm, pud, bm_pmd);
641
	pmd = fixmap_pmd(addr);
642 643 644 645
	pmd_populate_kernel(&init_mm, pmd, bm_pte);

	/*
	 * The boot-ioremap range spans multiple pmds, for which
646
	 * we are not prepared:
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
	 */
	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));

	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		pr_warn("pmd %p != %p, %p\n",
			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
			fix_to_virt(FIX_BTMAP_BEGIN));
		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
			fix_to_virt(FIX_BTMAP_END));

		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
	}
}

void __set_fixmap(enum fixed_addresses idx,
			       phys_addr_t phys, pgprot_t flags)
{
	unsigned long addr = __fix_to_virt(idx);
	pte_t *pte;

673
	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
674 675 676 677 678 679 680 681 682 683

	pte = fixmap_pte(addr);

	if (pgprot_val(flags)) {
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
	} else {
		pte_clear(&init_mm, addr, pte);
		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
	}
}
684

685
void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
686 687
{
	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
688
	int offset;
689 690 691 692 693
	void *dt_virt;

	/*
	 * Check whether the physical FDT address is set and meets the minimum
	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
694 695 696
	 * at least 8 bytes so that we can always access the magic and size
	 * fields of the FDT header after mapping the first chunk, double check
	 * here if that is indeed the case.
697 698 699 700 701 702 703 704
	 */
	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
		return NULL;

	/*
	 * Make sure that the FDT region can be mapped without the need to
	 * allocate additional translation table pages, so that it is safe
705
	 * to call create_mapping_noalloc() this early.
706 707 708 709 710 711 712 713
	 *
	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
	 * be in the same PMD as the rest of the fixmap.
	 * On 4k pages, we'll use section mappings for the FDT so we only
	 * have to be in the same PUD.
	 */
	BUILD_BUG_ON(dt_virt_base % SZ_2M);

714 715
	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
716

717
	offset = dt_phys % SWAPPER_BLOCK_SIZE;
718 719 720
	dt_virt = (void *)dt_virt_base + offset;

	/* map the first chunk so we can read the size from the header */
721 722
	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
723

724
	if (fdt_magic(dt_virt) != FDT_MAGIC)
725 726
		return NULL;

727 728
	*size = fdt_totalsize(dt_virt);
	if (*size > MAX_FDT_SIZE)
729 730
		return NULL;

731
	if (offset + *size > SWAPPER_BLOCK_SIZE)
732
		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
733
			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
734

735 736
	return dt_virt;
}
737

738 739 740 741 742 743 744 745 746 747
void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
{
	void *dt_virt;
	int size;

	dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO);
	if (!dt_virt)
		return NULL;

	memblock_reserve(dt_phys, size);
748 749
	return dt_virt;
}
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790

int __init arch_ioremap_pud_supported(void)
{
	/* only 4k granule supports level 1 block mappings */
	return IS_ENABLED(CONFIG_ARM64_4K_PAGES);
}

int __init arch_ioremap_pmd_supported(void)
{
	return 1;
}

int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PUD_MASK);
	set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PMD_MASK);
	set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pud_clear_huge(pud_t *pud)
{
	if (!pud_sect(*pud))
		return 0;
	pud_clear(pud);
	return 1;
}

int pmd_clear_huge(pmd_t *pmd)
{
	if (!pmd_sect(*pmd))
		return 0;
	pmd_clear(pmd);
	return 1;
}