mmu.c 21.0 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Based on arch/arm/mm/mmu.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/cache.h>
C
Catalin Marinas 已提交
21 22 23 24
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
25
#include <linux/libfdt.h>
C
Catalin Marinas 已提交
26 27 28 29
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/memblock.h>
#include <linux/fs.h>
30
#include <linux/io.h>
C
Catalin Marinas 已提交
31

32
#include <asm/barrier.h>
C
Catalin Marinas 已提交
33
#include <asm/cputype.h>
34
#include <asm/fixmap.h>
35
#include <asm/kasan.h>
36
#include <asm/kernel-pgtable.h>
C
Catalin Marinas 已提交
37 38 39 40
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
41
#include <asm/memblock.h>
C
Catalin Marinas 已提交
42
#include <asm/mmu_context.h>
43
#include <asm/ptdump.h>
C
Catalin Marinas 已提交
44

45 46
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);

47
u64 kimage_voffset __ro_after_init;
48 49
EXPORT_SYMBOL(kimage_voffset);

C
Catalin Marinas 已提交
50 51 52 53
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
54
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
C
Catalin Marinas 已提交
55 56
EXPORT_SYMBOL(empty_zero_page);

57 58 59 60
static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;

C
Catalin Marinas 已提交
61 62 63 64 65 66 67 68 69 70 71
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (!pfn_valid(pfn))
		return pgprot_noncached(vma_prot);
	else if (file->f_flags & O_SYNC)
		return pgprot_writecombine(vma_prot);
	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

72
static phys_addr_t __init early_pgtable_alloc(void)
C
Catalin Marinas 已提交
73
{
74 75 76
	phys_addr_t phys;
	void *ptr;

77
	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
78 79 80 81 82 83 84 85

	/*
	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
	 * any level of table.
	 */
	ptr = pte_set_fixmap(phys);

86 87
	memset(ptr, 0, PAGE_SIZE);

88 89 90 91 92 93 94
	/*
	 * Implicit barriers also ensure the zeroed page is visible to the page
	 * table walker
	 */
	pte_clear_fixmap();

	return phys;
C
Catalin Marinas 已提交
95 96
}

97 98 99 100 101 102 103 104 105 106 107
static bool pgattr_change_is_safe(u64 old, u64 new)
{
	/*
	 * The following mapping attributes may be updated in live
	 * kernel mappings without the need for break-before-make.
	 */
	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;

	return old  == 0 || new  == 0 || ((old ^ new) & ~mask) == 0;
}

108
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
109
				  unsigned long end, unsigned long pfn,
110
				  pgprot_t prot,
111 112
				  phys_addr_t (*pgtable_alloc)(void),
				  bool page_mappings_only)
C
Catalin Marinas 已提交
113
{
114
	pgprot_t __prot = prot;
C
Catalin Marinas 已提交
115 116
	pte_t *pte;

117 118
	BUG_ON(pmd_sect(*pmd));
	if (pmd_none(*pmd)) {
119 120 121
		phys_addr_t pte_phys;
		BUG_ON(!pgtable_alloc);
		pte_phys = pgtable_alloc();
122 123 124
		pte = pte_set_fixmap(pte_phys);
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
		pte_clear_fixmap();
C
Catalin Marinas 已提交
125
	}
126
	BUG_ON(pmd_bad(*pmd));
C
Catalin Marinas 已提交
127

128
	pte = pte_set_fixmap_offset(pmd, addr);
C
Catalin Marinas 已提交
129
	do {
130 131
		pte_t old_pte = *pte;

132 133 134 135 136 137 138 139 140 141 142 143
		/*
		 * Set the contiguous bit for the subsequent group of PTEs if
		 * its size and alignment are appropriate.
		 */
		if (((addr | PFN_PHYS(pfn)) & ~CONT_PTE_MASK) == 0) {
			if (end - addr >= CONT_PTE_SIZE && !page_mappings_only)
				__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
			else
				__prot = prot;
		}

		set_pte(pte, pfn_pte(pfn, __prot));
144
		pfn++;
145 146 147 148 149 150 151

		/*
		 * After the PTE entry has been populated once, we
		 * only allow updates to the permission attributes.
		 */
		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));

152
	} while (pte++, addr += PAGE_SIZE, addr != end);
153 154

	pte_clear_fixmap();
C
Catalin Marinas 已提交
155 156
}

157
static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
158
				  phys_addr_t phys, pgprot_t prot,
159
				  phys_addr_t (*pgtable_alloc)(void),
160
				  bool page_mappings_only)
C
Catalin Marinas 已提交
161
{
162
	pgprot_t __prot = prot;
C
Catalin Marinas 已提交
163 164 165 166 167 168
	pmd_t *pmd;
	unsigned long next;

	/*
	 * Check for initial section mappings in the pgd/pud and remove them.
	 */
169 170
	BUG_ON(pud_sect(*pud));
	if (pud_none(*pud)) {
171 172 173
		phys_addr_t pmd_phys;
		BUG_ON(!pgtable_alloc);
		pmd_phys = pgtable_alloc();
174 175 176
		pmd = pmd_set_fixmap(pmd_phys);
		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
		pmd_clear_fixmap();
C
Catalin Marinas 已提交
177
	}
178
	BUG_ON(pud_bad(*pud));
C
Catalin Marinas 已提交
179

180
	pmd = pmd_set_fixmap_offset(pud, addr);
C
Catalin Marinas 已提交
181
	do {
182 183
		pmd_t old_pmd = *pmd;

C
Catalin Marinas 已提交
184
		next = pmd_addr_end(addr, end);
185

C
Catalin Marinas 已提交
186
		/* try section mapping first */
187
		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
188
		      !page_mappings_only) {
189 190 191 192 193 194 195 196 197 198 199 200
			/*
			 * Set the contiguous bit for the subsequent group of
			 * PMDs if its size and alignment are appropriate.
			 */
			if (((addr | phys) & ~CONT_PMD_MASK) == 0) {
				if (end - addr >= CONT_PMD_SIZE)
					__prot = __pgprot(pgprot_val(prot) |
							  PTE_CONT);
				else
					__prot = prot;
			}
			pmd_set_huge(pmd, phys, __prot);
201

202
			/*
203 204
			 * After the PMD entry has been populated once, we
			 * only allow updates to the permission attributes.
205
			 */
206 207
			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
						      pmd_val(*pmd)));
208
		} else {
209
			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
210 211
				       prot, pgtable_alloc,
				       page_mappings_only);
212 213 214

			BUG_ON(pmd_val(old_pmd) != 0 &&
			       pmd_val(old_pmd) != pmd_val(*pmd));
215
		}
C
Catalin Marinas 已提交
216 217
		phys += next - addr;
	} while (pmd++, addr = next, addr != end);
218 219

	pmd_clear_fixmap();
C
Catalin Marinas 已提交
220 221
}

222 223 224 225 226 227 228 229 230 231 232 233
static inline bool use_1G_block(unsigned long addr, unsigned long next,
			unsigned long phys)
{
	if (PAGE_SHIFT != 12)
		return false;

	if (((addr | next | phys) & ~PUD_MASK) != 0)
		return false;

	return true;
}

234
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
235
				  phys_addr_t phys, pgprot_t prot,
236
				  phys_addr_t (*pgtable_alloc)(void),
237
				  bool page_mappings_only)
C
Catalin Marinas 已提交
238
{
239
	pud_t *pud;
C
Catalin Marinas 已提交
240 241
	unsigned long next;

242
	if (pgd_none(*pgd)) {
243 244 245
		phys_addr_t pud_phys;
		BUG_ON(!pgtable_alloc);
		pud_phys = pgtable_alloc();
246
		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
247 248 249
	}
	BUG_ON(pgd_bad(*pgd));

250
	pud = pud_set_fixmap_offset(pgd, addr);
C
Catalin Marinas 已提交
251
	do {
252 253
		pud_t old_pud = *pud;

C
Catalin Marinas 已提交
254
		next = pud_addr_end(addr, end);
255 256 257 258

		/*
		 * For 4K granule only, attempt to put down a 1GB block
		 */
259
		if (use_1G_block(addr, next, phys) && !page_mappings_only) {
260
			pud_set_huge(pud, phys, prot);
261 262

			/*
263 264
			 * After the PUD entry has been populated once, we
			 * only allow updates to the permission attributes.
265
			 */
266 267
			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
						      pud_val(*pud)));
268
		} else {
269
			alloc_init_pmd(pud, addr, next, phys, prot,
270
				       pgtable_alloc, page_mappings_only);
271 272 273

			BUG_ON(pud_val(old_pud) != 0 &&
			       pud_val(old_pud) != pud_val(*pud));
274
		}
C
Catalin Marinas 已提交
275 276
		phys += next - addr;
	} while (pud++, addr = next, addr != end);
277 278

	pud_clear_fixmap();
C
Catalin Marinas 已提交
279 280
}

281 282 283 284
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
				 unsigned long virt, phys_addr_t size,
				 pgprot_t prot,
				 phys_addr_t (*pgtable_alloc)(void),
285
				 bool page_mappings_only)
C
Catalin Marinas 已提交
286 287
{
	unsigned long addr, length, end, next;
288
	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
C
Catalin Marinas 已提交
289

290 291 292 293 294 295 296
	/*
	 * If the virtual and physical address don't have the same offset
	 * within a page, we cannot map the region as the caller expects.
	 */
	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
		return;

297
	phys &= PAGE_MASK;
C
Catalin Marinas 已提交
298 299 300 301 302 303
	addr = virt & PAGE_MASK;
	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));

	end = addr + length;
	do {
		next = pgd_addr_end(addr, end);
304
		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
305
			       page_mappings_only);
C
Catalin Marinas 已提交
306 307 308 309
		phys += next - addr;
	} while (pgd++, addr = next, addr != end);
}

310
static phys_addr_t pgd_pgtable_alloc(void)
311
{
312
	void *ptr = (void *)__get_free_page(PGALLOC_GFP);
313 314
	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
		BUG();
315 316 317

	/* Ensure the zeroed page is visible to the page table walker */
	dsb(ishst);
318
	return __pa(ptr);
319 320
}

321 322 323 324 325 326
/*
 * This function can only be used to modify existing table entries,
 * without allocating new levels of table. Note that this permits the
 * creation of new section or page entries.
 */
static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
327
				  phys_addr_t size, pgprot_t prot)
328 329 330 331 332 333
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}
334
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, false);
335 336
}

337 338
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
			       unsigned long virt, phys_addr_t size,
339
			       pgprot_t prot, bool page_mappings_only)
340
{
341 342
	BUG_ON(mm == &init_mm);

343
	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
344
			     pgd_pgtable_alloc, page_mappings_only);
345 346
}

347 348 349 350 351 352 353 354 355
static void create_mapping_late(phys_addr_t phys, unsigned long virt,
				  phys_addr_t size, pgprot_t prot)
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}

356
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot,
357
			     NULL, debug_pagealloc_enabled());
358 359
}

360
static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
361
{
362
	unsigned long kernel_start = __pa(_text);
363
	unsigned long kernel_end = __pa(__init_begin);
364

365
	/*
366 367
	 * Take care not to create a writable alias for the
	 * read-only text and rodata sections of the kernel image.
368
	 */
369

370
	/* No overlap with the kernel text/rodata */
371 372 373
	if (end < kernel_start || start >= kernel_end) {
		__create_pgd_mapping(pgd, start, __phys_to_virt(start),
				     end - start, PAGE_KERNEL,
374
				     early_pgtable_alloc,
375
				     debug_pagealloc_enabled());
376
		return;
377 378
	}

379
	/*
380
	 * This block overlaps the kernel text/rodata mappings.
381
	 * Map the portion(s) which don't overlap.
382 383 384 385 386
	 */
	if (start < kernel_start)
		__create_pgd_mapping(pgd, start,
				     __phys_to_virt(start),
				     kernel_start - start, PAGE_KERNEL,
387
				     early_pgtable_alloc,
388
				     debug_pagealloc_enabled());
389 390 391 392
	if (kernel_end < end)
		__create_pgd_mapping(pgd, kernel_end,
				     __phys_to_virt(kernel_end),
				     end - kernel_end, PAGE_KERNEL,
393
				     early_pgtable_alloc,
394
				     debug_pagealloc_enabled());
395 396

	/*
397
	 * Map the linear alias of the [_text, __init_begin) interval as
398 399 400 401 402 403
	 * read-only/non-executable. This makes the contents of the
	 * region accessible to subsystems such as hibernate, but
	 * protects it from inadvertent modification or execution.
	 */
	__create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
			     kernel_end - kernel_start, PAGE_KERNEL_RO,
404
			     early_pgtable_alloc, debug_pagealloc_enabled());
405 406
}

407
static void __init map_mem(pgd_t *pgd)
C
Catalin Marinas 已提交
408 409
{
	struct memblock_region *reg;
410

C
Catalin Marinas 已提交
411 412 413 414 415 416 417
	/* map all the memory banks */
	for_each_memblock(memory, reg) {
		phys_addr_t start = reg->base;
		phys_addr_t end = start + reg->size;

		if (start >= end)
			break;
418 419
		if (memblock_is_nomap(reg))
			continue;
C
Catalin Marinas 已提交
420

421
		__map_memblock(pgd, start, end);
C
Catalin Marinas 已提交
422 423 424
	}
}

425 426
void mark_rodata_ro(void)
{
J
Jeremy Linton 已提交
427
	unsigned long section_size;
428

429
	section_size = (unsigned long)_etext - (unsigned long)_text;
430
	create_mapping_late(__pa(_text), (unsigned long)_text,
J
Jeremy Linton 已提交
431 432
			    section_size, PAGE_KERNEL_ROX);
	/*
433 434
	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
	 * to cover NOTES and EXCEPTION_TABLE.
J
Jeremy Linton 已提交
435
	 */
436
	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
J
Jeremy Linton 已提交
437 438
	create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata,
			    section_size, PAGE_KERNEL_RO);
439 440 441

	/* flush the TLBs after updating live kernel mappings */
	flush_tlb_all();
442 443

	debug_checkwx();
444 445
}

446 447
static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
				      pgprot_t prot, struct vm_struct *vma)
448 449 450 451 452 453 454 455
{
	phys_addr_t pa_start = __pa(va_start);
	unsigned long size = va_end - va_start;

	BUG_ON(!PAGE_ALIGNED(pa_start));
	BUG_ON(!PAGE_ALIGNED(size));

	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
456
			     early_pgtable_alloc, debug_pagealloc_enabled());
457 458 459 460 461 462 463 464

	vma->addr	= va_start;
	vma->phys_addr	= pa_start;
	vma->size	= size;
	vma->flags	= VM_MAP;
	vma->caller	= __builtin_return_address(0);

	vm_area_add_early(vma);
465 466 467 468 469 470 471
}

/*
 * Create fine-grained mappings for the kernel.
 */
static void __init map_kernel(pgd_t *pgd)
{
J
Jeremy Linton 已提交
472
	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
473

474 475
	map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text);
	map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata);
476 477 478
	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
			   &vmlinux_init);
	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
479

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
		/*
		 * The fixmap falls in a separate pgd to the kernel, and doesn't
		 * live in the carveout for the swapper_pg_dir. We can simply
		 * re-use the existing dir for the fixmap.
		 */
		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
			*pgd_offset_k(FIXADDR_START));
	} else if (CONFIG_PGTABLE_LEVELS > 3) {
		/*
		 * The fixmap shares its top level pgd entry with the kernel
		 * mapping. This can really only occur when we are running
		 * with 16k/4 levels, so we can simply reuse the pud level
		 * entry instead.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START),
			__pud(__pa(bm_pmd) | PUD_TYPE_TABLE));
		pud_clear_fixmap();
	} else {
		BUG();
	}
502 503 504 505

	kasan_copy_shadow(pgd);
}

C
Catalin Marinas 已提交
506 507 508 509 510 511
/*
 * paging_init() sets up the page tables, initialises the zone memory
 * maps and sets up the zero page.
 */
void __init paging_init(void)
{
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
	phys_addr_t pgd_phys = early_pgtable_alloc();
	pgd_t *pgd = pgd_set_fixmap(pgd_phys);

	map_kernel(pgd);
	map_mem(pgd);

	/*
	 * We want to reuse the original swapper_pg_dir so we don't have to
	 * communicate the new address to non-coherent secondaries in
	 * secondary_entry, and so cpu_switch_mm can generate the address with
	 * adrp+add rather than a load from some global variable.
	 *
	 * To do this we need to go via a temporary pgd.
	 */
	cpu_replace_ttbr1(__va(pgd_phys));
	memcpy(swapper_pg_dir, pgd, PAGE_SIZE);
	cpu_replace_ttbr1(swapper_pg_dir);

	pgd_clear_fixmap();
	memblock_free(pgd_phys, PAGE_SIZE);

	/*
	 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
	 * allocated with it.
	 */
	memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE,
		      SWAPPER_DIR_SIZE - PAGE_SIZE);
C
Catalin Marinas 已提交
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
}

/*
 * Check whether a kernel address is valid (derived from arch/x86/).
 */
int kern_addr_valid(unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	if ((((long)addr) >> VA_BITS) != -1UL)
		return 0;

	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0;

562 563 564
	if (pud_sect(*pud))
		return pfn_valid(pud_pfn(*pud));

C
Catalin Marinas 已提交
565 566 567 568
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;

569 570 571
	if (pmd_sect(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

C
Catalin Marinas 已提交
572 573 574 575 576 577 578
	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;

	return pfn_valid(pte_pfn(*pte));
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
579
#if !ARM64_SWAPPER_USES_SECTION_MAPS
580
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
581
{
582
	return vmemmap_populate_basepages(start, end, node);
C
Catalin Marinas 已提交
583
}
584
#else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
585
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
586
{
587
	unsigned long addr = start;
C
Catalin Marinas 已提交
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	do {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(*pmd)) {
			void *p = NULL;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
			if (!p)
				return -ENOMEM;

612
			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
C
Catalin Marinas 已提交
613 614 615 616 617 618 619
		} else
			vmemmap_verify((pte_t *)pmd, node, addr, next);
	} while (addr = next, addr != end);

	return 0;
}
#endif	/* CONFIG_ARM64_64K_PAGES */
620
void vmemmap_free(unsigned long start, unsigned long end)
621 622
{
}
C
Catalin Marinas 已提交
623
#endif	/* CONFIG_SPARSEMEM_VMEMMAP */
624 625 626 627 628 629 630

static inline pud_t * fixmap_pud(unsigned long addr)
{
	pgd_t *pgd = pgd_offset_k(addr);

	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));

631
	return pud_offset_kimg(pgd, addr);
632 633 634 635 636 637 638 639
}

static inline pmd_t * fixmap_pmd(unsigned long addr)
{
	pud_t *pud = fixmap_pud(addr);

	BUG_ON(pud_none(*pud) || pud_bad(*pud));

640
	return pmd_offset_kimg(pud, addr);
641 642 643 644
}

static inline pte_t * fixmap_pte(unsigned long addr)
{
645
	return &bm_pte[pte_index(addr)];
646 647 648 649 650 651 652 653 654 655
}

void __init early_fixmap_init(void)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	unsigned long addr = FIXADDR_START;

	pgd = pgd_offset_k(addr);
656 657
	if (CONFIG_PGTABLE_LEVELS > 3 &&
	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) {
658 659 660 661 662 663 664 665 666 667 668
		/*
		 * We only end up here if the kernel mapping and the fixmap
		 * share the top level pgd entry, which should only happen on
		 * 16k/4 levels configurations.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		pud = pud_offset_kimg(pgd, addr);
	} else {
		pgd_populate(&init_mm, pgd, bm_pud);
		pud = fixmap_pud(addr);
	}
669
	pud_populate(&init_mm, pud, bm_pmd);
670
	pmd = fixmap_pmd(addr);
671 672 673 674
	pmd_populate_kernel(&init_mm, pmd, bm_pte);

	/*
	 * The boot-ioremap range spans multiple pmds, for which
675
	 * we are not prepared:
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
	 */
	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));

	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		pr_warn("pmd %p != %p, %p\n",
			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
			fix_to_virt(FIX_BTMAP_BEGIN));
		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
			fix_to_virt(FIX_BTMAP_END));

		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
	}
}

void __set_fixmap(enum fixed_addresses idx,
			       phys_addr_t phys, pgprot_t flags)
{
	unsigned long addr = __fix_to_virt(idx);
	pte_t *pte;

702
	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
703 704 705 706 707 708 709 710 711 712

	pte = fixmap_pte(addr);

	if (pgprot_val(flags)) {
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
	} else {
		pte_clear(&init_mm, addr, pte);
		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
	}
}
713

714
void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
715 716
{
	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
717
	int offset;
718 719 720 721 722
	void *dt_virt;

	/*
	 * Check whether the physical FDT address is set and meets the minimum
	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
723 724 725
	 * at least 8 bytes so that we can always access the magic and size
	 * fields of the FDT header after mapping the first chunk, double check
	 * here if that is indeed the case.
726 727 728 729 730 731 732 733
	 */
	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
		return NULL;

	/*
	 * Make sure that the FDT region can be mapped without the need to
	 * allocate additional translation table pages, so that it is safe
734
	 * to call create_mapping_noalloc() this early.
735 736 737 738 739 740 741 742
	 *
	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
	 * be in the same PMD as the rest of the fixmap.
	 * On 4k pages, we'll use section mappings for the FDT so we only
	 * have to be in the same PUD.
	 */
	BUILD_BUG_ON(dt_virt_base % SZ_2M);

743 744
	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
745

746
	offset = dt_phys % SWAPPER_BLOCK_SIZE;
747 748 749
	dt_virt = (void *)dt_virt_base + offset;

	/* map the first chunk so we can read the size from the header */
750 751
	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
752

753
	if (fdt_magic(dt_virt) != FDT_MAGIC)
754 755
		return NULL;

756 757
	*size = fdt_totalsize(dt_virt);
	if (*size > MAX_FDT_SIZE)
758 759
		return NULL;

760
	if (offset + *size > SWAPPER_BLOCK_SIZE)
761
		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
762
			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
763

764 765
	return dt_virt;
}
766

767 768 769 770 771 772 773 774 775 776
void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
{
	void *dt_virt;
	int size;

	dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO);
	if (!dt_virt)
		return NULL;

	memblock_reserve(dt_phys, size);
777 778
	return dt_virt;
}
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819

int __init arch_ioremap_pud_supported(void)
{
	/* only 4k granule supports level 1 block mappings */
	return IS_ENABLED(CONFIG_ARM64_4K_PAGES);
}

int __init arch_ioremap_pmd_supported(void)
{
	return 1;
}

int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PUD_MASK);
	set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PMD_MASK);
	set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pud_clear_huge(pud_t *pud)
{
	if (!pud_sect(*pud))
		return 0;
	pud_clear(pud);
	return 1;
}

int pmd_clear_huge(pmd_t *pmd)
{
	if (!pmd_sect(*pmd))
		return 0;
	pmd_clear(pmd);
	return 1;
}