mmu.c 21.1 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Based on arch/arm/mm/mmu.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/cache.h>
C
Catalin Marinas 已提交
21 22 23 24
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
25
#include <linux/libfdt.h>
C
Catalin Marinas 已提交
26 27 28 29
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/memblock.h>
#include <linux/fs.h>
30
#include <linux/io.h>
31
#include <linux/mm.h>
C
Catalin Marinas 已提交
32

33
#include <asm/barrier.h>
C
Catalin Marinas 已提交
34
#include <asm/cputype.h>
35
#include <asm/fixmap.h>
36
#include <asm/kasan.h>
37
#include <asm/kernel-pgtable.h>
C
Catalin Marinas 已提交
38 39 40 41
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
42
#include <asm/memblock.h>
C
Catalin Marinas 已提交
43
#include <asm/mmu_context.h>
44
#include <asm/ptdump.h>
C
Catalin Marinas 已提交
45

46 47
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);

48
u64 kimage_voffset __ro_after_init;
49 50
EXPORT_SYMBOL(kimage_voffset);

C
Catalin Marinas 已提交
51 52 53 54
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
55
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
C
Catalin Marinas 已提交
56 57
EXPORT_SYMBOL(empty_zero_page);

58 59 60 61
static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;

C
Catalin Marinas 已提交
62 63 64 65 66 67 68 69 70 71 72
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (!pfn_valid(pfn))
		return pgprot_noncached(vma_prot);
	else if (file->f_flags & O_SYNC)
		return pgprot_writecombine(vma_prot);
	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

73
static phys_addr_t __init early_pgtable_alloc(void)
C
Catalin Marinas 已提交
74
{
75 76 77
	phys_addr_t phys;
	void *ptr;

78
	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
79 80 81 82 83 84 85 86

	/*
	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
	 * any level of table.
	 */
	ptr = pte_set_fixmap(phys);

87 88
	memset(ptr, 0, PAGE_SIZE);

89 90 91 92 93 94 95
	/*
	 * Implicit barriers also ensure the zeroed page is visible to the page
	 * table walker
	 */
	pte_clear_fixmap();

	return phys;
C
Catalin Marinas 已提交
96 97
}

98 99 100 101 102 103 104 105 106 107 108
static bool pgattr_change_is_safe(u64 old, u64 new)
{
	/*
	 * The following mapping attributes may be updated in live
	 * kernel mappings without the need for break-before-make.
	 */
	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;

	return old  == 0 || new  == 0 || ((old ^ new) & ~mask) == 0;
}

109
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
110
				  unsigned long end, unsigned long pfn,
111
				  pgprot_t prot,
112
				  phys_addr_t (*pgtable_alloc)(void))
C
Catalin Marinas 已提交
113 114 115
{
	pte_t *pte;

116 117
	BUG_ON(pmd_sect(*pmd));
	if (pmd_none(*pmd)) {
118 119 120
		phys_addr_t pte_phys;
		BUG_ON(!pgtable_alloc);
		pte_phys = pgtable_alloc();
121 122 123
		pte = pte_set_fixmap(pte_phys);
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
		pte_clear_fixmap();
C
Catalin Marinas 已提交
124
	}
125
	BUG_ON(pmd_bad(*pmd));
C
Catalin Marinas 已提交
126

127
	pte = pte_set_fixmap_offset(pmd, addr);
C
Catalin Marinas 已提交
128
	do {
129 130
		pte_t old_pte = *pte;

131
		set_pte(pte, pfn_pte(pfn, prot));
132
		pfn++;
133 134 135 136 137 138 139

		/*
		 * After the PTE entry has been populated once, we
		 * only allow updates to the permission attributes.
		 */
		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));

140
	} while (pte++, addr += PAGE_SIZE, addr != end);
141 142

	pte_clear_fixmap();
C
Catalin Marinas 已提交
143 144
}

145
static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
146
				  phys_addr_t phys, pgprot_t prot,
147
				  phys_addr_t (*pgtable_alloc)(void),
148
				  bool page_mappings_only)
C
Catalin Marinas 已提交
149 150 151 152 153 154 155
{
	pmd_t *pmd;
	unsigned long next;

	/*
	 * Check for initial section mappings in the pgd/pud and remove them.
	 */
156 157
	BUG_ON(pud_sect(*pud));
	if (pud_none(*pud)) {
158 159 160
		phys_addr_t pmd_phys;
		BUG_ON(!pgtable_alloc);
		pmd_phys = pgtable_alloc();
161 162 163
		pmd = pmd_set_fixmap(pmd_phys);
		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
		pmd_clear_fixmap();
C
Catalin Marinas 已提交
164
	}
165
	BUG_ON(pud_bad(*pud));
C
Catalin Marinas 已提交
166

167
	pmd = pmd_set_fixmap_offset(pud, addr);
C
Catalin Marinas 已提交
168
	do {
169 170
		pmd_t old_pmd = *pmd;

C
Catalin Marinas 已提交
171
		next = pmd_addr_end(addr, end);
172

C
Catalin Marinas 已提交
173
		/* try section mapping first */
174
		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
175
		      !page_mappings_only) {
176
			pmd_set_huge(pmd, phys, prot);
177

178
			/*
179 180
			 * After the PMD entry has been populated once, we
			 * only allow updates to the permission attributes.
181
			 */
182 183
			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
						      pmd_val(*pmd)));
184
		} else {
185
			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
186
				       prot, pgtable_alloc);
187 188 189

			BUG_ON(pmd_val(old_pmd) != 0 &&
			       pmd_val(old_pmd) != pmd_val(*pmd));
190
		}
C
Catalin Marinas 已提交
191 192
		phys += next - addr;
	} while (pmd++, addr = next, addr != end);
193 194

	pmd_clear_fixmap();
C
Catalin Marinas 已提交
195 196
}

197 198 199 200 201 202 203 204 205 206 207 208
static inline bool use_1G_block(unsigned long addr, unsigned long next,
			unsigned long phys)
{
	if (PAGE_SHIFT != 12)
		return false;

	if (((addr | next | phys) & ~PUD_MASK) != 0)
		return false;

	return true;
}

209
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
210
				  phys_addr_t phys, pgprot_t prot,
211
				  phys_addr_t (*pgtable_alloc)(void),
212
				  bool page_mappings_only)
C
Catalin Marinas 已提交
213
{
214
	pud_t *pud;
C
Catalin Marinas 已提交
215 216
	unsigned long next;

217
	if (pgd_none(*pgd)) {
218 219 220
		phys_addr_t pud_phys;
		BUG_ON(!pgtable_alloc);
		pud_phys = pgtable_alloc();
221
		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
222 223 224
	}
	BUG_ON(pgd_bad(*pgd));

225
	pud = pud_set_fixmap_offset(pgd, addr);
C
Catalin Marinas 已提交
226
	do {
227 228
		pud_t old_pud = *pud;

C
Catalin Marinas 已提交
229
		next = pud_addr_end(addr, end);
230 231 232 233

		/*
		 * For 4K granule only, attempt to put down a 1GB block
		 */
234
		if (use_1G_block(addr, next, phys) && !page_mappings_only) {
235
			pud_set_huge(pud, phys, prot);
236 237

			/*
238 239
			 * After the PUD entry has been populated once, we
			 * only allow updates to the permission attributes.
240
			 */
241 242
			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
						      pud_val(*pud)));
243
		} else {
244
			alloc_init_pmd(pud, addr, next, phys, prot,
245
				       pgtable_alloc, page_mappings_only);
246 247 248

			BUG_ON(pud_val(old_pud) != 0 &&
			       pud_val(old_pud) != pud_val(*pud));
249
		}
C
Catalin Marinas 已提交
250 251
		phys += next - addr;
	} while (pud++, addr = next, addr != end);
252 253

	pud_clear_fixmap();
C
Catalin Marinas 已提交
254 255
}

256 257 258 259
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
				 unsigned long virt, phys_addr_t size,
				 pgprot_t prot,
				 phys_addr_t (*pgtable_alloc)(void),
260
				 bool page_mappings_only)
C
Catalin Marinas 已提交
261 262
{
	unsigned long addr, length, end, next;
263
	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
C
Catalin Marinas 已提交
264

265 266 267 268 269 270 271
	/*
	 * If the virtual and physical address don't have the same offset
	 * within a page, we cannot map the region as the caller expects.
	 */
	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
		return;

272
	phys &= PAGE_MASK;
C
Catalin Marinas 已提交
273 274 275 276 277 278
	addr = virt & PAGE_MASK;
	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));

	end = addr + length;
	do {
		next = pgd_addr_end(addr, end);
279
		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
280
			       page_mappings_only);
C
Catalin Marinas 已提交
281 282 283 284
		phys += next - addr;
	} while (pgd++, addr = next, addr != end);
}

285
static phys_addr_t pgd_pgtable_alloc(void)
286
{
287
	void *ptr = (void *)__get_free_page(PGALLOC_GFP);
288 289
	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
		BUG();
290 291 292

	/* Ensure the zeroed page is visible to the page table walker */
	dsb(ishst);
293
	return __pa(ptr);
294 295
}

296 297 298 299 300 301
/*
 * This function can only be used to modify existing table entries,
 * without allocating new levels of table. Note that this permits the
 * creation of new section or page entries.
 */
static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
302
				  phys_addr_t size, pgprot_t prot)
303 304 305 306 307 308
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}
309
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, false);
310 311
}

312 313
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
			       unsigned long virt, phys_addr_t size,
314
			       pgprot_t prot, bool page_mappings_only)
315
{
316 317
	BUG_ON(mm == &init_mm);

318
	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
319
			     pgd_pgtable_alloc, page_mappings_only);
320 321
}

322 323
static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
				phys_addr_t size, pgprot_t prot)
324 325
{
	if (virt < VMALLOC_START) {
326
		pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
327 328 329 330
			&phys, virt);
		return;
	}

331
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot,
332
			     NULL, debug_pagealloc_enabled());
333 334 335

	/* flush the TLBs after updating live kernel mappings */
	flush_tlb_kernel_range(virt, virt + size);
336 337
}

338
static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
339
{
340 341
	phys_addr_t kernel_start = __pa_symbol(_text);
	phys_addr_t kernel_end = __pa_symbol(__init_begin);
342

343
	/*
344 345
	 * Take care not to create a writable alias for the
	 * read-only text and rodata sections of the kernel image.
346
	 */
347

348
	/* No overlap with the kernel text/rodata */
349 350 351
	if (end < kernel_start || start >= kernel_end) {
		__create_pgd_mapping(pgd, start, __phys_to_virt(start),
				     end - start, PAGE_KERNEL,
352
				     early_pgtable_alloc,
353
				     debug_pagealloc_enabled());
354
		return;
355 356
	}

357
	/*
358
	 * This block overlaps the kernel text/rodata mappings.
359
	 * Map the portion(s) which don't overlap.
360 361 362 363 364
	 */
	if (start < kernel_start)
		__create_pgd_mapping(pgd, start,
				     __phys_to_virt(start),
				     kernel_start - start, PAGE_KERNEL,
365
				     early_pgtable_alloc,
366
				     debug_pagealloc_enabled());
367 368 369 370
	if (kernel_end < end)
		__create_pgd_mapping(pgd, kernel_end,
				     __phys_to_virt(kernel_end),
				     end - kernel_end, PAGE_KERNEL,
371
				     early_pgtable_alloc,
372
				     debug_pagealloc_enabled());
373 374

	/*
375 376 377 378 379 380
	 * Map the linear alias of the [_text, __init_begin) interval
	 * as non-executable now, and remove the write permission in
	 * mark_linear_text_alias_ro() below (which will be called after
	 * alternative patching has completed). This makes the contents
	 * of the region accessible to subsystems such as hibernate,
	 * but protects it from inadvertent modification or execution.
381 382
	 */
	__create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
383
			     kernel_end - kernel_start, PAGE_KERNEL,
384
			     early_pgtable_alloc, debug_pagealloc_enabled());
385 386
}

387 388 389 390 391 392 393 394 395 396
void __init mark_linear_text_alias_ro(void)
{
	/*
	 * Remove the write permissions from the linear alias of .text/.rodata
	 */
	update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
			    (unsigned long)__init_begin - (unsigned long)_text,
			    PAGE_KERNEL_RO);
}

397
static void __init map_mem(pgd_t *pgd)
C
Catalin Marinas 已提交
398 399
{
	struct memblock_region *reg;
400

C
Catalin Marinas 已提交
401 402 403 404 405 406 407
	/* map all the memory banks */
	for_each_memblock(memory, reg) {
		phys_addr_t start = reg->base;
		phys_addr_t end = start + reg->size;

		if (start >= end)
			break;
408 409
		if (memblock_is_nomap(reg))
			continue;
C
Catalin Marinas 已提交
410

411
		__map_memblock(pgd, start, end);
C
Catalin Marinas 已提交
412 413 414
	}
}

415 416
void mark_rodata_ro(void)
{
J
Jeremy Linton 已提交
417
	unsigned long section_size;
418

419
	section_size = (unsigned long)_etext - (unsigned long)_text;
420
	update_mapping_prot(__pa_symbol(_text), (unsigned long)_text,
J
Jeremy Linton 已提交
421 422
			    section_size, PAGE_KERNEL_ROX);
	/*
423 424
	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
	 * to cover NOTES and EXCEPTION_TABLE.
J
Jeremy Linton 已提交
425
	 */
426
	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
427
	update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
J
Jeremy Linton 已提交
428
			    section_size, PAGE_KERNEL_RO);
429

430
	debug_checkwx();
431 432
}

433 434
static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
				      pgprot_t prot, struct vm_struct *vma)
435
{
436
	phys_addr_t pa_start = __pa_symbol(va_start);
437 438 439 440 441 442
	unsigned long size = va_end - va_start;

	BUG_ON(!PAGE_ALIGNED(pa_start));
	BUG_ON(!PAGE_ALIGNED(size));

	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
443
			     early_pgtable_alloc, debug_pagealloc_enabled());
444 445 446 447 448 449 450 451

	vma->addr	= va_start;
	vma->phys_addr	= pa_start;
	vma->size	= size;
	vma->flags	= VM_MAP;
	vma->caller	= __builtin_return_address(0);

	vm_area_add_early(vma);
452 453 454 455 456 457 458
}

/*
 * Create fine-grained mappings for the kernel.
 */
static void __init map_kernel(pgd_t *pgd)
{
J
Jeremy Linton 已提交
459
	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
460

461 462
	map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text);
	map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata);
463 464 465
	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
			   &vmlinux_init);
	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
466

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
		/*
		 * The fixmap falls in a separate pgd to the kernel, and doesn't
		 * live in the carveout for the swapper_pg_dir. We can simply
		 * re-use the existing dir for the fixmap.
		 */
		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
			*pgd_offset_k(FIXADDR_START));
	} else if (CONFIG_PGTABLE_LEVELS > 3) {
		/*
		 * The fixmap shares its top level pgd entry with the kernel
		 * mapping. This can really only occur when we are running
		 * with 16k/4 levels, so we can simply reuse the pud level
		 * entry instead.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START),
484
			__pud(__pa_symbol(bm_pmd) | PUD_TYPE_TABLE));
485 486 487 488
		pud_clear_fixmap();
	} else {
		BUG();
	}
489 490 491 492

	kasan_copy_shadow(pgd);
}

C
Catalin Marinas 已提交
493 494 495 496 497 498
/*
 * paging_init() sets up the page tables, initialises the zone memory
 * maps and sets up the zero page.
 */
void __init paging_init(void)
{
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
	phys_addr_t pgd_phys = early_pgtable_alloc();
	pgd_t *pgd = pgd_set_fixmap(pgd_phys);

	map_kernel(pgd);
	map_mem(pgd);

	/*
	 * We want to reuse the original swapper_pg_dir so we don't have to
	 * communicate the new address to non-coherent secondaries in
	 * secondary_entry, and so cpu_switch_mm can generate the address with
	 * adrp+add rather than a load from some global variable.
	 *
	 * To do this we need to go via a temporary pgd.
	 */
	cpu_replace_ttbr1(__va(pgd_phys));
514
	memcpy(swapper_pg_dir, pgd, PGD_SIZE);
515
	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
516 517 518 519 520 521 522 523

	pgd_clear_fixmap();
	memblock_free(pgd_phys, PAGE_SIZE);

	/*
	 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
	 * allocated with it.
	 */
524
	memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
525
		      SWAPPER_DIR_SIZE - PAGE_SIZE);
C
Catalin Marinas 已提交
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
}

/*
 * Check whether a kernel address is valid (derived from arch/x86/).
 */
int kern_addr_valid(unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	if ((((long)addr) >> VA_BITS) != -1UL)
		return 0;

	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0;

549 550 551
	if (pud_sect(*pud))
		return pfn_valid(pud_pfn(*pud));

C
Catalin Marinas 已提交
552 553 554 555
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;

556 557 558
	if (pmd_sect(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

C
Catalin Marinas 已提交
559 560 561 562 563 564 565
	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;

	return pfn_valid(pte_pfn(*pte));
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
566
#if !ARM64_SWAPPER_USES_SECTION_MAPS
567
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
568
{
569
	return vmemmap_populate_basepages(start, end, node);
C
Catalin Marinas 已提交
570
}
571
#else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
572
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
573
{
574
	unsigned long addr = start;
C
Catalin Marinas 已提交
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	do {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(*pmd)) {
			void *p = NULL;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
			if (!p)
				return -ENOMEM;

599
			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
C
Catalin Marinas 已提交
600 601 602 603 604 605 606
		} else
			vmemmap_verify((pte_t *)pmd, node, addr, next);
	} while (addr = next, addr != end);

	return 0;
}
#endif	/* CONFIG_ARM64_64K_PAGES */
607
void vmemmap_free(unsigned long start, unsigned long end)
608 609
{
}
C
Catalin Marinas 已提交
610
#endif	/* CONFIG_SPARSEMEM_VMEMMAP */
611 612 613 614 615 616 617

static inline pud_t * fixmap_pud(unsigned long addr)
{
	pgd_t *pgd = pgd_offset_k(addr);

	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));

618
	return pud_offset_kimg(pgd, addr);
619 620 621 622 623 624 625 626
}

static inline pmd_t * fixmap_pmd(unsigned long addr)
{
	pud_t *pud = fixmap_pud(addr);

	BUG_ON(pud_none(*pud) || pud_bad(*pud));

627
	return pmd_offset_kimg(pud, addr);
628 629 630 631
}

static inline pte_t * fixmap_pte(unsigned long addr)
{
632
	return &bm_pte[pte_index(addr)];
633 634
}

635 636 637 638 639 640
/*
 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
 * directly on kernel symbols (bm_p*d). This function is called too early to use
 * lm_alias so __p*d_populate functions must be used to populate with the
 * physical address from __pa_symbol.
 */
641 642 643 644 645 646 647 648
void __init early_fixmap_init(void)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	unsigned long addr = FIXADDR_START;

	pgd = pgd_offset_k(addr);
649
	if (CONFIG_PGTABLE_LEVELS > 3 &&
650
	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) {
651 652 653 654 655 656 657 658
		/*
		 * We only end up here if the kernel mapping and the fixmap
		 * share the top level pgd entry, which should only happen on
		 * 16k/4 levels configurations.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		pud = pud_offset_kimg(pgd, addr);
	} else {
659 660
		if (pgd_none(*pgd))
			__pgd_populate(pgd, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
661 662
		pud = fixmap_pud(addr);
	}
663 664
	if (pud_none(*pud))
		__pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
665
	pmd = fixmap_pmd(addr);
666
	__pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
667 668 669

	/*
	 * The boot-ioremap range spans multiple pmds, for which
670
	 * we are not prepared:
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
	 */
	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));

	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		pr_warn("pmd %p != %p, %p\n",
			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
			fix_to_virt(FIX_BTMAP_BEGIN));
		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
			fix_to_virt(FIX_BTMAP_END));

		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
	}
}

void __set_fixmap(enum fixed_addresses idx,
			       phys_addr_t phys, pgprot_t flags)
{
	unsigned long addr = __fix_to_virt(idx);
	pte_t *pte;

697
	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
698 699 700 701 702 703 704 705 706 707

	pte = fixmap_pte(addr);

	if (pgprot_val(flags)) {
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
	} else {
		pte_clear(&init_mm, addr, pte);
		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
	}
}
708

709
void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
710 711
{
	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
712
	int offset;
713 714 715 716 717
	void *dt_virt;

	/*
	 * Check whether the physical FDT address is set and meets the minimum
	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
718 719 720
	 * at least 8 bytes so that we can always access the magic and size
	 * fields of the FDT header after mapping the first chunk, double check
	 * here if that is indeed the case.
721 722 723 724 725 726 727 728
	 */
	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
		return NULL;

	/*
	 * Make sure that the FDT region can be mapped without the need to
	 * allocate additional translation table pages, so that it is safe
729
	 * to call create_mapping_noalloc() this early.
730 731 732 733 734 735 736 737
	 *
	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
	 * be in the same PMD as the rest of the fixmap.
	 * On 4k pages, we'll use section mappings for the FDT so we only
	 * have to be in the same PUD.
	 */
	BUILD_BUG_ON(dt_virt_base % SZ_2M);

738 739
	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
740

741
	offset = dt_phys % SWAPPER_BLOCK_SIZE;
742 743 744
	dt_virt = (void *)dt_virt_base + offset;

	/* map the first chunk so we can read the size from the header */
745 746
	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
747

748
	if (fdt_magic(dt_virt) != FDT_MAGIC)
749 750
		return NULL;

751 752
	*size = fdt_totalsize(dt_virt);
	if (*size > MAX_FDT_SIZE)
753 754
		return NULL;

755
	if (offset + *size > SWAPPER_BLOCK_SIZE)
756
		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
757
			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
758

759 760
	return dt_virt;
}
761

762 763 764 765 766 767 768 769 770 771
void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
{
	void *dt_virt;
	int size;

	dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO);
	if (!dt_virt)
		return NULL;

	memblock_reserve(dt_phys, size);
772 773
	return dt_virt;
}
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814

int __init arch_ioremap_pud_supported(void)
{
	/* only 4k granule supports level 1 block mappings */
	return IS_ENABLED(CONFIG_ARM64_4K_PAGES);
}

int __init arch_ioremap_pmd_supported(void)
{
	return 1;
}

int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PUD_MASK);
	set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PMD_MASK);
	set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pud_clear_huge(pud_t *pud)
{
	if (!pud_sect(*pud))
		return 0;
	pud_clear(pud);
	return 1;
}

int pmd_clear_huge(pmd_t *pmd)
{
	if (!pmd_sect(*pmd))
		return 0;
	pmd_clear(pmd);
	return 1;
}