mmu.c 21.2 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Based on arch/arm/mm/mmu.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
24
#include <linux/libfdt.h>
C
Catalin Marinas 已提交
25 26 27 28
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/memblock.h>
#include <linux/fs.h>
29
#include <linux/io.h>
30
#include <linux/slab.h>
31
#include <linux/stop_machine.h>
C
Catalin Marinas 已提交
32

33
#include <asm/barrier.h>
C
Catalin Marinas 已提交
34
#include <asm/cputype.h>
35
#include <asm/fixmap.h>
36
#include <asm/kasan.h>
37
#include <asm/kernel-pgtable.h>
C
Catalin Marinas 已提交
38 39 40 41
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
42
#include <asm/memblock.h>
C
Catalin Marinas 已提交
43 44 45 46
#include <asm/mmu_context.h>

#include "mm.h"

47 48
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);

49 50 51
u64 kimage_voffset __read_mostly;
EXPORT_SYMBOL(kimage_voffset);

C
Catalin Marinas 已提交
52 53 54 55
/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
56
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
C
Catalin Marinas 已提交
57 58
EXPORT_SYMBOL(empty_zero_page);

59 60 61 62
static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;

C
Catalin Marinas 已提交
63 64 65 66 67 68 69 70 71 72 73
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (!pfn_valid(pfn))
		return pgprot_noncached(vma_prot);
	else if (file->f_flags & O_SYNC)
		return pgprot_writecombine(vma_prot);
	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

74
static phys_addr_t __init early_pgtable_alloc(void)
C
Catalin Marinas 已提交
75
{
76 77 78
	phys_addr_t phys;
	void *ptr;

79
	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
80 81 82 83 84 85 86 87

	/*
	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
	 * any level of table.
	 */
	ptr = pte_set_fixmap(phys);

88 89
	memset(ptr, 0, PAGE_SIZE);

90 91 92 93 94 95 96
	/*
	 * Implicit barriers also ensure the zeroed page is visible to the page
	 * table walker
	 */
	pte_clear_fixmap();

	return phys;
C
Catalin Marinas 已提交
97 98
}

99 100 101 102 103 104 105 106 107 108 109
/*
 * remap a PMD into pages
 */
static void split_pmd(pmd_t *pmd, pte_t *pte)
{
	unsigned long pfn = pmd_pfn(*pmd);
	int i = 0;

	do {
		/*
		 * Need to have the least restrictive permissions available
110
		 * permissions will be fixed up later
111
		 */
112
		set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
113 114 115 116 117
		pfn++;
	} while (pte++, i++, i < PTRS_PER_PTE);
}

static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
118
				  unsigned long end, unsigned long pfn,
119
				  pgprot_t prot,
120
				  phys_addr_t (*pgtable_alloc)(void))
C
Catalin Marinas 已提交
121 122 123
{
	pte_t *pte;

124
	if (pmd_none(*pmd) || pmd_sect(*pmd)) {
125 126 127
		phys_addr_t pte_phys;
		BUG_ON(!pgtable_alloc);
		pte_phys = pgtable_alloc();
128
		pte = pte_set_fixmap(pte_phys);
129 130
		if (pmd_sect(*pmd))
			split_pmd(pmd, pte);
131
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
132
		flush_tlb_all();
133
		pte_clear_fixmap();
C
Catalin Marinas 已提交
134
	}
135
	BUG_ON(pmd_bad(*pmd));
C
Catalin Marinas 已提交
136

137
	pte = pte_set_fixmap_offset(pmd, addr);
C
Catalin Marinas 已提交
138
	do {
139 140 141
		set_pte(pte, pfn_pte(pfn, prot));
		pfn++;
	} while (pte++, addr += PAGE_SIZE, addr != end);
142 143

	pte_clear_fixmap();
C
Catalin Marinas 已提交
144 145
}

146
static void split_pud(pud_t *old_pud, pmd_t *pmd)
147 148 149 150 151 152
{
	unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
	pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
	int i = 0;

	do {
153
		set_pmd(pmd, __pmd(addr | pgprot_val(prot)));
154 155 156 157
		addr += PMD_SIZE;
	} while (pmd++, i++, i < PTRS_PER_PMD);
}

158
static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
159
				  phys_addr_t phys, pgprot_t prot,
160 161
				  phys_addr_t (*pgtable_alloc)(void),
				  bool allow_block_mappings)
C
Catalin Marinas 已提交
162 163 164 165 166 167 168
{
	pmd_t *pmd;
	unsigned long next;

	/*
	 * Check for initial section mappings in the pgd/pud and remove them.
	 */
169
	if (pud_none(*pud) || pud_sect(*pud)) {
170 171 172
		phys_addr_t pmd_phys;
		BUG_ON(!pgtable_alloc);
		pmd_phys = pgtable_alloc();
173
		pmd = pmd_set_fixmap(pmd_phys);
174 175 176 177 178 179 180
		if (pud_sect(*pud)) {
			/*
			 * need to have the 1G of mappings continue to be
			 * present
			 */
			split_pud(pud, pmd);
		}
181
		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
182
		flush_tlb_all();
183
		pmd_clear_fixmap();
C
Catalin Marinas 已提交
184
	}
185
	BUG_ON(pud_bad(*pud));
C
Catalin Marinas 已提交
186

187
	pmd = pmd_set_fixmap_offset(pud, addr);
C
Catalin Marinas 已提交
188 189 190
	do {
		next = pmd_addr_end(addr, end);
		/* try section mapping first */
191
		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
192
		      allow_block_mappings) {
193
			pmd_t old_pmd =*pmd;
194
			pmd_set_huge(pmd, phys, prot);
195 196 197 198
			/*
			 * Check for previous table entries created during
			 * boot (__create_page_tables) and flush them.
			 */
199
			if (!pmd_none(old_pmd)) {
200
				flush_tlb_all();
201
				if (pmd_table(old_pmd)) {
202
					phys_addr_t table = pmd_page_paddr(old_pmd);
203 204
					if (!WARN_ON_ONCE(slab_is_available()))
						memblock_free(table, PAGE_SIZE);
205 206
				}
			}
207
		} else {
208
			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
209
				       prot, pgtable_alloc);
210
		}
C
Catalin Marinas 已提交
211 212
		phys += next - addr;
	} while (pmd++, addr = next, addr != end);
213 214

	pmd_clear_fixmap();
C
Catalin Marinas 已提交
215 216
}

217 218 219 220 221 222 223 224 225 226 227 228
static inline bool use_1G_block(unsigned long addr, unsigned long next,
			unsigned long phys)
{
	if (PAGE_SHIFT != 12)
		return false;

	if (((addr | next | phys) & ~PUD_MASK) != 0)
		return false;

	return true;
}

229
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
230
				  phys_addr_t phys, pgprot_t prot,
231 232
				  phys_addr_t (*pgtable_alloc)(void),
				  bool allow_block_mappings)
C
Catalin Marinas 已提交
233
{
234
	pud_t *pud;
C
Catalin Marinas 已提交
235 236
	unsigned long next;

237
	if (pgd_none(*pgd)) {
238 239 240
		phys_addr_t pud_phys;
		BUG_ON(!pgtable_alloc);
		pud_phys = pgtable_alloc();
241
		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
242 243 244
	}
	BUG_ON(pgd_bad(*pgd));

245
	pud = pud_set_fixmap_offset(pgd, addr);
C
Catalin Marinas 已提交
246 247
	do {
		next = pud_addr_end(addr, end);
248 249 250 251

		/*
		 * For 4K granule only, attempt to put down a 1GB block
		 */
252
		if (use_1G_block(addr, next, phys) && allow_block_mappings) {
253
			pud_t old_pud = *pud;
254
			pud_set_huge(pud, phys, prot);
255 256 257 258 259 260 261 262 263 264

			/*
			 * If we have an old value for a pud, it will
			 * be pointing to a pmd table that we no longer
			 * need (from swapper_pg_dir).
			 *
			 * Look up the old pmd table and free it.
			 */
			if (!pud_none(old_pud)) {
				flush_tlb_all();
265
				if (pud_table(old_pud)) {
266
					phys_addr_t table = pud_page_paddr(old_pud);
267 268
					if (!WARN_ON_ONCE(slab_is_available()))
						memblock_free(table, PAGE_SIZE);
269
				}
270 271
			}
		} else {
272
			alloc_init_pmd(pud, addr, next, phys, prot,
273
				       pgtable_alloc, allow_block_mappings);
274
		}
C
Catalin Marinas 已提交
275 276
		phys += next - addr;
	} while (pud++, addr = next, addr != end);
277 278

	pud_clear_fixmap();
C
Catalin Marinas 已提交
279 280 281 282 283 284
}

/*
 * Create the page directory entries and any necessary page tables for the
 * mapping specified by 'md'.
 */
285
static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt,
286
				    phys_addr_t size, pgprot_t prot,
287 288
				    phys_addr_t (*pgtable_alloc)(void),
				    bool allow_block_mappings)
C
Catalin Marinas 已提交
289 290 291
{
	unsigned long addr, length, end, next;

292 293 294 295 296 297 298
	/*
	 * If the virtual and physical address don't have the same offset
	 * within a page, we cannot map the region as the caller expects.
	 */
	if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
		return;

299
	phys &= PAGE_MASK;
C
Catalin Marinas 已提交
300 301 302 303 304 305
	addr = virt & PAGE_MASK;
	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));

	end = addr + length;
	do {
		next = pgd_addr_end(addr, end);
306 307
		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
			       allow_block_mappings);
C
Catalin Marinas 已提交
308 309 310 311
		phys += next - addr;
	} while (pgd++, addr = next, addr != end);
}

312
static phys_addr_t late_pgtable_alloc(void)
313
{
314
	void *ptr = (void *)__get_free_page(PGALLOC_GFP);
315
	BUG_ON(!ptr);
316 317 318

	/* Ensure the zeroed page is visible to the page table walker */
	dsb(ishst);
319
	return __pa(ptr);
320 321
}

322 323 324
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
				 unsigned long virt, phys_addr_t size,
				 pgprot_t prot,
325 326
				 phys_addr_t (*alloc)(void),
				 bool allow_block_mappings)
327
{
328 329
	init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc,
		 allow_block_mappings);
330 331
}

332 333 334 335 336 337
/*
 * This function can only be used to modify existing table entries,
 * without allocating new levels of table. Note that this permits the
 * creation of new section or page entries.
 */
static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
338
				  phys_addr_t size, pgprot_t prot)
339 340 341 342 343 344
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}
345
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true);
346 347
}

348 349
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
			       unsigned long virt, phys_addr_t size,
350
			       pgprot_t prot, bool allow_block_mappings)
351
{
352
	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
353
			     late_pgtable_alloc, allow_block_mappings);
354 355
}

356 357 358 359 360 361 362 363 364
static void create_mapping_late(phys_addr_t phys, unsigned long virt,
				  phys_addr_t size, pgprot_t prot)
{
	if (virt < VMALLOC_START) {
		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
			&phys, virt);
		return;
	}

365
	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot,
366
			     late_pgtable_alloc, !debug_pagealloc_enabled());
367 368
}

369
static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
370
{
371
	unsigned long kernel_start = __pa(_text);
372
	unsigned long kernel_end = __pa(__init_begin);
373

374
	/*
375 376
	 * Take care not to create a writable alias for the
	 * read-only text and rodata sections of the kernel image.
377
	 */
378

379
	/* No overlap with the kernel text/rodata */
380 381 382
	if (end < kernel_start || start >= kernel_end) {
		__create_pgd_mapping(pgd, start, __phys_to_virt(start),
				     end - start, PAGE_KERNEL,
383 384
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
385
		return;
386 387
	}

388
	/*
389
	 * This block overlaps the kernel text/rodata mappings.
390
	 * Map the portion(s) which don't overlap.
391 392 393 394 395
	 */
	if (start < kernel_start)
		__create_pgd_mapping(pgd, start,
				     __phys_to_virt(start),
				     kernel_start - start, PAGE_KERNEL,
396 397
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
398 399 400 401
	if (kernel_end < end)
		__create_pgd_mapping(pgd, kernel_end,
				     __phys_to_virt(kernel_end),
				     end - kernel_end, PAGE_KERNEL,
402 403
				     early_pgtable_alloc,
				     !debug_pagealloc_enabled());
404 405

	/*
406
	 * Map the linear alias of the [_text, __init_begin) interval as
407 408 409 410 411 412
	 * read-only/non-executable. This makes the contents of the
	 * region accessible to subsystems such as hibernate, but
	 * protects it from inadvertent modification or execution.
	 */
	__create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
			     kernel_end - kernel_start, PAGE_KERNEL_RO,
413
			     early_pgtable_alloc, !debug_pagealloc_enabled());
414 415
}

416
static void __init map_mem(pgd_t *pgd)
C
Catalin Marinas 已提交
417 418
{
	struct memblock_region *reg;
419

C
Catalin Marinas 已提交
420 421 422 423 424 425 426
	/* map all the memory banks */
	for_each_memblock(memory, reg) {
		phys_addr_t start = reg->base;
		phys_addr_t end = start + reg->size;

		if (start >= end)
			break;
427 428
		if (memblock_is_nomap(reg))
			continue;
C
Catalin Marinas 已提交
429

430
		__map_memblock(pgd, start, end);
C
Catalin Marinas 已提交
431 432 433
	}
}

434 435
void mark_rodata_ro(void)
{
J
Jeremy Linton 已提交
436
	unsigned long section_size;
437

438
	section_size = (unsigned long)_etext - (unsigned long)_text;
439
	create_mapping_late(__pa(_text), (unsigned long)_text,
J
Jeremy Linton 已提交
440 441
			    section_size, PAGE_KERNEL_ROX);
	/*
442 443
	 * mark .rodata as read only. Use __init_begin rather than __end_rodata
	 * to cover NOTES and EXCEPTION_TABLE.
J
Jeremy Linton 已提交
444
	 */
445
	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
J
Jeremy Linton 已提交
446 447
	create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata,
			    section_size, PAGE_KERNEL_RO);
448 449 450 451
}

void fixup_init(void)
{
452 453 454 455 456 457
	/*
	 * Unmap the __init region but leave the VM area in place. This
	 * prevents the region from being reused for kernel modules, which
	 * is not supported by kallsyms.
	 */
	unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin));
458 459
}

460 461
static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
				      pgprot_t prot, struct vm_struct *vma)
462 463 464 465 466 467 468 469
{
	phys_addr_t pa_start = __pa(va_start);
	unsigned long size = va_end - va_start;

	BUG_ON(!PAGE_ALIGNED(pa_start));
	BUG_ON(!PAGE_ALIGNED(size));

	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
470
			     early_pgtable_alloc, !debug_pagealloc_enabled());
471 472 473 474 475 476 477 478

	vma->addr	= va_start;
	vma->phys_addr	= pa_start;
	vma->size	= size;
	vma->flags	= VM_MAP;
	vma->caller	= __builtin_return_address(0);

	vm_area_add_early(vma);
479 480 481 482 483 484 485
}

/*
 * Create fine-grained mappings for the kernel.
 */
static void __init map_kernel(pgd_t *pgd)
{
J
Jeremy Linton 已提交
486
	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
487

488 489
	map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text);
	map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata);
490 491 492
	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
			   &vmlinux_init);
	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
493

494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
		/*
		 * The fixmap falls in a separate pgd to the kernel, and doesn't
		 * live in the carveout for the swapper_pg_dir. We can simply
		 * re-use the existing dir for the fixmap.
		 */
		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
			*pgd_offset_k(FIXADDR_START));
	} else if (CONFIG_PGTABLE_LEVELS > 3) {
		/*
		 * The fixmap shares its top level pgd entry with the kernel
		 * mapping. This can really only occur when we are running
		 * with 16k/4 levels, so we can simply reuse the pud level
		 * entry instead.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START),
			__pud(__pa(bm_pmd) | PUD_TYPE_TABLE));
		pud_clear_fixmap();
	} else {
		BUG();
	}
516 517 518 519

	kasan_copy_shadow(pgd);
}

C
Catalin Marinas 已提交
520 521 522 523 524 525
/*
 * paging_init() sets up the page tables, initialises the zone memory
 * maps and sets up the zero page.
 */
void __init paging_init(void)
{
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
	phys_addr_t pgd_phys = early_pgtable_alloc();
	pgd_t *pgd = pgd_set_fixmap(pgd_phys);

	map_kernel(pgd);
	map_mem(pgd);

	/*
	 * We want to reuse the original swapper_pg_dir so we don't have to
	 * communicate the new address to non-coherent secondaries in
	 * secondary_entry, and so cpu_switch_mm can generate the address with
	 * adrp+add rather than a load from some global variable.
	 *
	 * To do this we need to go via a temporary pgd.
	 */
	cpu_replace_ttbr1(__va(pgd_phys));
	memcpy(swapper_pg_dir, pgd, PAGE_SIZE);
	cpu_replace_ttbr1(swapper_pg_dir);

	pgd_clear_fixmap();
	memblock_free(pgd_phys, PAGE_SIZE);

	/*
	 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
	 * allocated with it.
	 */
	memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE,
		      SWAPPER_DIR_SIZE - PAGE_SIZE);
C
Catalin Marinas 已提交
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
}

/*
 * Check whether a kernel address is valid (derived from arch/x86/).
 */
int kern_addr_valid(unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	if ((((long)addr) >> VA_BITS) != -1UL)
		return 0;

	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0;

576 577 578
	if (pud_sect(*pud))
		return pfn_valid(pud_pfn(*pud));

C
Catalin Marinas 已提交
579 580 581 582
	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;

583 584 585
	if (pmd_sect(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

C
Catalin Marinas 已提交
586 587 588 589 590 591 592
	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;

	return pfn_valid(pte_pfn(*pte));
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
593
#if !ARM64_SWAPPER_USES_SECTION_MAPS
594
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
595
{
596
	return vmemmap_populate_basepages(start, end, node);
C
Catalin Marinas 已提交
597
}
598
#else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
599
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
C
Catalin Marinas 已提交
600
{
601
	unsigned long addr = start;
C
Catalin Marinas 已提交
602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	do {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(*pmd)) {
			void *p = NULL;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
			if (!p)
				return -ENOMEM;

626
			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
C
Catalin Marinas 已提交
627 628 629 630 631 632 633
		} else
			vmemmap_verify((pte_t *)pmd, node, addr, next);
	} while (addr = next, addr != end);

	return 0;
}
#endif	/* CONFIG_ARM64_64K_PAGES */
634
void vmemmap_free(unsigned long start, unsigned long end)
635 636
{
}
C
Catalin Marinas 已提交
637
#endif	/* CONFIG_SPARSEMEM_VMEMMAP */
638 639 640 641 642 643 644

static inline pud_t * fixmap_pud(unsigned long addr)
{
	pgd_t *pgd = pgd_offset_k(addr);

	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));

645
	return pud_offset_kimg(pgd, addr);
646 647 648 649 650 651 652 653
}

static inline pmd_t * fixmap_pmd(unsigned long addr)
{
	pud_t *pud = fixmap_pud(addr);

	BUG_ON(pud_none(*pud) || pud_bad(*pud));

654
	return pmd_offset_kimg(pud, addr);
655 656 657 658
}

static inline pte_t * fixmap_pte(unsigned long addr)
{
659
	return &bm_pte[pte_index(addr)];
660 661 662 663 664 665 666 667 668 669
}

void __init early_fixmap_init(void)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	unsigned long addr = FIXADDR_START;

	pgd = pgd_offset_k(addr);
670 671
	if (CONFIG_PGTABLE_LEVELS > 3 &&
	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) {
672 673 674 675 676 677 678 679 680 681 682
		/*
		 * We only end up here if the kernel mapping and the fixmap
		 * share the top level pgd entry, which should only happen on
		 * 16k/4 levels configurations.
		 */
		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
		pud = pud_offset_kimg(pgd, addr);
	} else {
		pgd_populate(&init_mm, pgd, bm_pud);
		pud = fixmap_pud(addr);
	}
683
	pud_populate(&init_mm, pud, bm_pmd);
684
	pmd = fixmap_pmd(addr);
685 686 687 688
	pmd_populate_kernel(&init_mm, pmd, bm_pte);

	/*
	 * The boot-ioremap range spans multiple pmds, for which
689
	 * we are not prepared:
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
	 */
	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));

	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		pr_warn("pmd %p != %p, %p\n",
			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
			fix_to_virt(FIX_BTMAP_BEGIN));
		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
			fix_to_virt(FIX_BTMAP_END));

		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
	}
}

void __set_fixmap(enum fixed_addresses idx,
			       phys_addr_t phys, pgprot_t flags)
{
	unsigned long addr = __fix_to_virt(idx);
	pte_t *pte;

716
	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
717 718 719 720 721 722 723 724 725 726

	pte = fixmap_pte(addr);

	if (pgprot_val(flags)) {
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
	} else {
		pte_clear(&init_mm, addr, pte);
		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
	}
}
727

728
void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
729 730
{
	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
731
	int offset;
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	void *dt_virt;

	/*
	 * Check whether the physical FDT address is set and meets the minimum
	 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
	 * at least 8 bytes so that we can always access the size field of the
	 * FDT header after mapping the first chunk, double check here if that
	 * is indeed the case.
	 */
	BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
		return NULL;

	/*
	 * Make sure that the FDT region can be mapped without the need to
	 * allocate additional translation table pages, so that it is safe
748
	 * to call create_mapping_noalloc() this early.
749 750 751 752 753 754 755 756
	 *
	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
	 * be in the same PMD as the rest of the fixmap.
	 * On 4k pages, we'll use section mappings for the FDT so we only
	 * have to be in the same PUD.
	 */
	BUILD_BUG_ON(dt_virt_base % SZ_2M);

757 758
	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
759

760
	offset = dt_phys % SWAPPER_BLOCK_SIZE;
761 762 763
	dt_virt = (void *)dt_virt_base + offset;

	/* map the first chunk so we can read the size from the header */
764 765
	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
766 767 768 769

	if (fdt_check_header(dt_virt) != 0)
		return NULL;

770 771
	*size = fdt_totalsize(dt_virt);
	if (*size > MAX_FDT_SIZE)
772 773
		return NULL;

774
	if (offset + *size > SWAPPER_BLOCK_SIZE)
775
		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
776
			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
777

778 779
	return dt_virt;
}
780

781 782 783 784 785 786 787 788 789 790
void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
{
	void *dt_virt;
	int size;

	dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO);
	if (!dt_virt)
		return NULL;

	memblock_reserve(dt_phys, size);
791 792
	return dt_virt;
}
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833

int __init arch_ioremap_pud_supported(void)
{
	/* only 4k granule supports level 1 block mappings */
	return IS_ENABLED(CONFIG_ARM64_4K_PAGES);
}

int __init arch_ioremap_pmd_supported(void)
{
	return 1;
}

int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PUD_MASK);
	set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
{
	BUG_ON(phys & ~PMD_MASK);
	set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
	return 1;
}

int pud_clear_huge(pud_t *pud)
{
	if (!pud_sect(*pud))
		return 0;
	pud_clear(pud);
	return 1;
}

int pmd_clear_huge(pmd_t *pmd)
{
	if (!pmd_sect(*pmd))
		return 0;
	pmd_clear(pmd);
	return 1;
}