init.c 14.4 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Based on arch/arm/mm/init.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/errno.h>
#include <linux/swap.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/initrd.h>
#include <linux/gfp.h>
#include <linux/memblock.h>
#include <linux/sort.h>
#include <linux/of_fdt.h>
33
#include <linux/dma-mapping.h>
L
Laura Abbott 已提交
34
#include <linux/dma-contiguous.h>
35
#include <linux/efi.h>
36
#include <linux/swiotlb.h>
C
Catalin Marinas 已提交
37

38
#include <asm/boot.h>
39
#include <asm/fixmap.h>
40
#include <asm/kasan.h>
41
#include <asm/kernel-pgtable.h>
42
#include <asm/memory.h>
43
#include <asm/numa.h>
C
Catalin Marinas 已提交
44 45 46 47
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
48
#include <asm/alternative.h>
C
Catalin Marinas 已提交
49 50 51

#include "mm.h"

52 53 54 55 56 57
/*
 * We need to be able to catch inadvertent references to memstart_addr
 * that occur (potentially in generic code) before arm64_memblock_init()
 * executes, which assigns it its actual value. So use a default value
 * that cannot be mistaken for a real physical address.
 */
58
s64 memstart_addr __read_mostly = -1;
59
phys_addr_t arm64_dma_phys_limit __read_mostly;
C
Catalin Marinas 已提交
60

61
#ifdef CONFIG_BLK_DEV_INITRD
C
Catalin Marinas 已提交
62 63 64 65 66 67 68 69 70
static int __init early_initrd(char *p)
{
	unsigned long start, size;
	char *endp;

	start = memparse(p, &endp);
	if (*endp == ',') {
		size = memparse(endp + 1, NULL);

71 72
		initrd_start = start;
		initrd_end = start + size;
C
Catalin Marinas 已提交
73 74 75 76
	}
	return 0;
}
early_param("initrd", early_initrd);
77
#endif
C
Catalin Marinas 已提交
78

79 80 81 82 83
/*
 * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
 * currently assumes that for memory starting above 4G, 32-bit devices will
 * use a DMA offset.
 */
84
static phys_addr_t __init max_zone_dma_phys(void)
85 86 87 88 89
{
	phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32);
	return min(offset + (1ULL << 32), memblock_end_of_DRAM());
}

90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#ifdef CONFIG_NUMA

static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
	unsigned long max_zone_pfns[MAX_NR_ZONES]  = {0};

	if (IS_ENABLED(CONFIG_ZONE_DMA))
		max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
	max_zone_pfns[ZONE_NORMAL] = max;

	free_area_init_nodes(max_zone_pfns);
}

#else

C
Catalin Marinas 已提交
105 106 107 108
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
	struct memblock_region *reg;
	unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
109
	unsigned long max_dma = min;
C
Catalin Marinas 已提交
110 111 112 113

	memset(zone_size, 0, sizeof(zone_size));

	/* 4GB maximum for 32-bit only capable devices */
114 115 116 117
#ifdef CONFIG_ZONE_DMA
	max_dma = PFN_DOWN(arm64_dma_phys_limit);
	zone_size[ZONE_DMA] = max_dma - min;
#endif
118
	zone_size[ZONE_NORMAL] = max - max_dma;
C
Catalin Marinas 已提交
119 120 121 122 123 124 125 126 127

	memcpy(zhole_size, zone_size, sizeof(zhole_size));

	for_each_memblock(memory, reg) {
		unsigned long start = memblock_region_memory_base_pfn(reg);
		unsigned long end = memblock_region_memory_end_pfn(reg);

		if (start >= max)
			continue;
128

129 130
#ifdef CONFIG_ZONE_DMA
		if (start < max_dma) {
131 132
			unsigned long dma_end = min(end, max_dma);
			zhole_size[ZONE_DMA] -= dma_end - start;
C
Catalin Marinas 已提交
133
		}
134
#endif
135
		if (end > max_dma) {
C
Catalin Marinas 已提交
136
			unsigned long normal_end = min(end, max);
137
			unsigned long normal_start = max(start, max_dma);
C
Catalin Marinas 已提交
138 139 140 141 142 143 144
			zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
		}
	}

	free_area_init_node(0, zone_size, min, zhole_size);
}

145 146
#endif /* CONFIG_NUMA */

C
Catalin Marinas 已提交
147 148 149
#ifdef CONFIG_HAVE_ARCH_PFN_VALID
int pfn_valid(unsigned long pfn)
{
150
	return memblock_is_map_memory(pfn << PAGE_SHIFT);
C
Catalin Marinas 已提交
151 152 153 154 155
}
EXPORT_SYMBOL(pfn_valid);
#endif

#ifndef CONFIG_SPARSEMEM
156
static void __init arm64_memory_present(void)
C
Catalin Marinas 已提交
157 158 159
{
}
#else
160
static void __init arm64_memory_present(void)
C
Catalin Marinas 已提交
161 162 163
{
	struct memblock_region *reg;

164
	for_each_memblock(memory, reg) {
165 166
		int nid = memblock_get_region_node(reg);

167 168 169
		memory_present(nid, memblock_region_memory_base_pfn(reg),
				memblock_region_memory_end_pfn(reg));
	}
C
Catalin Marinas 已提交
170 171 172
}
#endif

M
Mark Rutland 已提交
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;

/*
 * Limit the memory size that was specified via FDT.
 */
static int __init early_mem(char *p)
{
	if (!p)
		return 1;

	memory_limit = memparse(p, &p) & PAGE_MASK;
	pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);

	return 0;
}
early_param("mem", early_mem);

C
Catalin Marinas 已提交
190 191
void __init arm64_memblock_init(void)
{
192 193
	const s64 linear_region_size = -(s64)PAGE_OFFSET;

194 195 196 197 198 199 200
	/*
	 * Ensure that the linear region takes up exactly half of the kernel
	 * virtual address space. This way, we can distinguish a linear address
	 * from a kernel/module/vmalloc address by testing a single bit.
	 */
	BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1));

201 202 203 204 205 206 207 208 209 210 211
	/*
	 * Select a suitable value for the base of physical memory.
	 */
	memstart_addr = round_down(memblock_start_of_DRAM(),
				   ARM64_MEMSTART_ALIGN);

	/*
	 * Remove the memory that we will not be able to cover with the
	 * linear mapping. Take care not to clip the kernel which may be
	 * high in memory.
	 */
212
	memblock_remove(max_t(u64, memstart_addr + linear_region_size, __pa(_end)),
213
			ULLONG_MAX);
214 215 216 217 218 219
	if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) {
		/* ensure that memstart_addr remains sufficiently aligned */
		memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size,
					 ARM64_MEMSTART_ALIGN);
		memblock_remove(0, memstart_addr);
	}
220 221 222 223 224 225 226 227 228 229

	/*
	 * Apply the memory limit if it was set. Since the kernel may be loaded
	 * high up in memory, add back the kernel region that must be accessible
	 * via the linear mapping.
	 */
	if (memory_limit != (phys_addr_t)ULLONG_MAX) {
		memblock_enforce_memory_limit(memory_limit);
		memblock_add(__pa(_text), (u64)(_end - _text));
	}
M
Mark Rutland 已提交
230

231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && initrd_start) {
		/*
		 * Add back the memory we just removed if it results in the
		 * initrd to become inaccessible via the linear mapping.
		 * Otherwise, this is a no-op
		 */
		u64 base = initrd_start & PAGE_MASK;
		u64 size = PAGE_ALIGN(initrd_end) - base;

		/*
		 * We can only add back the initrd memory if we don't end up
		 * with more memory than we can address via the linear mapping.
		 * It is up to the bootloader to position the kernel and the
		 * initrd reasonably close to each other (i.e., within 32 GB of
		 * each other) so that all granule/#levels combinations can
		 * always access both.
		 */
		if (WARN(base < memblock_start_of_DRAM() ||
			 base + size > memblock_start_of_DRAM() +
				       linear_region_size,
			"initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) {
			initrd_start = 0;
		} else {
			memblock_remove(base, size); /* clear MEMBLOCK_ flags */
			memblock_add(base, size);
			memblock_reserve(base, size);
		}
	}

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
		extern u16 memstart_offset_seed;
		u64 range = linear_region_size -
			    (memblock_end_of_DRAM() - memblock_start_of_DRAM());

		/*
		 * If the size of the linear region exceeds, by a sufficient
		 * margin, the size of the region that the available physical
		 * memory spans, randomize the linear region as well.
		 */
		if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) {
			range = range / ARM64_MEMSTART_ALIGN + 1;
			memstart_addr -= ARM64_MEMSTART_ALIGN *
					 ((range * memstart_offset_seed) >> 16);
		}
	}
M
Mark Rutland 已提交
276

277 278 279 280
	/*
	 * Register the kernel text, kernel data, initrd, and initial
	 * pagetables with memblock.
	 */
C
Catalin Marinas 已提交
281 282
	memblock_reserve(__pa(_text), _end - _text);
#ifdef CONFIG_BLK_DEV_INITRD
283 284 285 286 287 288 289
	if (initrd_start) {
		memblock_reserve(initrd_start, initrd_end - initrd_start);

		/* the generic initrd code expects virtual addresses */
		initrd_start = __phys_to_virt(initrd_start);
		initrd_end = __phys_to_virt(initrd_end);
	}
C
Catalin Marinas 已提交
290 291
#endif

292
	early_init_fdt_scan_reserved_mem();
293 294 295

	/* 4GB maximum for 32-bit only capable devices */
	if (IS_ENABLED(CONFIG_ZONE_DMA))
296 297 298 299
		arm64_dma_phys_limit = max_zone_dma_phys();
	else
		arm64_dma_phys_limit = PHYS_MASK + 1;
	dma_contiguous_reserve(arm64_dma_phys_limit);
L
Laura Abbott 已提交
300

C
Catalin Marinas 已提交
301 302 303 304 305 306 307 308 309 310
	memblock_allow_resize();
}

void __init bootmem_init(void)
{
	unsigned long min, max;

	min = PFN_UP(memblock_start_of_DRAM());
	max = PFN_DOWN(memblock_end_of_DRAM());

V
Vladimir Murzin 已提交
311 312
	early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);

313 314 315
	max_pfn = max_low_pfn = max;

	arm64_numa_init();
C
Catalin Marinas 已提交
316 317 318 319 320 321 322 323 324 325
	/*
	 * Sparsemem tries to allocate bootmem in memory_present(), so must be
	 * done after the fixed reservations.
	 */
	arm64_memory_present();

	sparse_init();
	zone_sizes_init(min, max);

	high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
326
	memblock_dump_all();
C
Catalin Marinas 已提交
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
}

#ifndef CONFIG_SPARSEMEM_VMEMMAP
static inline void free_memmap(unsigned long start_pfn, unsigned long end_pfn)
{
	struct page *start_pg, *end_pg;
	unsigned long pg, pgend;

	/*
	 * Convert start_pfn/end_pfn to a struct page pointer.
	 */
	start_pg = pfn_to_page(start_pfn - 1) + 1;
	end_pg = pfn_to_page(end_pfn - 1) + 1;

	/*
	 * Convert to physical addresses, and round start upwards and end
	 * downwards.
	 */
	pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
	pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;

	/*
	 * If there are free pages between these, free the section of the
	 * memmap array.
	 */
	if (pg < pgend)
		free_bootmem(pg, pgend - pg);
}

/*
 * The mem_map array can get very big. Free the unused area of the memory map.
 */
static void __init free_unused_memmap(void)
{
	unsigned long start, prev_end = 0;
	struct memblock_region *reg;

	for_each_memblock(memory, reg) {
		start = __phys_to_pfn(reg->base);

#ifdef CONFIG_SPARSEMEM
		/*
		 * Take care not to free memmap entries that don't exist due
		 * to SPARSEMEM sections which aren't present.
		 */
		start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
#endif
		/*
		 * If we had a previous bank, and there is a space between the
		 * current bank and the previous, free it.
		 */
		if (prev_end && prev_end < start)
			free_memmap(prev_end, start);

		/*
		 * Align up here since the VM subsystem insists that the
		 * memmap entries are valid from the bank end aligned to
		 * MAX_ORDER_NR_PAGES.
		 */
386
		prev_end = ALIGN(__phys_to_pfn(reg->base + reg->size),
C
Catalin Marinas 已提交
387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
				 MAX_ORDER_NR_PAGES);
	}

#ifdef CONFIG_SPARSEMEM
	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
		free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
#endif
}
#endif	/* !CONFIG_SPARSEMEM_VMEMMAP */

/*
 * mem_init() marks the free areas in the mem_map and tells us how much memory
 * is free.  This is done after various parts of the system have claimed their
 * memory after the kernel image.
 */
void __init mem_init(void)
{
404 405
	if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT))
		swiotlb_init(1);
406

407
	set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
C
Catalin Marinas 已提交
408 409 410 411

#ifndef CONFIG_SPARSEMEM_VMEMMAP
	free_unused_memmap();
#endif
412
	/* this will put all unused low memory onto the freelists */
413
	free_all_bootmem();
C
Catalin Marinas 已提交
414

415
	mem_init_print_info(NULL);
C
Catalin Marinas 已提交
416 417 418

#define MLK(b, t) b, t, ((t) - (b)) >> 10
#define MLM(b, t) b, t, ((t) - (b)) >> 20
419
#define MLG(b, t) b, t, ((t) - (b)) >> 30
C
Catalin Marinas 已提交
420 421
#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)

422
	pr_notice("Virtual kernel memory layout:\n");
423
#ifdef CONFIG_KASAN
424 425
	pr_cont("    kasan   : 0x%16lx - 0x%16lx   (%6ld GB)\n",
		MLG(KASAN_SHADOW_START, KASAN_SHADOW_END));
426
#endif
427 428 429 430
	pr_cont("    modules : 0x%16lx - 0x%16lx   (%6ld MB)\n",
		MLM(MODULES_VADDR, MODULES_END));
	pr_cont("    vmalloc : 0x%16lx - 0x%16lx   (%6ld GB)\n",
		MLG(VMALLOC_START, VMALLOC_END));
431 432 433 434 435 436 437
	pr_cont("      .text : 0x%p" " - 0x%p" "   (%6ld KB)\n",
		MLK_ROUNDUP(_text, __start_rodata));
	pr_cont("    .rodata : 0x%p" " - 0x%p" "   (%6ld KB)\n",
		MLK_ROUNDUP(__start_rodata, _etext));
	pr_cont("      .init : 0x%p" " - 0x%p" "   (%6ld KB)\n",
		MLK_ROUNDUP(__init_begin, __init_end));
	pr_cont("      .data : 0x%p" " - 0x%p" "   (%6ld KB)\n",
438
		MLK_ROUNDUP(_sdata, _edata));
439 440
	pr_cont("       .bss : 0x%p" " - 0x%p" "   (%6ld KB)\n",
		MLK_ROUNDUP(__bss_start, __bss_stop));
441 442 443 444
	pr_cont("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",
		MLK(FIXADDR_START, FIXADDR_TOP));
	pr_cont("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",
		MLM(PCI_IO_START, PCI_IO_END));
C
Catalin Marinas 已提交
445
#ifdef CONFIG_SPARSEMEM_VMEMMAP
446 447 448
	pr_cont("    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n",
		MLG(VMEMMAP_START, VMEMMAP_START + VMEMMAP_SIZE));
	pr_cont("              0x%16lx - 0x%16lx   (%6ld MB actual)\n",
449 450
		MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),
		    (unsigned long)virt_to_page(high_memory)));
C
Catalin Marinas 已提交
451
#endif
452 453 454
	pr_cont("    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n",
		MLM(__phys_to_virt(memblock_start_of_DRAM()),
		    (unsigned long)high_memory));
C
Catalin Marinas 已提交
455 456 457 458 459 460 461 462 463 464 465 466 467

#undef MLK
#undef MLM
#undef MLK_ROUNDUP

	/*
	 * Check boundaries twice: Some fundamental inconsistencies can be
	 * detected at build time already.
	 */
#ifdef CONFIG_COMPAT
	BUILD_BUG_ON(TASK_SIZE_32			> TASK_SIZE_64);
#endif

468 469 470 471 472 473
	/*
	 * Make sure we chose the upper bound of sizeof(struct page)
	 * correctly.
	 */
	BUILD_BUG_ON(sizeof(struct page) > (1 << STRUCT_PAGE_MAX_SHIFT));

474
	if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
C
Catalin Marinas 已提交
475 476 477 478 479 480 481 482 483 484 485
		extern int sysctl_overcommit_memory;
		/*
		 * On a machine this small we won't get anywhere without
		 * overcommit, so turn it on by default.
		 */
		sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
	}
}

void free_initmem(void)
{
486 487
	free_reserved_area(__va(__pa(__init_begin)), __va(__pa(__init_end)),
			   0, "unused kernel");
488
	fixup_init();
C
Catalin Marinas 已提交
489 490 491 492
}

#ifdef CONFIG_BLK_DEV_INITRD

493
static int keep_initrd __initdata;
C
Catalin Marinas 已提交
494

495
void __init free_initrd_mem(unsigned long start, unsigned long end)
C
Catalin Marinas 已提交
496
{
497
	if (!keep_initrd)
J
Jiang Liu 已提交
498
		free_reserved_area((void *)start, (void *)end, 0, "initrd");
C
Catalin Marinas 已提交
499 500 501 502 503 504 505 506 507 508
}

static int __init keepinitrd_setup(char *__unused)
{
	keep_initrd = 1;
	return 1;
}

__setup("keepinitrd", keepinitrd_setup);
#endif
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533

/*
 * Dump out memory limit information on panic.
 */
static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p)
{
	if (memory_limit != (phys_addr_t)ULLONG_MAX) {
		pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
	} else {
		pr_emerg("Memory Limit: none\n");
	}
	return 0;
}

static struct notifier_block mem_limit_notifier = {
	.notifier_call = dump_mem_limit,
};

static int __init register_mem_limit_dumper(void)
{
	atomic_notifier_chain_register(&panic_notifier_list,
				       &mem_limit_notifier);
	return 0;
}
__initcall(register_mem_limit_dumper);