init.c 9.7 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Based on arch/arm/mm/init.c
 *
 * Copyright (C) 1995-2005 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/errno.h>
#include <linux/swap.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/initrd.h>
#include <linux/gfp.h>
#include <linux/memblock.h>
#include <linux/sort.h>
#include <linux/of_fdt.h>
33
#include <linux/dma-mapping.h>
L
Laura Abbott 已提交
34
#include <linux/dma-contiguous.h>
35
#include <linux/efi.h>
36
#include <linux/swiotlb.h>
C
Catalin Marinas 已提交
37

38
#include <asm/fixmap.h>
39
#include <asm/memory.h>
C
Catalin Marinas 已提交
40 41 42 43
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
44
#include <asm/alternative.h>
C
Catalin Marinas 已提交
45 46 47 48

#include "mm.h"

phys_addr_t memstart_addr __read_mostly = 0;
49
phys_addr_t arm64_dma_phys_limit __read_mostly;
C
Catalin Marinas 已提交
50

51
#ifdef CONFIG_BLK_DEV_INITRD
C
Catalin Marinas 已提交
52 53 54 55 56 57 58 59 60
static int __init early_initrd(char *p)
{
	unsigned long start, size;
	char *endp;

	start = memparse(p, &endp);
	if (*endp == ',') {
		size = memparse(endp + 1, NULL);

61 62
		initrd_start = (unsigned long)__va(start);
		initrd_end = (unsigned long)__va(start + size);
C
Catalin Marinas 已提交
63 64 65 66
	}
	return 0;
}
early_param("initrd", early_initrd);
67
#endif
C
Catalin Marinas 已提交
68

69 70 71 72 73
/*
 * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
 * currently assumes that for memory starting above 4G, 32-bit devices will
 * use a DMA offset.
 */
74
static phys_addr_t __init max_zone_dma_phys(void)
75 76 77 78 79
{
	phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32);
	return min(offset + (1ULL << 32), memblock_end_of_DRAM());
}

C
Catalin Marinas 已提交
80 81 82 83
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
	struct memblock_region *reg;
	unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
84
	unsigned long max_dma = min;
C
Catalin Marinas 已提交
85 86 87 88

	memset(zone_size, 0, sizeof(zone_size));

	/* 4GB maximum for 32-bit only capable devices */
89 90 91 92
#ifdef CONFIG_ZONE_DMA
	max_dma = PFN_DOWN(arm64_dma_phys_limit);
	zone_size[ZONE_DMA] = max_dma - min;
#endif
93
	zone_size[ZONE_NORMAL] = max - max_dma;
C
Catalin Marinas 已提交
94 95 96 97 98 99 100 101 102

	memcpy(zhole_size, zone_size, sizeof(zhole_size));

	for_each_memblock(memory, reg) {
		unsigned long start = memblock_region_memory_base_pfn(reg);
		unsigned long end = memblock_region_memory_end_pfn(reg);

		if (start >= max)
			continue;
103

104 105
#ifdef CONFIG_ZONE_DMA
		if (start < max_dma) {
106 107
			unsigned long dma_end = min(end, max_dma);
			zhole_size[ZONE_DMA] -= dma_end - start;
C
Catalin Marinas 已提交
108
		}
109
#endif
110
		if (end > max_dma) {
C
Catalin Marinas 已提交
111
			unsigned long normal_end = min(end, max);
112
			unsigned long normal_start = max(start, max_dma);
C
Catalin Marinas 已提交
113 114 115 116 117 118 119 120 121 122
			zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
		}
	}

	free_area_init_node(0, zone_size, min, zhole_size);
}

#ifdef CONFIG_HAVE_ARCH_PFN_VALID
int pfn_valid(unsigned long pfn)
{
123
	return memblock_is_map_memory(pfn << PAGE_SHIFT);
C
Catalin Marinas 已提交
124 125 126 127 128
}
EXPORT_SYMBOL(pfn_valid);
#endif

#ifndef CONFIG_SPARSEMEM
129
static void __init arm64_memory_present(void)
C
Catalin Marinas 已提交
130 131 132
{
}
#else
133
static void __init arm64_memory_present(void)
C
Catalin Marinas 已提交
134 135 136 137 138 139 140 141 142
{
	struct memblock_region *reg;

	for_each_memblock(memory, reg)
		memory_present(0, memblock_region_memory_base_pfn(reg),
			       memblock_region_memory_end_pfn(reg));
}
#endif

M
Mark Rutland 已提交
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;

/*
 * Limit the memory size that was specified via FDT.
 */
static int __init early_mem(char *p)
{
	if (!p)
		return 1;

	memory_limit = memparse(p, &p) & PAGE_MASK;
	pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);

	return 0;
}
early_param("mem", early_mem);

C
Catalin Marinas 已提交
160 161
void __init arm64_memblock_init(void)
{
M
Mark Rutland 已提交
162 163
	memblock_enforce_memory_limit(memory_limit);

164 165 166 167
	/*
	 * Register the kernel text, kernel data, initrd, and initial
	 * pagetables with memblock.
	 */
C
Catalin Marinas 已提交
168 169
	memblock_reserve(__pa(_text), _end - _text);
#ifdef CONFIG_BLK_DEV_INITRD
170 171
	if (initrd_start)
		memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start);
C
Catalin Marinas 已提交
172 173
#endif

174
	early_init_fdt_scan_reserved_mem();
175 176 177

	/* 4GB maximum for 32-bit only capable devices */
	if (IS_ENABLED(CONFIG_ZONE_DMA))
178 179 180 181
		arm64_dma_phys_limit = max_zone_dma_phys();
	else
		arm64_dma_phys_limit = PHYS_MASK + 1;
	dma_contiguous_reserve(arm64_dma_phys_limit);
L
Laura Abbott 已提交
182

C
Catalin Marinas 已提交
183 184 185 186 187 188 189 190 191 192 193
	memblock_allow_resize();
	memblock_dump_all();
}

void __init bootmem_init(void)
{
	unsigned long min, max;

	min = PFN_UP(memblock_start_of_DRAM());
	max = PFN_DOWN(memblock_end_of_DRAM());

V
Vladimir Murzin 已提交
194 195
	early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);

C
Catalin Marinas 已提交
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	/*
	 * Sparsemem tries to allocate bootmem in memory_present(), so must be
	 * done after the fixed reservations.
	 */
	arm64_memory_present();

	sparse_init();
	zone_sizes_init(min, max);

	high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
	max_pfn = max_low_pfn = max;
}

#ifndef CONFIG_SPARSEMEM_VMEMMAP
static inline void free_memmap(unsigned long start_pfn, unsigned long end_pfn)
{
	struct page *start_pg, *end_pg;
	unsigned long pg, pgend;

	/*
	 * Convert start_pfn/end_pfn to a struct page pointer.
	 */
	start_pg = pfn_to_page(start_pfn - 1) + 1;
	end_pg = pfn_to_page(end_pfn - 1) + 1;

	/*
	 * Convert to physical addresses, and round start upwards and end
	 * downwards.
	 */
	pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
	pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;

	/*
	 * If there are free pages between these, free the section of the
	 * memmap array.
	 */
	if (pg < pgend)
		free_bootmem(pg, pgend - pg);
}

/*
 * The mem_map array can get very big. Free the unused area of the memory map.
 */
static void __init free_unused_memmap(void)
{
	unsigned long start, prev_end = 0;
	struct memblock_region *reg;

	for_each_memblock(memory, reg) {
		start = __phys_to_pfn(reg->base);

#ifdef CONFIG_SPARSEMEM
		/*
		 * Take care not to free memmap entries that don't exist due
		 * to SPARSEMEM sections which aren't present.
		 */
		start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
#endif
		/*
		 * If we had a previous bank, and there is a space between the
		 * current bank and the previous, free it.
		 */
		if (prev_end && prev_end < start)
			free_memmap(prev_end, start);

		/*
		 * Align up here since the VM subsystem insists that the
		 * memmap entries are valid from the bank end aligned to
		 * MAX_ORDER_NR_PAGES.
		 */
266
		prev_end = ALIGN(__phys_to_pfn(reg->base + reg->size),
C
Catalin Marinas 已提交
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
				 MAX_ORDER_NR_PAGES);
	}

#ifdef CONFIG_SPARSEMEM
	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
		free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
#endif
}
#endif	/* !CONFIG_SPARSEMEM_VMEMMAP */

/*
 * mem_init() marks the free areas in the mem_map and tells us how much memory
 * is free.  This is done after various parts of the system have claimed their
 * memory after the kernel image.
 */
void __init mem_init(void)
{
284 285
	swiotlb_init(1);

286
	set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
C
Catalin Marinas 已提交
287 288 289 290

#ifndef CONFIG_SPARSEMEM_VMEMMAP
	free_unused_memmap();
#endif
291
	/* this will put all unused low memory onto the freelists */
292
	free_all_bootmem();
C
Catalin Marinas 已提交
293

294
	mem_init_print_info(NULL);
C
Catalin Marinas 已提交
295 296 297

#define MLK(b, t) b, t, ((t) - (b)) >> 10
#define MLM(b, t) b, t, ((t) - (b)) >> 20
298
#define MLG(b, t) b, t, ((t) - (b)) >> 30
C
Catalin Marinas 已提交
299 300 301
#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)

	pr_notice("Virtual kernel memory layout:\n"
302 303 304
#ifdef CONFIG_KASAN
		  "    kasan   : 0x%16lx - 0x%16lx   (%6ld GB)\n"
#endif
305
		  "    vmalloc : 0x%16lx - 0x%16lx   (%6ld GB)\n"
C
Catalin Marinas 已提交
306
#ifdef CONFIG_SPARSEMEM_VMEMMAP
307 308
		  "    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n"
		  "              0x%16lx - 0x%16lx   (%6ld MB actual)\n"
C
Catalin Marinas 已提交
309
#endif
310
		  "    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n"
311
		  "    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n"
C
Catalin Marinas 已提交
312 313
		  "    modules : 0x%16lx - 0x%16lx   (%6ld MB)\n"
		  "    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n"
314 315 316
		  "      .init : 0x%p" " - 0x%p" "   (%6ld KB)\n"
		  "      .text : 0x%p" " - 0x%p" "   (%6ld KB)\n"
		  "      .data : 0x%p" " - 0x%p" "   (%6ld KB)\n",
317 318 319
#ifdef CONFIG_KASAN
		  MLG(KASAN_SHADOW_START, KASAN_SHADOW_END),
#endif
320
		  MLG(VMALLOC_START, VMALLOC_END),
C
Catalin Marinas 已提交
321
#ifdef CONFIG_SPARSEMEM_VMEMMAP
322 323
		  MLG((unsigned long)vmemmap,
		      (unsigned long)vmemmap + VMEMMAP_SIZE),
C
Catalin Marinas 已提交
324 325 326
		  MLM((unsigned long)virt_to_page(PAGE_OFFSET),
		      (unsigned long)virt_to_page(high_memory)),
#endif
327
		  MLK(FIXADDR_START, FIXADDR_TOP),
328
		  MLM(PCI_IO_START, PCI_IO_END),
C
Catalin Marinas 已提交
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
		  MLM(MODULES_VADDR, MODULES_END),
		  MLM(PAGE_OFFSET, (unsigned long)high_memory),
		  MLK_ROUNDUP(__init_begin, __init_end),
		  MLK_ROUNDUP(_text, _etext),
		  MLK_ROUNDUP(_sdata, _edata));

#undef MLK
#undef MLM
#undef MLK_ROUNDUP

	/*
	 * Check boundaries twice: Some fundamental inconsistencies can be
	 * detected at build time already.
	 */
#ifdef CONFIG_COMPAT
	BUILD_BUG_ON(TASK_SIZE_32			> TASK_SIZE_64);
#endif
	BUILD_BUG_ON(TASK_SIZE_64			> MODULES_VADDR);
	BUG_ON(TASK_SIZE_64				> MODULES_VADDR);

349
	if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
C
Catalin Marinas 已提交
350 351 352 353 354 355 356 357 358 359 360
		extern int sysctl_overcommit_memory;
		/*
		 * On a machine this small we won't get anywhere without
		 * overcommit, so turn it on by default.
		 */
		sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
	}
}

void free_initmem(void)
{
361
	fixup_init();
J
Jiang Liu 已提交
362
	free_initmem_default(0);
C
Catalin Marinas 已提交
363 364 365 366
}

#ifdef CONFIG_BLK_DEV_INITRD

367
static int keep_initrd __initdata;
C
Catalin Marinas 已提交
368

369
void __init free_initrd_mem(unsigned long start, unsigned long end)
C
Catalin Marinas 已提交
370
{
371
	if (!keep_initrd)
J
Jiang Liu 已提交
372
		free_reserved_area((void *)start, (void *)end, 0, "initrd");
C
Catalin Marinas 已提交
373 374 375 376 377 378 379 380 381 382
}

static int __init keepinitrd_setup(char *__unused)
{
	keep_initrd = 1;
	return 1;
}

__setup("keepinitrd", keepinitrd_setup);
#endif