init_32.c 24.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 *
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 */

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
23
#include <linux/pci.h>
24
#include <linux/pfn.h>
25
#include <linux/poison.h>
L
Linus Torvalds 已提交
26
#include <linux/bootmem.h>
27
#include <linux/memblock.h>
L
Linus Torvalds 已提交
28
#include <linux/proc_fs.h>
29
#include <linux/memory_hotplug.h>
30
#include <linux/initrd.h>
31
#include <linux/cpumask.h>
32
#include <linux/gfp.h>
L
Linus Torvalds 已提交
33

34
#include <asm/asm.h>
35
#include <asm/bios_ebda.h>
L
Linus Torvalds 已提交
36
#include <asm/processor.h>
37
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
38 39 40
#include <asm/pgtable.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
41
#include <asm/e820/api.h>
L
Linus Torvalds 已提交
42
#include <asm/apic.h>
I
Ingo Molnar 已提交
43
#include <asm/bugs.h>
L
Linus Torvalds 已提交
44 45
#include <asm/tlb.h>
#include <asm/tlbflush.h>
46
#include <asm/olpc_ofw.h>
47
#include <asm/pgalloc.h>
L
Linus Torvalds 已提交
48
#include <asm/sections.h>
49
#include <asm/paravirt.h>
50
#include <asm/setup.h>
L
Laura Abbott 已提交
51
#include <asm/set_memory.h>
52
#include <asm/page_types.h>
53
#include <asm/init.h>
L
Linus Torvalds 已提交
54

55 56
#include "mm_internal.h"

L
Linus Torvalds 已提交
57 58
unsigned long highstart_pfn, highend_pfn;

59
bool __read_mostly __vmalloc_start_set = false;
60

L
Linus Torvalds 已提交
61 62 63 64 65 66 67
/*
 * Creates a middle page table and puts a pointer to it in the
 * given global directory entry. This only returns the gd entry
 * in non-PAE compilation mode, since the middle layer is folded.
 */
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
68
	p4d_t *p4d;
L
Linus Torvalds 已提交
69 70
	pud_t *pud;
	pmd_t *pmd_table;
I
Ingo Molnar 已提交
71

L
Linus Torvalds 已提交
72
#ifdef CONFIG_X86_PAE
73
	if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
74
		pmd_table = (pmd_t *)alloc_low_page();
75
		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
76
		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
77 78
		p4d = p4d_offset(pgd, 0);
		pud = pud_offset(p4d, 0);
I
Ingo Molnar 已提交
79
		BUG_ON(pmd_table != pmd_offset(pud, 0));
80 81

		return pmd_table;
82 83
	}
#endif
84 85
	p4d = p4d_offset(pgd, 0);
	pud = pud_offset(p4d, 0);
L
Linus Torvalds 已提交
86
	pmd_table = pmd_offset(pud, 0);
I
Ingo Molnar 已提交
87

L
Linus Torvalds 已提交
88 89 90 91 92
	return pmd_table;
}

/*
 * Create a page table and place a pointer to it in a middle page
I
Ingo Molnar 已提交
93
 * directory entry:
L
Linus Torvalds 已提交
94 95 96
 */
static pte_t * __init one_page_table_init(pmd_t *pmd)
{
97
	if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
98
		pte_t *page_table = (pte_t *)alloc_low_page();
99

100
		paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
L
Linus Torvalds 已提交
101
		set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
102
		BUG_ON(page_table != pte_offset_kernel(pmd, 0));
L
Linus Torvalds 已提交
103
	}
104

L
Linus Torvalds 已提交
105 106 107
	return pte_offset_kernel(pmd, 0);
}

108
pmd_t * __init populate_extra_pmd(unsigned long vaddr)
109 110 111
{
	int pgd_idx = pgd_index(vaddr);
	int pmd_idx = pmd_index(vaddr);
112 113 114 115 116 117 118

	return one_md_table_init(swapper_pg_dir + pgd_idx) + pmd_idx;
}

pte_t * __init populate_extra_pte(unsigned long vaddr)
{
	int pte_idx = pte_index(vaddr);
119 120
	pmd_t *pmd;

121 122
	pmd = populate_extra_pmd(vaddr);
	return one_page_table_init(pmd) + pte_idx;
123 124
}

125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
static unsigned long __init
page_table_range_init_count(unsigned long start, unsigned long end)
{
	unsigned long count = 0;
#ifdef CONFIG_HIGHMEM
	int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
	int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
	int pgd_idx, pmd_idx;
	unsigned long vaddr;

	if (pmd_idx_kmap_begin == pmd_idx_kmap_end)
		return 0;

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
140
	pmd_idx = pmd_index(vaddr);
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd_idx++) {
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd_idx++) {
			if ((vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin &&
			    (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end)
				count++;
			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
#endif
	return count;
}

156
static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
157 158
					   unsigned long vaddr, pte_t *lastpte,
					   void **adr)
159 160 161 162 163 164 165 166 167 168 169 170 171
{
#ifdef CONFIG_HIGHMEM
	/*
	 * Something (early fixmap) may already have put a pte
	 * page here, which causes the page table allocation
	 * to become nonlinear. Attempt to fix it, and if it
	 * is still nonlinear then we have to bug.
	 */
	int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
	int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;

	if (pmd_idx_kmap_begin != pmd_idx_kmap_end
	    && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
172
	    && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end) {
173 174 175
		pte_t *newpte;
		int i;

176
		BUG_ON(after_bootmem);
177
		newpte = *adr;
178 179
		for (i = 0; i < PTRS_PER_PTE; i++)
			set_pte(newpte + i, pte[i]);
180
		*adr = (void *)(((unsigned long)(*adr)) + PAGE_SIZE);
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196

		paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
		set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
		BUG_ON(newpte != pte_offset_kernel(pmd, 0));
		__flush_tlb_all();

		paravirt_release_pte(__pa(pte) >> PAGE_SHIFT);
		pte = newpte;
	}
	BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1)
	       && vaddr > fix_to_virt(FIX_KMAP_END)
	       && lastpte && lastpte + PTRS_PER_PTE != pte);
#endif
	return pte;
}

L
Linus Torvalds 已提交
197
/*
I
Ingo Molnar 已提交
198
 * This function initializes a certain range of kernel virtual memory
L
Linus Torvalds 已提交
199 200
 * with new bootmem page tables, everywhere page tables are missing in
 * the given range.
I
Ingo Molnar 已提交
201 202 203
 *
 * NOTE: The pagetables are allocated contiguous on the physical space
 * so we can cache the place of the first one and move around without
L
Linus Torvalds 已提交
204 205
 * checking the pgd every time.
 */
I
Ingo Molnar 已提交
206 207
static void __init
page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
L
Linus Torvalds 已提交
208 209 210
{
	int pgd_idx, pmd_idx;
	unsigned long vaddr;
I
Ingo Molnar 已提交
211 212
	pgd_t *pgd;
	pmd_t *pmd;
213
	pte_t *pte = NULL;
214 215 216 217 218
	unsigned long count = page_table_range_init_count(start, end);
	void *adr = NULL;

	if (count)
		adr = alloc_low_pages(count);
L
Linus Torvalds 已提交
219 220 221 222 223 224 225

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
	pmd_idx = pmd_index(vaddr);
	pgd = pgd_base + pgd_idx;

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
226 227
		pmd = one_md_table_init(pgd);
		pmd = pmd + pmd_index(vaddr);
I
Ingo Molnar 已提交
228 229
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd++, pmd_idx++) {
230
			pte = page_table_kmap_check(one_page_table_init(pmd),
231
						    pmd, vaddr, pte, &adr);
L
Linus Torvalds 已提交
232 233 234 235 236 237 238 239 240

			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
}

static inline int is_kernel_text(unsigned long addr)
{
241
	if (addr >= (unsigned long)_text && addr <= (unsigned long)__init_end)
L
Linus Torvalds 已提交
242 243 244 245 246
		return 1;
	return 0;
}

/*
I
Ingo Molnar 已提交
247 248 249
 * This maps the physical memory to kernel virtual address space, a total
 * of max_low_pfn pages, by creating page tables starting from address
 * PAGE_OFFSET:
L
Linus Torvalds 已提交
250
 */
251 252 253 254
unsigned long __init
kernel_physical_mapping_init(unsigned long start,
			     unsigned long end,
			     unsigned long page_size_mask)
L
Linus Torvalds 已提交
255
{
256
	int use_pse = page_size_mask == (1<<PG_LEVEL_2M);
257
	unsigned long last_map_addr = end;
258
	unsigned long start_pfn, end_pfn;
259
	pgd_t *pgd_base = swapper_pg_dir;
I
Ingo Molnar 已提交
260
	int pgd_idx, pmd_idx, pte_ofs;
L
Linus Torvalds 已提交
261 262 263 264
	unsigned long pfn;
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
265 266 267
	unsigned pages_2m, pages_4k;
	int mapping_iter;

268 269 270
	start_pfn = start >> PAGE_SHIFT;
	end_pfn = end >> PAGE_SHIFT;

271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
	/*
	 * First iteration will setup identity mapping using large/small pages
	 * based on use_pse, with other attributes same as set by
	 * the early code in head_32.S
	 *
	 * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
	 * as desired for the kernel identity mapping.
	 *
	 * This two pass mechanism conforms to the TLB app note which says:
	 *
	 *     "Software should not write to a paging-structure entry in a way
	 *      that would change, for any linear address, both the page size
	 *      and either the page frame or attributes."
	 */
	mapping_iter = 1;
L
Linus Torvalds 已提交
286

287
	if (!boot_cpu_has(X86_FEATURE_PSE))
288
		use_pse = 0;
L
Linus Torvalds 已提交
289

290 291
repeat:
	pages_2m = pages_4k = 0;
292 293 294
	pfn = start_pfn;
	pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
	pgd = pgd_base + pgd_idx;
L
Linus Torvalds 已提交
295 296
	for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
		pmd = one_md_table_init(pgd);
I
Ingo Molnar 已提交
297

298 299 300 301 302 303 304 305 306
		if (pfn >= end_pfn)
			continue;
#ifdef CONFIG_X86_PAE
		pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
		pmd += pmd_idx;
#else
		pmd_idx = 0;
#endif
		for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
J
Jeremy Fitzhardinge 已提交
307
		     pmd++, pmd_idx++) {
I
Ingo Molnar 已提交
308
			unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
L
Linus Torvalds 已提交
309

I
Ingo Molnar 已提交
310 311 312 313
			/*
			 * Map with big pages if possible, otherwise
			 * create normal page tables:
			 */
314
			if (use_pse) {
I
Ingo Molnar 已提交
315
				unsigned int addr2;
J
Jeremy Fitzhardinge 已提交
316
				pgprot_t prot = PAGE_KERNEL_LARGE;
317 318 319 320 321 322 323
				/*
				 * first pass will use the same initial
				 * identity mapping attribute + _PAGE_PSE.
				 */
				pgprot_t init_prot =
					__pgprot(PTE_IDENT_ATTR |
						 _PAGE_PSE);
J
Jeremy Fitzhardinge 已提交
324

325
				pfn &= PMD_MASK >> PAGE_SHIFT;
I
Ingo Molnar 已提交
326
				addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
J
Jeremy Fitzhardinge 已提交
327 328
					PAGE_OFFSET + PAGE_SIZE-1;

I
Ingo Molnar 已提交
329 330
				if (is_kernel_text(addr) ||
				    is_kernel_text(addr2))
J
Jeremy Fitzhardinge 已提交
331 332
					prot = PAGE_KERNEL_LARGE_EXEC;

333
				pages_2m++;
334 335 336 337
				if (mapping_iter == 1)
					set_pmd(pmd, pfn_pmd(pfn, init_prot));
				else
					set_pmd(pmd, pfn_pmd(pfn, prot));
338

L
Linus Torvalds 已提交
339
				pfn += PTRS_PER_PTE;
I
Ingo Molnar 已提交
340 341 342
				continue;
			}
			pte = one_page_table_init(pmd);
L
Linus Torvalds 已提交
343

344 345 346
			pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
			pte += pte_ofs;
			for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
I
Ingo Molnar 已提交
347 348
			     pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
				pgprot_t prot = PAGE_KERNEL;
349 350 351 352 353
				/*
				 * first pass will use the same initial
				 * identity mapping attribute.
				 */
				pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
J
Jeremy Fitzhardinge 已提交
354

I
Ingo Molnar 已提交
355 356
				if (is_kernel_text(addr))
					prot = PAGE_KERNEL_EXEC;
J
Jeremy Fitzhardinge 已提交
357

358
				pages_4k++;
359
				if (mapping_iter == 1) {
360
					set_pte(pte, pfn_pte(pfn, init_prot));
361 362
					last_map_addr = (pfn << PAGE_SHIFT) + PAGE_SIZE;
				} else
363
					set_pte(pte, pfn_pte(pfn, prot));
L
Linus Torvalds 已提交
364 365 366
			}
		}
	}
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
	if (mapping_iter == 1) {
		/*
		 * update direct mapping page count only in the first
		 * iteration.
		 */
		update_page_count(PG_LEVEL_2M, pages_2m);
		update_page_count(PG_LEVEL_4K, pages_4k);

		/*
		 * local global flush tlb, which will flush the previous
		 * mappings present in both small and large page TLB's.
		 */
		__flush_tlb_all();

		/*
		 * Second iteration will set the actual desired PTE attributes.
		 */
		mapping_iter = 2;
		goto repeat;
	}
387
	return last_map_addr;
388 389
}

L
Linus Torvalds 已提交
390 391
pte_t *kmap_pte;

I
Ingo Molnar 已提交
392 393
static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
{
394 395 396 397 398
	pgd_t *pgd = pgd_offset_k(vaddr);
	p4d_t *p4d = p4d_offset(pgd, vaddr);
	pud_t *pud = pud_offset(p4d, vaddr);
	pmd_t *pmd = pmd_offset(pud, vaddr);
	return pte_offset_kernel(pmd, vaddr);
I
Ingo Molnar 已提交
399
}
L
Linus Torvalds 已提交
400 401 402 403 404

static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

I
Ingo Molnar 已提交
405 406 407
	/*
	 * Cache the first kmap pte:
	 */
L
Linus Torvalds 已提交
408 409 410 411
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
}

412
#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
413 414
static void __init permanent_kmaps_init(pgd_t *pgd_base)
{
I
Ingo Molnar 已提交
415
	unsigned long vaddr;
L
Linus Torvalds 已提交
416
	pgd_t *pgd;
417
	p4d_t *p4d;
L
Linus Torvalds 已提交
418 419 420 421 422 423 424 425
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	vaddr = PKMAP_BASE;
	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);

	pgd = swapper_pg_dir + pgd_index(vaddr);
426 427
	p4d = p4d_offset(pgd, vaddr);
	pud = pud_offset(p4d, vaddr);
L
Linus Torvalds 已提交
428 429
	pmd = pmd_offset(pud, vaddr);
	pte = pte_offset_kernel(pmd, vaddr);
I
Ingo Molnar 已提交
430
	pkmap_page_table = pte;
L
Linus Torvalds 已提交
431 432
}

433 434
void __init add_highpages_with_active_regions(int nid,
			 unsigned long start_pfn, unsigned long end_pfn)
L
Linus Torvalds 已提交
435
{
436 437 438
	phys_addr_t start, end;
	u64 i;

439
	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &start, &end, NULL) {
440 441 442 443 444 445
		unsigned long pfn = clamp_t(unsigned long, PFN_UP(start),
					    start_pfn, end_pfn);
		unsigned long e_pfn = clamp_t(unsigned long, PFN_DOWN(end),
					      start_pfn, end_pfn);
		for ( ; pfn < e_pfn; pfn++)
			if (pfn_valid(pfn))
446
				free_highmem_page(pfn_to_page(pfn));
447
	}
448
}
L
Linus Torvalds 已提交
449
#else
450 451 452
static inline void permanent_kmaps_init(pgd_t *pgd_base)
{
}
L
Linus Torvalds 已提交
453 454
#endif /* CONFIG_HIGHMEM */

455
void __init native_pagetable_init(void)
L
Linus Torvalds 已提交
456
{
457
	unsigned long pfn, va;
458
	pgd_t *pgd, *base = swapper_pg_dir;
459
	p4d_t *p4d;
460 461 462
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
463 464

	/*
465
	 * Remove any mappings which extend past the end of physical
466 467 468 469 470 471
	 * memory from the boot time page table.
	 * In virtual address space, we should have at least two pages
	 * from VMALLOC_END to pkmap or fixmap according to VMALLOC_END
	 * definition. And max_low_pfn is set to VMALLOC_END physical
	 * address. If initial memory mapping is doing right job, we
	 * should have pte used near max_low_pfn or one pmd is not present.
472
	 */
473
	for (pfn = max_low_pfn; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
474 475 476 477 478
		va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
		pgd = base + pgd_index(va);
		if (!pgd_present(*pgd))
			break;

479 480
		p4d = p4d_offset(pgd, va);
		pud = pud_offset(p4d, va);
481 482 483 484
		pmd = pmd_offset(pud, va);
		if (!pmd_present(*pmd))
			break;

485 486 487 488 489 490 491
		/* should not be large page here */
		if (pmd_large(*pmd)) {
			pr_warn("try to clear pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx, but pmd is big page and is not using pte !\n",
				pfn, pmd, __pa(pmd));
			BUG_ON(1);
		}

492 493 494 495
		pte = pte_offset_kernel(pmd, va);
		if (!pte_present(*pte))
			break;

496 497
		printk(KERN_DEBUG "clearing pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx pte: %p pte phys: %lx\n",
				pfn, pmd, __pa(pmd), pte, __pa(pte));
498 499
		pte_clear(NULL, va, pte);
	}
500
	paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
501
	paging_init();
502 503 504 505 506 507 508 509
}

/*
 * Build a proper pagetable for the kernel mappings.  Up until this
 * point, we've been running on some set of pagetables constructed by
 * the boot process.
 *
 * If we're booting on native hardware, this will be a pagetable
510 511
 * constructed in arch/x86/kernel/head_32.S.  The root of the
 * pagetable will be swapper_pg_dir.
512 513 514 515
 *
 * If we're booting paravirtualized under a hypervisor, then there are
 * more options: we may already be running PAE, and the pagetable may
 * or may not be based in swapper_pg_dir.  In any case,
516
 * paravirt_pagetable_init() will set up swapper_pg_dir
517 518 519 520 521 522
 * appropriately for the rest of the initialization to work.
 *
 * In general, pagetable_init() assumes that the pagetable may already
 * be partially populated, and so it avoids stomping on any existing
 * mappings.
 */
523
void __init early_ioremap_page_table_range_init(void)
524
{
525
	pgd_t *pgd_base = swapper_pg_dir;
I
Ingo Molnar 已提交
526
	unsigned long vaddr, end;
527

L
Linus Torvalds 已提交
528 529 530 531 532
	/*
	 * Fixed mappings, only the page table structure has to be
	 * created - mappings will be set by set_fixmap():
	 */
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
533 534
	end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
	page_table_range_init(vaddr, end, pgd_base);
535
	early_ioremap_reset();
536 537 538 539 540 541
}

static void __init pagetable_init(void)
{
	pgd_t *pgd_base = swapper_pg_dir;

L
Linus Torvalds 已提交
542 543 544
	permanent_kmaps_init(pgd_base);
}

545
pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL);
546 547
EXPORT_SYMBOL_GPL(__supported_pte_mask);

548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
/* user-defined highmem size */
static unsigned int highmem_pages = -1;

/*
 * highmem=size forces highmem to be exactly 'size' bytes.
 * This works even on boxes that have no highmem otherwise.
 * This also works to reduce highmem size on bigger boxes.
 */
static int __init parse_highmem(char *arg)
{
	if (!arg)
		return -EINVAL;

	highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
	return 0;
}
early_param("highmem", parse_highmem);

566 567 568 569 570
#define MSG_HIGHMEM_TOO_BIG \
	"highmem size (%luMB) is bigger than pages available (%luMB)!\n"

#define MSG_LOWMEM_TOO_SMALL \
	"highmem size (%luMB) results in <64MB lowmem, ignoring it!\n"
571
/*
572 573 574
 * All of RAM fits into lowmem - but if user wants highmem
 * artificially via the highmem=x boot parameter then create
 * it:
575
 */
576
static void __init lowmem_pfn_init(void)
577
{
Y
Yinghai Lu 已提交
578
	/* max_low_pfn is 0, we already have early_res support */
579
	max_low_pfn = max_pfn;
580

581 582 583 584 585 586 587 588 589 590 591
	if (highmem_pages == -1)
		highmem_pages = 0;
#ifdef CONFIG_HIGHMEM
	if (highmem_pages >= max_pfn) {
		printk(KERN_ERR MSG_HIGHMEM_TOO_BIG,
			pages_to_mb(highmem_pages), pages_to_mb(max_pfn));
		highmem_pages = 0;
	}
	if (highmem_pages) {
		if (max_low_pfn - highmem_pages < 64*1024*1024/PAGE_SIZE) {
			printk(KERN_ERR MSG_LOWMEM_TOO_SMALL,
592 593 594
				pages_to_mb(highmem_pages));
			highmem_pages = 0;
		}
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
		max_low_pfn -= highmem_pages;
	}
#else
	if (highmem_pages)
		printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n");
#endif
}

#define MSG_HIGHMEM_TOO_SMALL \
	"only %luMB highmem pages available, ignoring highmem size of %luMB!\n"

#define MSG_HIGHMEM_TRIMMED \
	"Warning: only 4GB will be used. Use a HIGHMEM64G enabled kernel!\n"
/*
 * We have more RAM than fits into lowmem - we try to put it into
 * highmem, also taking the highmem=x boot parameter into account:
 */
612
static void __init highmem_pfn_init(void)
613
{
614 615
	max_low_pfn = MAXMEM_PFN;

616 617 618 619 620 621 622 623 624 625 626 627
	if (highmem_pages == -1)
		highmem_pages = max_pfn - MAXMEM_PFN;

	if (highmem_pages + MAXMEM_PFN < max_pfn)
		max_pfn = MAXMEM_PFN + highmem_pages;

	if (highmem_pages + MAXMEM_PFN > max_pfn) {
		printk(KERN_WARNING MSG_HIGHMEM_TOO_SMALL,
			pages_to_mb(max_pfn - MAXMEM_PFN),
			pages_to_mb(highmem_pages));
		highmem_pages = 0;
	}
628
#ifndef CONFIG_HIGHMEM
629 630 631 632 633 634 635
	/* Maximum memory usable is what is directly addressable */
	printk(KERN_WARNING "Warning only %ldMB will be used.\n", MAXMEM>>20);
	if (max_pfn > MAX_NONPAE_PFN)
		printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
	else
		printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
	max_pfn = MAXMEM_PFN;
636 637
#else /* !CONFIG_HIGHMEM */
#ifndef CONFIG_HIGHMEM64G
638 639 640 641
	if (max_pfn > MAX_NONPAE_PFN) {
		max_pfn = MAX_NONPAE_PFN;
		printk(KERN_WARNING MSG_HIGHMEM_TRIMMED);
	}
642 643
#endif /* !CONFIG_HIGHMEM64G */
#endif /* !CONFIG_HIGHMEM */
644 645 646 647 648 649 650 651 652
}

/*
 * Determine low and high memory ranges:
 */
void __init find_low_pfn_range(void)
{
	/* it could update max_pfn */

653
	if (max_pfn <= MAXMEM_PFN)
654
		lowmem_pfn_init();
655 656
	else
		highmem_pfn_init();
657 658
}

659
#ifndef CONFIG_NEED_MULTIPLE_NODES
660
void __init initmem_init(void)
661 662 663 664 665 666 667 668 669 670 671
{
#ifdef CONFIG_HIGHMEM
	highstart_pfn = highend_pfn = max_pfn;
	if (max_pfn > max_low_pfn)
		highstart_pfn = max_low_pfn;
	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
		pages_to_mb(highend_pfn - highstart_pfn));
	high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
#else
	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
#endif
T
Tejun Heo 已提交
672

673
	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
T
Tejun Heo 已提交
674 675
	sparse_memory_present_with_active_regions(0);

676
#ifdef CONFIG_FLATMEM
677
	max_mapnr = IS_ENABLED(CONFIG_HIGHMEM) ? highend_pfn : max_low_pfn;
678
#endif
679 680
	__vmalloc_start_set = true;

681 682 683 684 685
	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
			pages_to_mb(max_low_pfn));

	setup_bootmem_allocator();
}
686
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
687 688 689 690 691

void __init setup_bootmem_allocator(void)
{
	printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
		 max_pfn_mapped<<PAGE_SHIFT);
692
	printk(KERN_INFO "  low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
693 694
}

L
Linus Torvalds 已提交
695 696 697 698 699 700 701 702 703 704 705 706 707 708
/*
 * paging_init() sets up the page tables - note that the first 8MB are
 * already mapped by head.S.
 *
 * This routines also unmaps the page at virtual kernel address 0, so
 * that we can trap those pesky NULL-reference errors in the kernel.
 */
void __init paging_init(void)
{
	pagetable_init();

	__flush_tlb_all();

	kmap_init();
709 710 711 712

	/*
	 * NOTE: at this point the bootmem allocator is fully available.
	 */
713
	olpc_dt_build_devicetree();
714
	sparse_memory_present_with_active_regions(MAX_NUMNODES);
715 716
	sparse_init();
	zone_sizes_init();
L
Linus Torvalds 已提交
717 718 719 720
}

/*
 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
D
Dmitri Vorobiev 已提交
721 722 723
 * and also on some strange 486's. All 586+'s are OK. This used to involve
 * black magic jumps to work around some nasty CPU bugs, but fortunately the
 * switch to using exceptions got rid of all that.
L
Linus Torvalds 已提交
724 725 726
 */
static void __init test_wp_bit(void)
{
727
	char z = 0;
728

729
	printk(KERN_INFO "Checking if this processor honours the WP bit even in supervisor mode...");
L
Linus Torvalds 已提交
730

731
	__set_fixmap(FIX_WP_TEST, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO);
L
Linus Torvalds 已提交
732

733 734 735 736
	if (probe_kernel_write((char *)fix_to_virt(FIX_WP_TEST), &z, 1)) {
		clear_fixmap(FIX_WP_TEST);
		printk(KERN_CONT "Ok.\n");
		return;
L
Linus Torvalds 已提交
737
	}
738

739 740
	printk(KERN_CONT "No.\n");
	panic("Linux doesn't support CPUs with broken WP.");
L
Linus Torvalds 已提交
741 742 743 744
}

void __init mem_init(void)
{
745 746
	pci_iommu_alloc();

747
#ifdef CONFIG_FLATMEM
E
Eric Sesterhenn 已提交
748
	BUG_ON(!mem_map);
L
Linus Torvalds 已提交
749
#endif
750 751 752 753 754 755 756 757 758 759 760
	/*
	 * With CONFIG_DEBUG_PAGEALLOC initialization of highmem pages has to
	 * be done before free_all_bootmem(). Memblock use free low memory for
	 * temporary data (see find_range_array()) and for this purpose can use
	 * pages that was already passed to the buddy allocator, hence marked as
	 * not accessible in the page tables when compiled with
	 * CONFIG_DEBUG_PAGEALLOC. Otherwise order of initialization is not
	 * important here.
	 */
	set_highmem_pages_init();

L
Linus Torvalds 已提交
761
	/* this will put all low memory onto the freelists */
762
	free_all_bootmem();
L
Linus Torvalds 已提交
763

764 765
	after_bootmem = 1;

766
	mem_init_print_info(NULL);
767
	printk(KERN_INFO "virtual kernel memory layout:\n"
I
Ingo Molnar 已提交
768
		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
769
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
770
		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n"
771
#endif
I
Ingo Molnar 已提交
772 773 774 775 776 777 778
		"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"      .init : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .data : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .text : 0x%08lx - 0x%08lx   (%4ld kB)\n",
		FIXADDR_START, FIXADDR_TOP,
		(FIXADDR_TOP - FIXADDR_START) >> 10,
779 780

#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
781 782
		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
		(LAST_PKMAP*PAGE_SIZE) >> 10,
783 784
#endif

I
Ingo Molnar 已提交
785 786
		VMALLOC_START, VMALLOC_END,
		(VMALLOC_END - VMALLOC_START) >> 20,
787

I
Ingo Molnar 已提交
788 789
		(unsigned long)__va(0), (unsigned long)high_memory,
		((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
790

I
Ingo Molnar 已提交
791 792 793
		(unsigned long)&__init_begin, (unsigned long)&__init_end,
		((unsigned long)&__init_end -
		 (unsigned long)&__init_begin) >> 10,
794

I
Ingo Molnar 已提交
795 796
		(unsigned long)&_etext, (unsigned long)&_edata,
		((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
797

I
Ingo Molnar 已提交
798 799
		(unsigned long)&_text, (unsigned long)&_etext,
		((unsigned long)&_etext - (unsigned long)&_text) >> 10);
800

801 802 803 804 805 806 807 808 809 810 811 812 813 814
	/*
	 * Check boundaries twice: Some fundamental inconsistencies can
	 * be detected at build time already.
	 */
#define __FIXADDR_TOP (-PAGE_SIZE)
#ifdef CONFIG_HIGHMEM
	BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUILD_BUG_ON(VMALLOC_END			> PKMAP_BASE);
#endif
#define high_memory (-128UL << 20)
	BUILD_BUG_ON(VMALLOC_START			>= VMALLOC_END);
#undef high_memory
#undef __FIXADDR_TOP

815
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
816 817
	BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUG_ON(VMALLOC_END				> PKMAP_BASE);
818
#endif
819
	BUG_ON(VMALLOC_START				>= VMALLOC_END);
I
Ingo Molnar 已提交
820
	BUG_ON((unsigned long)high_memory		> VMALLOC_START);
821

822
	test_wp_bit();
L
Linus Torvalds 已提交
823 824
}

825
#ifdef CONFIG_MEMORY_HOTPLUG
826
int arch_add_memory(int nid, u64 start, u64 size, bool for_device)
827 828 829 830
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

831
	return __add_pages(nid, start_pfn, nr_pages, !for_device);
832
}
833 834 835 836 837 838 839 840 841 842 843 844

#ifdef CONFIG_MEMORY_HOTREMOVE
int arch_remove_memory(u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;
	struct zone *zone;

	zone = page_zone(pfn_to_page(start_pfn));
	return __remove_pages(zone, start_pfn, nr_pages);
}
#endif
845
#endif
846

847
int kernel_set_to_readonly __read_mostly;
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876

void set_kernel_text_rw(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read write\n",
		 start, start+size);

	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
}

void set_kernel_text_ro(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read only\n",
		 start, start+size);

	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
}

877 878 879 880
static void mark_nxdata_nx(void)
{
	/*
	 * When this called, init has already been executed and released,
L
Lucas De Marchi 已提交
881
	 * so everything past _etext should be NX.
882 883 884 885 886 887 888 889 890 891 892 893
	 */
	unsigned long start = PFN_ALIGN(_etext);
	/*
	 * This comes from is_kernel_text upper limit. Also HPAGE where used:
	 */
	unsigned long size = (((unsigned long)__init_end + HPAGE_SIZE) & HPAGE_MASK) - start;

	if (__supported_pte_mask & _PAGE_NX)
		printk(KERN_INFO "NX-protecting the kernel data: %luk\n", size >> 10);
	set_pages_nx(virt_to_page(start), size >> PAGE_SHIFT);
}

894 895
void mark_rodata_ro(void)
{
896 897
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;
898

899 900 901
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
	printk(KERN_INFO "Write protecting the kernel text: %luk\n",
		size >> 10);
902

903 904
	kernel_set_to_readonly = 1;

905
#ifdef CONFIG_CPA_DEBUG
906 907 908
	printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n",
		start, start+size);
	set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
909

910 911
	printk(KERN_INFO "Testing CPA: write protecting again\n");
	set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
912
#endif
913

914 915
	start += size;
	size = (unsigned long)__end_rodata - start;
916
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
917 918
	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
		size >> 10);
919

920
#ifdef CONFIG_CPA_DEBUG
921
	printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size);
922
	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
923

924
	printk(KERN_INFO "Testing CPA: write protecting again\n");
925
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
926
#endif
927
	mark_nxdata_nx();
S
Stephen Smalley 已提交
928 929
	if (__supported_pte_mask & _PAGE_NX)
		debug_checkwx();
930
}