init_32.c 27.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 *
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 */

#include <linux/module.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
24
#include <linux/pci.h>
25
#include <linux/pfn.h>
26
#include <linux/poison.h>
L
Linus Torvalds 已提交
27 28
#include <linux/bootmem.h>
#include <linux/proc_fs.h>
29
#include <linux/memory_hotplug.h>
30
#include <linux/initrd.h>
31
#include <linux/cpumask.h>
32
#include <linux/gfp.h>
L
Linus Torvalds 已提交
33

34
#include <asm/asm.h>
35
#include <asm/bios_ebda.h>
L
Linus Torvalds 已提交
36 37 38 39 40 41 42 43
#include <asm/processor.h>
#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/pgtable.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/apic.h>
I
Ingo Molnar 已提交
44
#include <asm/bugs.h>
L
Linus Torvalds 已提交
45 46
#include <asm/tlb.h>
#include <asm/tlbflush.h>
47
#include <asm/pgalloc.h>
L
Linus Torvalds 已提交
48
#include <asm/sections.h>
49
#include <asm/paravirt.h>
50
#include <asm/setup.h>
51
#include <asm/cacheflush.h>
52
#include <asm/page_types.h>
53
#include <asm/init.h>
L
Linus Torvalds 已提交
54 55 56

unsigned long highstart_pfn, highend_pfn;

I
Ingo Molnar 已提交
57
static noinline int do_test_wp_bit(void);
L
Linus Torvalds 已提交
58

59
bool __read_mostly __vmalloc_start_set = false;
60

61
static __init void *alloc_low_page(void)
62
{
63
	unsigned long pfn = e820_table_end++;
64 65
	void *adr;

66
	if (pfn >= e820_table_top)
67 68 69 70 71 72 73
		panic("alloc_low_page: ran out of memory");

	adr = __va(pfn * PAGE_SIZE);
	memset(adr, 0, PAGE_SIZE);
	return adr;
}

L
Linus Torvalds 已提交
74 75 76 77 78 79 80 81 82
/*
 * Creates a middle page table and puts a pointer to it in the
 * given global directory entry. This only returns the gd entry
 * in non-PAE compilation mode, since the middle layer is folded.
 */
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
	pud_t *pud;
	pmd_t *pmd_table;
I
Ingo Molnar 已提交
83

L
Linus Torvalds 已提交
84
#ifdef CONFIG_X86_PAE
85
	if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
86
		if (after_bootmem)
87
			pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE);
88
		else
89
			pmd_table = (pmd_t *)alloc_low_page();
90
		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
91 92
		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
		pud = pud_offset(pgd, 0);
I
Ingo Molnar 已提交
93
		BUG_ON(pmd_table != pmd_offset(pud, 0));
94 95

		return pmd_table;
96 97
	}
#endif
L
Linus Torvalds 已提交
98 99
	pud = pud_offset(pgd, 0);
	pmd_table = pmd_offset(pud, 0);
I
Ingo Molnar 已提交
100

L
Linus Torvalds 已提交
101 102 103 104 105
	return pmd_table;
}

/*
 * Create a page table and place a pointer to it in a middle page
I
Ingo Molnar 已提交
106
 * directory entry:
L
Linus Torvalds 已提交
107 108 109
 */
static pte_t * __init one_page_table_init(pmd_t *pmd)
{
110
	if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
111 112
		pte_t *page_table = NULL;

113
		if (after_bootmem) {
V
Vegard Nossum 已提交
114
#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
115
			page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
116
#endif
117 118
			if (!page_table)
				page_table =
119
				(pte_t *)alloc_bootmem_pages(PAGE_SIZE);
120 121
		} else
			page_table = (pte_t *)alloc_low_page();
122

123
		paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
L
Linus Torvalds 已提交
124
		set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
125
		BUG_ON(page_table != pte_offset_kernel(pmd, 0));
L
Linus Torvalds 已提交
126
	}
127

L
Linus Torvalds 已提交
128 129 130
	return pte_offset_kernel(pmd, 0);
}

131
pmd_t * __init populate_extra_pmd(unsigned long vaddr)
132 133 134
{
	int pgd_idx = pgd_index(vaddr);
	int pmd_idx = pmd_index(vaddr);
135 136 137 138 139 140 141

	return one_md_table_init(swapper_pg_dir + pgd_idx) + pmd_idx;
}

pte_t * __init populate_extra_pte(unsigned long vaddr)
{
	int pte_idx = pte_index(vaddr);
142 143
	pmd_t *pmd;

144 145
	pmd = populate_extra_pmd(vaddr);
	return one_page_table_init(pmd) + pte_idx;
146 147
}

148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
					   unsigned long vaddr, pte_t *lastpte)
{
#ifdef CONFIG_HIGHMEM
	/*
	 * Something (early fixmap) may already have put a pte
	 * page here, which causes the page table allocation
	 * to become nonlinear. Attempt to fix it, and if it
	 * is still nonlinear then we have to bug.
	 */
	int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
	int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;

	if (pmd_idx_kmap_begin != pmd_idx_kmap_end
	    && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
	    && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end
164 165
	    && ((__pa(pte) >> PAGE_SHIFT) < e820_table_start
		|| (__pa(pte) >> PAGE_SHIFT) >= e820_table_end)) {
166 167 168
		pte_t *newpte;
		int i;

169
		BUG_ON(after_bootmem);
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
		newpte = alloc_low_page();
		for (i = 0; i < PTRS_PER_PTE; i++)
			set_pte(newpte + i, pte[i]);

		paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
		set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
		BUG_ON(newpte != pte_offset_kernel(pmd, 0));
		__flush_tlb_all();

		paravirt_release_pte(__pa(pte) >> PAGE_SHIFT);
		pte = newpte;
	}
	BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1)
	       && vaddr > fix_to_virt(FIX_KMAP_END)
	       && lastpte && lastpte + PTRS_PER_PTE != pte);
#endif
	return pte;
}

L
Linus Torvalds 已提交
189
/*
I
Ingo Molnar 已提交
190
 * This function initializes a certain range of kernel virtual memory
L
Linus Torvalds 已提交
191 192
 * with new bootmem page tables, everywhere page tables are missing in
 * the given range.
I
Ingo Molnar 已提交
193 194 195
 *
 * NOTE: The pagetables are allocated contiguous on the physical space
 * so we can cache the place of the first one and move around without
L
Linus Torvalds 已提交
196 197
 * checking the pgd every time.
 */
I
Ingo Molnar 已提交
198 199
static void __init
page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
L
Linus Torvalds 已提交
200 201 202
{
	int pgd_idx, pmd_idx;
	unsigned long vaddr;
I
Ingo Molnar 已提交
203 204
	pgd_t *pgd;
	pmd_t *pmd;
205
	pte_t *pte = NULL;
L
Linus Torvalds 已提交
206 207 208 209 210 211 212

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
	pmd_idx = pmd_index(vaddr);
	pgd = pgd_base + pgd_idx;

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
213 214
		pmd = one_md_table_init(pgd);
		pmd = pmd + pmd_index(vaddr);
I
Ingo Molnar 已提交
215 216
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd++, pmd_idx++) {
217 218
			pte = page_table_kmap_check(one_page_table_init(pmd),
			                            pmd, vaddr, pte);
L
Linus Torvalds 已提交
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233

			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
}

static inline int is_kernel_text(unsigned long addr)
{
	if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
		return 1;
	return 0;
}

/*
I
Ingo Molnar 已提交
234 235 236
 * This maps the physical memory to kernel virtual address space, a total
 * of max_low_pfn pages, by creating page tables starting from address
 * PAGE_OFFSET:
L
Linus Torvalds 已提交
237
 */
238 239 240 241
unsigned long __init
kernel_physical_mapping_init(unsigned long start,
			     unsigned long end,
			     unsigned long page_size_mask)
L
Linus Torvalds 已提交
242
{
243
	int use_pse = page_size_mask == (1<<PG_LEVEL_2M);
244
	unsigned long last_map_addr = end;
245
	unsigned long start_pfn, end_pfn;
246
	pgd_t *pgd_base = swapper_pg_dir;
I
Ingo Molnar 已提交
247
	int pgd_idx, pmd_idx, pte_ofs;
L
Linus Torvalds 已提交
248 249 250 251
	unsigned long pfn;
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
252 253 254
	unsigned pages_2m, pages_4k;
	int mapping_iter;

255 256 257
	start_pfn = start >> PAGE_SHIFT;
	end_pfn = end >> PAGE_SHIFT;

258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
	/*
	 * First iteration will setup identity mapping using large/small pages
	 * based on use_pse, with other attributes same as set by
	 * the early code in head_32.S
	 *
	 * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
	 * as desired for the kernel identity mapping.
	 *
	 * This two pass mechanism conforms to the TLB app note which says:
	 *
	 *     "Software should not write to a paging-structure entry in a way
	 *      that would change, for any linear address, both the page size
	 *      and either the page frame or attributes."
	 */
	mapping_iter = 1;
L
Linus Torvalds 已提交
273

274 275
	if (!cpu_has_pse)
		use_pse = 0;
L
Linus Torvalds 已提交
276

277 278
repeat:
	pages_2m = pages_4k = 0;
279 280 281
	pfn = start_pfn;
	pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
	pgd = pgd_base + pgd_idx;
L
Linus Torvalds 已提交
282 283
	for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
		pmd = one_md_table_init(pgd);
I
Ingo Molnar 已提交
284

285 286 287 288 289 290 291 292 293
		if (pfn >= end_pfn)
			continue;
#ifdef CONFIG_X86_PAE
		pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
		pmd += pmd_idx;
#else
		pmd_idx = 0;
#endif
		for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
J
Jeremy Fitzhardinge 已提交
294
		     pmd++, pmd_idx++) {
I
Ingo Molnar 已提交
295
			unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
L
Linus Torvalds 已提交
296

I
Ingo Molnar 已提交
297 298 299 300
			/*
			 * Map with big pages if possible, otherwise
			 * create normal page tables:
			 */
301
			if (use_pse) {
I
Ingo Molnar 已提交
302
				unsigned int addr2;
J
Jeremy Fitzhardinge 已提交
303
				pgprot_t prot = PAGE_KERNEL_LARGE;
304 305 306 307 308 309 310
				/*
				 * first pass will use the same initial
				 * identity mapping attribute + _PAGE_PSE.
				 */
				pgprot_t init_prot =
					__pgprot(PTE_IDENT_ATTR |
						 _PAGE_PSE);
J
Jeremy Fitzhardinge 已提交
311

I
Ingo Molnar 已提交
312
				addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
J
Jeremy Fitzhardinge 已提交
313 314
					PAGE_OFFSET + PAGE_SIZE-1;

I
Ingo Molnar 已提交
315 316
				if (is_kernel_text(addr) ||
				    is_kernel_text(addr2))
J
Jeremy Fitzhardinge 已提交
317 318
					prot = PAGE_KERNEL_LARGE_EXEC;

319
				pages_2m++;
320 321 322 323
				if (mapping_iter == 1)
					set_pmd(pmd, pfn_pmd(pfn, init_prot));
				else
					set_pmd(pmd, pfn_pmd(pfn, prot));
324

L
Linus Torvalds 已提交
325
				pfn += PTRS_PER_PTE;
I
Ingo Molnar 已提交
326 327 328
				continue;
			}
			pte = one_page_table_init(pmd);
L
Linus Torvalds 已提交
329

330 331 332
			pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
			pte += pte_ofs;
			for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
I
Ingo Molnar 已提交
333 334
			     pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
				pgprot_t prot = PAGE_KERNEL;
335 336 337 338 339
				/*
				 * first pass will use the same initial
				 * identity mapping attribute.
				 */
				pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
J
Jeremy Fitzhardinge 已提交
340

I
Ingo Molnar 已提交
341 342
				if (is_kernel_text(addr))
					prot = PAGE_KERNEL_EXEC;
J
Jeremy Fitzhardinge 已提交
343

344
				pages_4k++;
345
				if (mapping_iter == 1) {
346
					set_pte(pte, pfn_pte(pfn, init_prot));
347 348
					last_map_addr = (pfn << PAGE_SHIFT) + PAGE_SIZE;
				} else
349
					set_pte(pte, pfn_pte(pfn, prot));
L
Linus Torvalds 已提交
350 351 352
			}
		}
	}
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
	if (mapping_iter == 1) {
		/*
		 * update direct mapping page count only in the first
		 * iteration.
		 */
		update_page_count(PG_LEVEL_2M, pages_2m);
		update_page_count(PG_LEVEL_4K, pages_4k);

		/*
		 * local global flush tlb, which will flush the previous
		 * mappings present in both small and large page TLB's.
		 */
		__flush_tlb_all();

		/*
		 * Second iteration will set the actual desired PTE attributes.
		 */
		mapping_iter = 2;
		goto repeat;
	}
373
	return last_map_addr;
374 375
}

L
Linus Torvalds 已提交
376 377 378
pte_t *kmap_pte;
pgprot_t kmap_prot;

I
Ingo Molnar 已提交
379 380 381 382 383
static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
{
	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
			vaddr), vaddr), vaddr);
}
L
Linus Torvalds 已提交
384 385 386 387 388

static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

I
Ingo Molnar 已提交
389 390 391
	/*
	 * Cache the first kmap pte:
	 */
L
Linus Torvalds 已提交
392 393 394 395 396 397
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);

	kmap_prot = PAGE_KERNEL;
}

398
#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
399 400
static void __init permanent_kmaps_init(pgd_t *pgd_base)
{
I
Ingo Molnar 已提交
401
	unsigned long vaddr;
L
Linus Torvalds 已提交
402 403 404 405 406 407 408 409 410 411 412 413
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	vaddr = PKMAP_BASE;
	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);

	pgd = swapper_pg_dir + pgd_index(vaddr);
	pud = pud_offset(pgd, vaddr);
	pmd = pmd_offset(pud, vaddr);
	pte = pte_offset_kernel(pmd, vaddr);
I
Ingo Molnar 已提交
414
	pkmap_page_table = pte;
L
Linus Torvalds 已提交
415 416
}

417
static void __init add_one_highpage_init(struct page *page)
L
Linus Torvalds 已提交
418
{
Y
Yinghai Lu 已提交
419 420 421 422
	ClearPageReserved(page);
	init_page_count(page);
	__free_page(page);
	totalhigh_pages++;
L
Linus Torvalds 已提交
423 424
}

425 426 427 428 429
struct add_highpages_data {
	unsigned long start_pfn;
	unsigned long end_pfn;
};

430
static int __init add_highpages_work_fn(unsigned long start_pfn,
431
					 unsigned long end_pfn, void *datax)
L
Linus Torvalds 已提交
432
{
433 434 435 436
	int node_pfn;
	struct page *page;
	unsigned long final_start_pfn, final_end_pfn;
	struct add_highpages_data *data;
I
Ingo Molnar 已提交
437

438 439 440 441 442
	data = (struct add_highpages_data *)datax;

	final_start_pfn = max(start_pfn, data->start_pfn);
	final_end_pfn = min(end_pfn, data->end_pfn);
	if (final_start_pfn >= final_end_pfn)
443
		return 0;
444 445 446 447 448 449

	for (node_pfn = final_start_pfn; node_pfn < final_end_pfn;
	     node_pfn++) {
		if (!pfn_valid(node_pfn))
			continue;
		page = pfn_to_page(node_pfn);
450
		add_one_highpage_init(page);
451
	}
452

453 454
	return 0;

455 456 457
}

void __init add_highpages_with_active_regions(int nid, unsigned long start_pfn,
Y
Yinghai Lu 已提交
458
					      unsigned long end_pfn)
459 460 461 462 463 464 465 466 467
{
	struct add_highpages_data data;

	data.start_pfn = start_pfn;
	data.end_pfn = end_pfn;

	work_with_active_regions(nid, add_highpages_work_fn, &data);
}

L
Linus Torvalds 已提交
468
#else
469 470 471
static inline void permanent_kmaps_init(pgd_t *pgd_base)
{
}
L
Linus Torvalds 已提交
472 473
#endif /* CONFIG_HIGHMEM */

474
void __init native_pagetable_setup_start(pgd_t *base)
L
Linus Torvalds 已提交
475
{
476 477 478 479 480
	unsigned long pfn, va;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
481 482

	/*
483 484
	 * Remove any mappings which extend past the end of physical
	 * memory from the boot time page table:
485
	 */
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
	for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
		va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
		pgd = base + pgd_index(va);
		if (!pgd_present(*pgd))
			break;

		pud = pud_offset(pgd, va);
		pmd = pmd_offset(pud, va);
		if (!pmd_present(*pmd))
			break;

		pte = pte_offset_kernel(pmd, va);
		if (!pte_present(*pte))
			break;

		pte_clear(NULL, va, pte);
	}
503
	paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
504 505 506 507 508 509 510 511 512 513 514 515
}

void __init native_pagetable_setup_done(pgd_t *base)
{
}

/*
 * Build a proper pagetable for the kernel mappings.  Up until this
 * point, we've been running on some set of pagetables constructed by
 * the boot process.
 *
 * If we're booting on native hardware, this will be a pagetable
516 517
 * constructed in arch/x86/kernel/head_32.S.  The root of the
 * pagetable will be swapper_pg_dir.
518 519 520 521 522 523 524 525 526 527 528
 *
 * If we're booting paravirtualized under a hypervisor, then there are
 * more options: we may already be running PAE, and the pagetable may
 * or may not be based in swapper_pg_dir.  In any case,
 * paravirt_pagetable_setup_start() will set up swapper_pg_dir
 * appropriately for the rest of the initialization to work.
 *
 * In general, pagetable_init() assumes that the pagetable may already
 * be partially populated, and so it avoids stomping on any existing
 * mappings.
 */
529
void __init early_ioremap_page_table_range_init(void)
530
{
531
	pgd_t *pgd_base = swapper_pg_dir;
I
Ingo Molnar 已提交
532
	unsigned long vaddr, end;
533

L
Linus Torvalds 已提交
534 535 536 537 538
	/*
	 * Fixed mappings, only the page table structure has to be
	 * created - mappings will be set by set_fixmap():
	 */
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
539 540
	end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
	page_table_range_init(vaddr, end, pgd_base);
541
	early_ioremap_reset();
542 543 544 545 546 547
}

static void __init pagetable_init(void)
{
	pgd_t *pgd_base = swapper_pg_dir;

L
Linus Torvalds 已提交
548 549 550
	permanent_kmaps_init(pgd_base);
}

551
#ifdef CONFIG_ACPI_SLEEP
L
Linus Torvalds 已提交
552
/*
553
 * ACPI suspend needs this for resume, because things like the intel-agp
L
Linus Torvalds 已提交
554 555
 * driver might have split up a kernel 4MB mapping.
 */
556
char swsusp_pg_dir[PAGE_SIZE]
I
Ingo Molnar 已提交
557
	__attribute__ ((aligned(PAGE_SIZE)));
L
Linus Torvalds 已提交
558 559 560 561 562

static inline void save_pg_dir(void)
{
	memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
}
563
#else /* !CONFIG_ACPI_SLEEP */
L
Linus Torvalds 已提交
564 565 566
static inline void save_pg_dir(void)
{
}
567
#endif /* !CONFIG_ACPI_SLEEP */
L
Linus Torvalds 已提交
568

569
void zap_low_mappings(bool early)
L
Linus Torvalds 已提交
570 571 572 573 574 575 576 577 578
{
	int i;

	/*
	 * Zap initial low-memory mappings.
	 *
	 * Note that "pgd_clear()" doesn't do it for
	 * us, because pgd_clear() is a no-op on i386.
	 */
J
Jeremy Fitzhardinge 已提交
579
	for (i = 0; i < KERNEL_PGD_BOUNDARY; i++) {
L
Linus Torvalds 已提交
580 581 582 583 584
#ifdef CONFIG_X86_PAE
		set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
#else
		set_pgd(swapper_pg_dir+i, __pgd(0));
#endif
I
Ingo Molnar 已提交
585
	}
586 587 588 589 590

	if (early)
		__flush_tlb();
	else
		flush_tlb_all();
L
Linus Torvalds 已提交
591 592
}

593
pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL | _PAGE_IOMAP);
594 595
EXPORT_SYMBOL_GPL(__supported_pte_mask);

596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
/* user-defined highmem size */
static unsigned int highmem_pages = -1;

/*
 * highmem=size forces highmem to be exactly 'size' bytes.
 * This works even on boxes that have no highmem otherwise.
 * This also works to reduce highmem size on bigger boxes.
 */
static int __init parse_highmem(char *arg)
{
	if (!arg)
		return -EINVAL;

	highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
	return 0;
}
early_param("highmem", parse_highmem);

614 615 616 617 618
#define MSG_HIGHMEM_TOO_BIG \
	"highmem size (%luMB) is bigger than pages available (%luMB)!\n"

#define MSG_LOWMEM_TOO_SMALL \
	"highmem size (%luMB) results in <64MB lowmem, ignoring it!\n"
619
/*
620 621 622
 * All of RAM fits into lowmem - but if user wants highmem
 * artificially via the highmem=x boot parameter then create
 * it:
623
 */
624
void __init lowmem_pfn_init(void)
625
{
Y
Yinghai Lu 已提交
626
	/* max_low_pfn is 0, we already have early_res support */
627
	max_low_pfn = max_pfn;
628

629 630 631 632 633 634 635 636 637 638 639
	if (highmem_pages == -1)
		highmem_pages = 0;
#ifdef CONFIG_HIGHMEM
	if (highmem_pages >= max_pfn) {
		printk(KERN_ERR MSG_HIGHMEM_TOO_BIG,
			pages_to_mb(highmem_pages), pages_to_mb(max_pfn));
		highmem_pages = 0;
	}
	if (highmem_pages) {
		if (max_low_pfn - highmem_pages < 64*1024*1024/PAGE_SIZE) {
			printk(KERN_ERR MSG_LOWMEM_TOO_SMALL,
640 641 642
				pages_to_mb(highmem_pages));
			highmem_pages = 0;
		}
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661
		max_low_pfn -= highmem_pages;
	}
#else
	if (highmem_pages)
		printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n");
#endif
}

#define MSG_HIGHMEM_TOO_SMALL \
	"only %luMB highmem pages available, ignoring highmem size of %luMB!\n"

#define MSG_HIGHMEM_TRIMMED \
	"Warning: only 4GB will be used. Use a HIGHMEM64G enabled kernel!\n"
/*
 * We have more RAM than fits into lowmem - we try to put it into
 * highmem, also taking the highmem=x boot parameter into account:
 */
void __init highmem_pfn_init(void)
{
662 663
	max_low_pfn = MAXMEM_PFN;

664 665 666 667 668 669 670 671 672 673 674 675
	if (highmem_pages == -1)
		highmem_pages = max_pfn - MAXMEM_PFN;

	if (highmem_pages + MAXMEM_PFN < max_pfn)
		max_pfn = MAXMEM_PFN + highmem_pages;

	if (highmem_pages + MAXMEM_PFN > max_pfn) {
		printk(KERN_WARNING MSG_HIGHMEM_TOO_SMALL,
			pages_to_mb(max_pfn - MAXMEM_PFN),
			pages_to_mb(highmem_pages));
		highmem_pages = 0;
	}
676
#ifndef CONFIG_HIGHMEM
677 678 679 680 681 682 683
	/* Maximum memory usable is what is directly addressable */
	printk(KERN_WARNING "Warning only %ldMB will be used.\n", MAXMEM>>20);
	if (max_pfn > MAX_NONPAE_PFN)
		printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
	else
		printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
	max_pfn = MAXMEM_PFN;
684 685
#else /* !CONFIG_HIGHMEM */
#ifndef CONFIG_HIGHMEM64G
686 687 688 689
	if (max_pfn > MAX_NONPAE_PFN) {
		max_pfn = MAX_NONPAE_PFN;
		printk(KERN_WARNING MSG_HIGHMEM_TRIMMED);
	}
690 691
#endif /* !CONFIG_HIGHMEM64G */
#endif /* !CONFIG_HIGHMEM */
692 693 694 695 696 697 698 699 700
}

/*
 * Determine low and high memory ranges:
 */
void __init find_low_pfn_range(void)
{
	/* it could update max_pfn */

701
	if (max_pfn <= MAXMEM_PFN)
702
		lowmem_pfn_init();
703 704
	else
		highmem_pfn_init();
705 706
}

707
#ifndef CONFIG_NEED_MULTIPLE_NODES
D
David Rientjes 已提交
708 709
void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
				int acpi, int k8)
710 711 712 713 714
{
#ifdef CONFIG_HIGHMEM
	highstart_pfn = highend_pfn = max_pfn;
	if (max_pfn > max_low_pfn)
		highstart_pfn = max_low_pfn;
715
	e820_register_active_regions(0, 0, highend_pfn);
716
	sparse_memory_present_with_active_regions(0);
717 718 719 720 721
	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
		pages_to_mb(highend_pfn - highstart_pfn));
	num_physpages = highend_pfn;
	high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
#else
722
	e820_register_active_regions(0, 0, max_low_pfn);
723
	sparse_memory_present_with_active_regions(0);
724 725 726 727 728 729
	num_physpages = max_low_pfn;
	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
#endif
#ifdef CONFIG_FLATMEM
	max_mapnr = num_physpages;
#endif
730 731
	__vmalloc_start_set = true;

732 733 734 735 736
	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
			pages_to_mb(max_low_pfn));

	setup_bootmem_allocator();
}
737
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
738

739
static void __init zone_sizes_init(void)
740 741 742 743 744 745 746 747 748 749 750 751 752
{
	unsigned long max_zone_pfns[MAX_NR_ZONES];
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
	max_zone_pfns[ZONE_DMA] =
		virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
#ifdef CONFIG_HIGHMEM
	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
#endif

	free_area_init_nodes(max_zone_pfns);
}

Y
Yinghai Lu 已提交
753
#ifndef CONFIG_NO_BOOTMEM
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
static unsigned long __init setup_node_bootmem(int nodeid,
				 unsigned long start_pfn,
				 unsigned long end_pfn,
				 unsigned long bootmap)
{
	unsigned long bootmap_size;

	/* don't touch min_low_pfn */
	bootmap_size = init_bootmem_node(NODE_DATA(nodeid),
					 bootmap >> PAGE_SHIFT,
					 start_pfn, end_pfn);
	printk(KERN_INFO "  node %d low ram: %08lx - %08lx\n",
		nodeid, start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
	printk(KERN_INFO "  node %d bootmap %08lx - %08lx\n",
		 nodeid, bootmap, bootmap + bootmap_size);
	free_bootmem_with_active_regions(nodeid, end_pfn);

	return bootmap + bootmap_size;
}
Y
Yinghai Lu 已提交
773
#endif
774

775 776
void __init setup_bootmem_allocator(void)
{
Y
Yinghai Lu 已提交
777
#ifndef CONFIG_NO_BOOTMEM
778
	int nodeid;
779 780 781 782 783
	unsigned long bootmap_size, bootmap;
	/*
	 * Initialize the boot-time allocator (with low memory only):
	 */
	bootmap_size = bootmem_bootmap_pages(max_low_pfn)<<PAGE_SHIFT;
784
	bootmap = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, bootmap_size,
785 786 787 788
				 PAGE_SIZE);
	if (bootmap == -1L)
		panic("Cannot find bootmem map of size %ld\n", bootmap_size);
	reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP");
Y
Yinghai Lu 已提交
789
#endif
Y
Yinghai Lu 已提交
790

791 792
	printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
		 max_pfn_mapped<<PAGE_SHIFT);
793
	printk(KERN_INFO "  low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
794

Y
Yinghai Lu 已提交
795
#ifndef CONFIG_NO_BOOTMEM
Y
Yinghai Lu 已提交
796 797 798
	for_each_online_node(nodeid) {
		 unsigned long start_pfn, end_pfn;

799
#ifdef CONFIG_NEED_MULTIPLE_NODES
Y
Yinghai Lu 已提交
800 801 802 803 804 805
		start_pfn = node_start_pfn[nodeid];
		end_pfn = node_end_pfn[nodeid];
		if (start_pfn > max_low_pfn)
			continue;
		if (end_pfn > max_low_pfn)
			end_pfn = max_low_pfn;
806
#else
Y
Yinghai Lu 已提交
807 808
		start_pfn = 0;
		end_pfn = max_low_pfn;
809
#endif
Y
Yinghai Lu 已提交
810 811 812
		bootmap = setup_node_bootmem(nodeid, start_pfn, end_pfn,
						 bootmap);
	}
Y
Yinghai Lu 已提交
813
#endif
814

815
	after_bootmem = 1;
816 817
}

L
Linus Torvalds 已提交
818 819 820 821 822 823 824 825 826 827 828 829 830 831
/*
 * paging_init() sets up the page tables - note that the first 8MB are
 * already mapped by head.S.
 *
 * This routines also unmaps the page at virtual kernel address 0, so
 * that we can trap those pesky NULL-reference errors in the kernel.
 */
void __init paging_init(void)
{
	pagetable_init();

	__flush_tlb_all();

	kmap_init();
832 833 834 835 836 837

	/*
	 * NOTE: at this point the bootmem allocator is fully available.
	 */
	sparse_init();
	zone_sizes_init();
L
Linus Torvalds 已提交
838 839 840 841
}

/*
 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
D
Dmitri Vorobiev 已提交
842 843 844
 * and also on some strange 486's. All 586+'s are OK. This used to involve
 * black magic jumps to work around some nasty CPU bugs, but fortunately the
 * switch to using exceptions got rid of all that.
L
Linus Torvalds 已提交
845 846 847
 */
static void __init test_wp_bit(void)
{
848 849
	printk(KERN_INFO
  "Checking if this processor honours the WP bit even in supervisor mode...");
L
Linus Torvalds 已提交
850 851 852 853 854 855 856

	/* Any page-aligned address will do, the test is non-destructive */
	__set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
	boot_cpu_data.wp_works_ok = do_test_wp_bit();
	clear_fixmap(FIX_WP_TEST);

	if (!boot_cpu_data.wp_works_ok) {
857
		printk(KERN_CONT "No.\n");
L
Linus Torvalds 已提交
858
#ifdef CONFIG_X86_WP_WORKS_OK
859 860
		panic(
  "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
L
Linus Torvalds 已提交
861 862
#endif
	} else {
863
		printk(KERN_CONT "Ok.\n");
L
Linus Torvalds 已提交
864 865 866 867 868 869
	}
}

void __init mem_init(void)
{
	int codesize, reservedpages, datasize, initsize;
Y
Yinghai Lu 已提交
870
	int tmp;
L
Linus Torvalds 已提交
871

872 873
	pci_iommu_alloc();

874
#ifdef CONFIG_FLATMEM
E
Eric Sesterhenn 已提交
875
	BUG_ON(!mem_map);
L
Linus Torvalds 已提交
876 877 878 879 880 881 882
#endif
	/* this will put all low memory onto the freelists */
	totalram_pages += free_all_bootmem();

	reservedpages = 0;
	for (tmp = 0; tmp < max_low_pfn; tmp++)
		/*
I
Ingo Molnar 已提交
883
		 * Only count reserved RAM pages:
L
Linus Torvalds 已提交
884 885 886 887
		 */
		if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
			reservedpages++;

Y
Yinghai Lu 已提交
888
	set_highmem_pages_init();
L
Linus Torvalds 已提交
889 890 891 892 893

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

I
Ingo Molnar 已提交
894 895
	printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
			"%dk reserved, %dk data, %dk init, %ldk highmem)\n",
896
		nr_free_pages() << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
897 898 899 900 901
		num_physpages << (PAGE_SHIFT-10),
		codesize >> 10,
		reservedpages << (PAGE_SHIFT-10),
		datasize >> 10,
		initsize >> 10,
902
		totalhigh_pages << (PAGE_SHIFT-10));
L
Linus Torvalds 已提交
903

904
	printk(KERN_INFO "virtual kernel memory layout:\n"
I
Ingo Molnar 已提交
905
		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
906
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
907
		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n"
908
#endif
I
Ingo Molnar 已提交
909 910 911 912 913 914 915
		"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"      .init : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .data : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .text : 0x%08lx - 0x%08lx   (%4ld kB)\n",
		FIXADDR_START, FIXADDR_TOP,
		(FIXADDR_TOP - FIXADDR_START) >> 10,
916 917

#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
918 919
		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
		(LAST_PKMAP*PAGE_SIZE) >> 10,
920 921
#endif

I
Ingo Molnar 已提交
922 923
		VMALLOC_START, VMALLOC_END,
		(VMALLOC_END - VMALLOC_START) >> 20,
924

I
Ingo Molnar 已提交
925 926
		(unsigned long)__va(0), (unsigned long)high_memory,
		((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
927

I
Ingo Molnar 已提交
928 929 930
		(unsigned long)&__init_begin, (unsigned long)&__init_end,
		((unsigned long)&__init_end -
		 (unsigned long)&__init_begin) >> 10,
931

I
Ingo Molnar 已提交
932 933
		(unsigned long)&_etext, (unsigned long)&_edata,
		((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
934

I
Ingo Molnar 已提交
935 936
		(unsigned long)&_text, (unsigned long)&_etext,
		((unsigned long)&_etext - (unsigned long)&_text) >> 10);
937

938 939 940 941 942 943 944 945 946 947 948 949 950 951
	/*
	 * Check boundaries twice: Some fundamental inconsistencies can
	 * be detected at build time already.
	 */
#define __FIXADDR_TOP (-PAGE_SIZE)
#ifdef CONFIG_HIGHMEM
	BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUILD_BUG_ON(VMALLOC_END			> PKMAP_BASE);
#endif
#define high_memory (-128UL << 20)
	BUILD_BUG_ON(VMALLOC_START			>= VMALLOC_END);
#undef high_memory
#undef __FIXADDR_TOP

952
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
953 954
	BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUG_ON(VMALLOC_END				> PKMAP_BASE);
955
#endif
956
	BUG_ON(VMALLOC_START				>= VMALLOC_END);
I
Ingo Molnar 已提交
957
	BUG_ON((unsigned long)high_memory		> VMALLOC_START);
958

L
Linus Torvalds 已提交
959 960 961
	if (boot_cpu_data.wp_works_ok < 0)
		test_wp_bit();

962
	save_pg_dir();
963
	zap_low_mappings(true);
L
Linus Torvalds 已提交
964 965
}

966
#ifdef CONFIG_MEMORY_HOTPLUG
967
int arch_add_memory(int nid, u64 start, u64 size)
968
{
969
	struct pglist_data *pgdata = NODE_DATA(nid);
970
	struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
971 972 973
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

974
	return __add_pages(nid, zone, start_pfn, nr_pages);
975
}
976
#endif
977

L
Linus Torvalds 已提交
978 979 980 981
/*
 * This function cannot be __init, since exceptions don't work in that
 * section.  Put this after the callers, so that it cannot be inlined.
 */
I
Ingo Molnar 已提交
982
static noinline int do_test_wp_bit(void)
L
Linus Torvalds 已提交
983 984 985 986 987
{
	char tmp_reg;
	int flag;

	__asm__ __volatile__(
I
Ingo Molnar 已提交
988 989 990
		"	movb %0, %1	\n"
		"1:	movb %1, %0	\n"
		"	xorl %2, %2	\n"
L
Linus Torvalds 已提交
991
		"2:			\n"
992
		_ASM_EXTABLE(1b,2b)
L
Linus Torvalds 已提交
993 994 995 996 997
		:"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
		 "=q" (tmp_reg),
		 "=r" (flag)
		:"2" (1)
		:"memory");
I
Ingo Molnar 已提交
998

L
Linus Torvalds 已提交
999 1000 1001
	return flag;
}

1002
#ifdef CONFIG_DEBUG_RODATA
1003 1004
const int rodata_test_data = 0xC3;
EXPORT_SYMBOL_GPL(rodata_test_data);
1005

1006
int kernel_set_to_readonly __read_mostly;
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035

void set_kernel_text_rw(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read write\n",
		 start, start+size);

	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
}

void set_kernel_text_ro(void)
{
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read only\n",
		 start, start+size);

	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
}

1036 1037
void mark_rodata_ro(void)
{
1038 1039
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;
1040

1041 1042 1043
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
	printk(KERN_INFO "Write protecting the kernel text: %luk\n",
		size >> 10);
1044

1045 1046
	kernel_set_to_readonly = 1;

1047
#ifdef CONFIG_CPA_DEBUG
1048 1049 1050
	printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n",
		start, start+size);
	set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
1051

1052 1053
	printk(KERN_INFO "Testing CPA: write protecting again\n");
	set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
1054
#endif
1055

1056 1057
	start += size;
	size = (unsigned long)__end_rodata - start;
1058
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
1059 1060
	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
		size >> 10);
1061
	rodata_test();
1062

1063
#ifdef CONFIG_CPA_DEBUG
1064
	printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size);
1065
	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
1066

1067
	printk(KERN_INFO "Testing CPA: write protecting again\n");
1068
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
1069
#endif
1070 1071
}
#endif