init_32.c 30.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 *
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 */

#include <linux/module.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
24
#include <linux/pci.h>
25
#include <linux/pfn.h>
26
#include <linux/poison.h>
L
Linus Torvalds 已提交
27 28 29
#include <linux/bootmem.h>
#include <linux/slab.h>
#include <linux/proc_fs.h>
30
#include <linux/memory_hotplug.h>
31
#include <linux/initrd.h>
32
#include <linux/cpumask.h>
L
Linus Torvalds 已提交
33

34
#include <asm/asm.h>
35
#include <asm/bios_ebda.h>
L
Linus Torvalds 已提交
36 37 38 39 40 41 42 43
#include <asm/processor.h>
#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/pgtable.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/apic.h>
I
Ingo Molnar 已提交
44
#include <asm/bugs.h>
L
Linus Torvalds 已提交
45 46
#include <asm/tlb.h>
#include <asm/tlbflush.h>
47
#include <asm/pgalloc.h>
L
Linus Torvalds 已提交
48
#include <asm/sections.h>
49
#include <asm/paravirt.h>
50
#include <asm/setup.h>
51
#include <asm/cacheflush.h>
52
#include <asm/smp.h>
L
Linus Torvalds 已提交
53 54 55

unsigned int __VMALLOC_RESERVE = 128 << 20;

56
unsigned long max_low_pfn_mapped;
57
unsigned long max_pfn_mapped;
58

L
Linus Torvalds 已提交
59 60 61
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
unsigned long highstart_pfn, highend_pfn;

I
Ingo Molnar 已提交
62
static noinline int do_test_wp_bit(void);
L
Linus Torvalds 已提交
63

64 65 66 67 68 69 70

static unsigned long __initdata table_start;
static unsigned long __meminitdata table_end;
static unsigned long __meminitdata table_top;

static int __initdata after_init_bootmem;

71
static __init void *alloc_low_page(void)
72 73 74 75 76 77 78 79 80 81 82 83
{
	unsigned long pfn = table_end++;
	void *adr;

	if (pfn >= table_top)
		panic("alloc_low_page: ran out of memory");

	adr = __va(pfn * PAGE_SIZE);
	memset(adr, 0, PAGE_SIZE);
	return adr;
}

L
Linus Torvalds 已提交
84 85 86 87 88 89 90 91 92
/*
 * Creates a middle page table and puts a pointer to it in the
 * given global directory entry. This only returns the gd entry
 * in non-PAE compilation mode, since the middle layer is folded.
 */
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
	pud_t *pud;
	pmd_t *pmd_table;
I
Ingo Molnar 已提交
93

L
Linus Torvalds 已提交
94
#ifdef CONFIG_X86_PAE
95
	if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
96 97 98
		if (after_init_bootmem)
			pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE);
		else
99
			pmd_table = (pmd_t *)alloc_low_page();
100
		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
101 102
		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
		pud = pud_offset(pgd, 0);
I
Ingo Molnar 已提交
103
		BUG_ON(pmd_table != pmd_offset(pud, 0));
104 105

		return pmd_table;
106 107
	}
#endif
L
Linus Torvalds 已提交
108 109
	pud = pud_offset(pgd, 0);
	pmd_table = pmd_offset(pud, 0);
I
Ingo Molnar 已提交
110

L
Linus Torvalds 已提交
111 112 113 114 115
	return pmd_table;
}

/*
 * Create a page table and place a pointer to it in a middle page
I
Ingo Molnar 已提交
116
 * directory entry:
L
Linus Torvalds 已提交
117 118 119
 */
static pte_t * __init one_page_table_init(pmd_t *pmd)
{
120
	if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
121 122
		pte_t *page_table = NULL;

123
		if (after_init_bootmem) {
124
#ifdef CONFIG_DEBUG_PAGEALLOC
125
			page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
126
#endif
127 128
			if (!page_table)
				page_table =
129
				(pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
130 131
		} else
			page_table = (pte_t *)alloc_low_page();
132

133
		paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
L
Linus Torvalds 已提交
134
		set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
135
		BUG_ON(page_table != pte_offset_kernel(pmd, 0));
L
Linus Torvalds 已提交
136
	}
137

L
Linus Torvalds 已提交
138 139 140 141
	return pte_offset_kernel(pmd, 0);
}

/*
I
Ingo Molnar 已提交
142
 * This function initializes a certain range of kernel virtual memory
L
Linus Torvalds 已提交
143 144
 * with new bootmem page tables, everywhere page tables are missing in
 * the given range.
I
Ingo Molnar 已提交
145 146 147
 *
 * NOTE: The pagetables are allocated contiguous on the physical space
 * so we can cache the place of the first one and move around without
L
Linus Torvalds 已提交
148 149
 * checking the pgd every time.
 */
I
Ingo Molnar 已提交
150 151
static void __init
page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
L
Linus Torvalds 已提交
152 153 154
{
	int pgd_idx, pmd_idx;
	unsigned long vaddr;
I
Ingo Molnar 已提交
155 156
	pgd_t *pgd;
	pmd_t *pmd;
L
Linus Torvalds 已提交
157 158 159 160 161 162 163

	vaddr = start;
	pgd_idx = pgd_index(vaddr);
	pmd_idx = pmd_index(vaddr);
	pgd = pgd_base + pgd_idx;

	for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
164 165
		pmd = one_md_table_init(pgd);
		pmd = pmd + pmd_index(vaddr);
I
Ingo Molnar 已提交
166 167
		for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
							pmd++, pmd_idx++) {
168
			one_page_table_init(pmd);
L
Linus Torvalds 已提交
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183

			vaddr += PMD_SIZE;
		}
		pmd_idx = 0;
	}
}

static inline int is_kernel_text(unsigned long addr)
{
	if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
		return 1;
	return 0;
}

/*
I
Ingo Molnar 已提交
184 185 186
 * This maps the physical memory to kernel virtual address space, a total
 * of max_low_pfn pages, by creating page tables starting from address
 * PAGE_OFFSET:
L
Linus Torvalds 已提交
187
 */
188
static void __init kernel_physical_mapping_init(pgd_t *pgd_base,
189 190 191
						unsigned long start_pfn,
						unsigned long end_pfn,
						int use_pse)
L
Linus Torvalds 已提交
192
{
I
Ingo Molnar 已提交
193
	int pgd_idx, pmd_idx, pte_ofs;
L
Linus Torvalds 已提交
194 195 196 197
	unsigned long pfn;
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
	unsigned pages_2m, pages_4k;
	int mapping_iter;

	/*
	 * First iteration will setup identity mapping using large/small pages
	 * based on use_pse, with other attributes same as set by
	 * the early code in head_32.S
	 *
	 * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
	 * as desired for the kernel identity mapping.
	 *
	 * This two pass mechanism conforms to the TLB app note which says:
	 *
	 *     "Software should not write to a paging-structure entry in a way
	 *      that would change, for any linear address, both the page size
	 *      and either the page frame or attributes."
	 */
	mapping_iter = 1;
L
Linus Torvalds 已提交
216

217 218
	if (!cpu_has_pse)
		use_pse = 0;
L
Linus Torvalds 已提交
219

220 221
repeat:
	pages_2m = pages_4k = 0;
222 223 224
	pfn = start_pfn;
	pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
	pgd = pgd_base + pgd_idx;
L
Linus Torvalds 已提交
225 226
	for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
		pmd = one_md_table_init(pgd);
I
Ingo Molnar 已提交
227

228 229 230 231 232 233 234 235 236
		if (pfn >= end_pfn)
			continue;
#ifdef CONFIG_X86_PAE
		pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
		pmd += pmd_idx;
#else
		pmd_idx = 0;
#endif
		for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
J
Jeremy Fitzhardinge 已提交
237
		     pmd++, pmd_idx++) {
I
Ingo Molnar 已提交
238
			unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
L
Linus Torvalds 已提交
239

I
Ingo Molnar 已提交
240 241 242 243
			/*
			 * Map with big pages if possible, otherwise
			 * create normal page tables:
			 */
244
			if (use_pse) {
I
Ingo Molnar 已提交
245
				unsigned int addr2;
J
Jeremy Fitzhardinge 已提交
246
				pgprot_t prot = PAGE_KERNEL_LARGE;
247 248 249 250 251 252 253
				/*
				 * first pass will use the same initial
				 * identity mapping attribute + _PAGE_PSE.
				 */
				pgprot_t init_prot =
					__pgprot(PTE_IDENT_ATTR |
						 _PAGE_PSE);
J
Jeremy Fitzhardinge 已提交
254

I
Ingo Molnar 已提交
255
				addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
J
Jeremy Fitzhardinge 已提交
256 257
					PAGE_OFFSET + PAGE_SIZE-1;

I
Ingo Molnar 已提交
258 259
				if (is_kernel_text(addr) ||
				    is_kernel_text(addr2))
J
Jeremy Fitzhardinge 已提交
260 261
					prot = PAGE_KERNEL_LARGE_EXEC;

262
				pages_2m++;
263 264 265 266
				if (mapping_iter == 1)
					set_pmd(pmd, pfn_pmd(pfn, init_prot));
				else
					set_pmd(pmd, pfn_pmd(pfn, prot));
267

L
Linus Torvalds 已提交
268
				pfn += PTRS_PER_PTE;
I
Ingo Molnar 已提交
269 270 271
				continue;
			}
			pte = one_page_table_init(pmd);
L
Linus Torvalds 已提交
272

273 274 275
			pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
			pte += pte_ofs;
			for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
I
Ingo Molnar 已提交
276 277
			     pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
				pgprot_t prot = PAGE_KERNEL;
278 279 280 281 282
				/*
				 * first pass will use the same initial
				 * identity mapping attribute.
				 */
				pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
J
Jeremy Fitzhardinge 已提交
283

I
Ingo Molnar 已提交
284 285
				if (is_kernel_text(addr))
					prot = PAGE_KERNEL_EXEC;
J
Jeremy Fitzhardinge 已提交
286

287
				pages_4k++;
288 289 290 291
				if (mapping_iter == 1)
					set_pte(pte, pfn_pte(pfn, init_prot));
				else
					set_pte(pte, pfn_pte(pfn, prot));
L
Linus Torvalds 已提交
292 293 294
			}
		}
	}
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
	if (mapping_iter == 1) {
		/*
		 * update direct mapping page count only in the first
		 * iteration.
		 */
		update_page_count(PG_LEVEL_2M, pages_2m);
		update_page_count(PG_LEVEL_4K, pages_4k);

		/*
		 * local global flush tlb, which will flush the previous
		 * mappings present in both small and large page TLB's.
		 */
		__flush_tlb_all();

		/*
		 * Second iteration will set the actual desired PTE attributes.
		 */
		mapping_iter = 2;
		goto repeat;
	}
L
Linus Torvalds 已提交
315 316
}

317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
/*
 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
 * is valid. The argument is a physical page number.
 *
 *
 * On x86, access has to be given to the first megabyte of ram because that area
 * contains bios code and data regions used by X and dosemu and similar apps.
 * Access has to be given to non-kernel-ram areas as well, these contain the PCI
 * mmio resources as well as potential bios/acpi data regions.
 */
int devmem_is_allowed(unsigned long pagenr)
{
	if (pagenr <= 256)
		return 1;
	if (!page_is_ram(pagenr))
		return 1;
	return 0;
}

L
Linus Torvalds 已提交
336 337 338
pte_t *kmap_pte;
pgprot_t kmap_prot;

I
Ingo Molnar 已提交
339 340 341 342 343
static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
{
	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
			vaddr), vaddr), vaddr);
}
L
Linus Torvalds 已提交
344 345 346 347 348

static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

I
Ingo Molnar 已提交
349 350 351
	/*
	 * Cache the first kmap pte:
	 */
L
Linus Torvalds 已提交
352 353 354 355 356 357
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);

	kmap_prot = PAGE_KERNEL;
}

358
#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
359 360
static void __init permanent_kmaps_init(pgd_t *pgd_base)
{
I
Ingo Molnar 已提交
361
	unsigned long vaddr;
L
Linus Torvalds 已提交
362 363 364 365 366 367 368 369 370 371 372 373
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	vaddr = PKMAP_BASE;
	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);

	pgd = swapper_pg_dir + pgd_index(vaddr);
	pud = pud_offset(pgd, vaddr);
	pmd = pmd_offset(pud, vaddr);
	pte = pte_offset_kernel(pmd, vaddr);
I
Ingo Molnar 已提交
374
	pkmap_page_table = pte;
L
Linus Torvalds 已提交
375 376
}

Y
Yinghai Lu 已提交
377
static void __init add_one_highpage_init(struct page *page, int pfn)
L
Linus Torvalds 已提交
378
{
Y
Yinghai Lu 已提交
379 380 381 382
	ClearPageReserved(page);
	init_page_count(page);
	__free_page(page);
	totalhigh_pages++;
L
Linus Torvalds 已提交
383 384
}

385 386 387 388 389
struct add_highpages_data {
	unsigned long start_pfn;
	unsigned long end_pfn;
};

390
static int __init add_highpages_work_fn(unsigned long start_pfn,
391
					 unsigned long end_pfn, void *datax)
L
Linus Torvalds 已提交
392
{
393 394 395 396
	int node_pfn;
	struct page *page;
	unsigned long final_start_pfn, final_end_pfn;
	struct add_highpages_data *data;
I
Ingo Molnar 已提交
397

398 399 400 401 402
	data = (struct add_highpages_data *)datax;

	final_start_pfn = max(start_pfn, data->start_pfn);
	final_end_pfn = min(end_pfn, data->end_pfn);
	if (final_start_pfn >= final_end_pfn)
403
		return 0;
404 405 406 407 408 409

	for (node_pfn = final_start_pfn; node_pfn < final_end_pfn;
	     node_pfn++) {
		if (!pfn_valid(node_pfn))
			continue;
		page = pfn_to_page(node_pfn);
Y
Yinghai Lu 已提交
410
		add_one_highpage_init(page, node_pfn);
411
	}
412

413 414
	return 0;

415 416 417
}

void __init add_highpages_with_active_regions(int nid, unsigned long start_pfn,
Y
Yinghai Lu 已提交
418
					      unsigned long end_pfn)
419 420 421 422 423 424 425 426 427
{
	struct add_highpages_data data;

	data.start_pfn = start_pfn;
	data.end_pfn = end_pfn;

	work_with_active_regions(nid, add_highpages_work_fn, &data);
}

I
Ingo Molnar 已提交
428
#ifndef CONFIG_NUMA
Y
Yinghai Lu 已提交
429
static void __init set_highmem_pages_init(void)
L
Linus Torvalds 已提交
430
{
Y
Yinghai Lu 已提交
431
	add_highpages_with_active_regions(0, highstart_pfn, highend_pfn);
432

L
Linus Torvalds 已提交
433 434
	totalram_pages += totalhigh_pages;
}
I
Ingo Molnar 已提交
435
#endif /* !CONFIG_NUMA */
L
Linus Torvalds 已提交
436 437

#else
438 439 440 441 442 443
static inline void permanent_kmaps_init(pgd_t *pgd_base)
{
}
static inline void set_highmem_pages_init(void)
{
}
L
Linus Torvalds 已提交
444 445
#endif /* CONFIG_HIGHMEM */

446
void __init native_pagetable_setup_start(pgd_t *base)
L
Linus Torvalds 已提交
447
{
448 449 450 451 452
	unsigned long pfn, va;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
453 454

	/*
455 456
	 * Remove any mappings which extend past the end of physical
	 * memory from the boot time page table:
457
	 */
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
	for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
		va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
		pgd = base + pgd_index(va);
		if (!pgd_present(*pgd))
			break;

		pud = pud_offset(pgd, va);
		pmd = pmd_offset(pud, va);
		if (!pmd_present(*pmd))
			break;

		pte = pte_offset_kernel(pmd, va);
		if (!pte_present(*pte))
			break;

		pte_clear(NULL, va, pte);
	}
475
	paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
476 477 478 479 480 481 482 483 484 485 486 487
}

void __init native_pagetable_setup_done(pgd_t *base)
{
}

/*
 * Build a proper pagetable for the kernel mappings.  Up until this
 * point, we've been running on some set of pagetables constructed by
 * the boot process.
 *
 * If we're booting on native hardware, this will be a pagetable
488 489
 * constructed in arch/x86/kernel/head_32.S.  The root of the
 * pagetable will be swapper_pg_dir.
490 491 492 493 494 495 496 497 498 499 500
 *
 * If we're booting paravirtualized under a hypervisor, then there are
 * more options: we may already be running PAE, and the pagetable may
 * or may not be based in swapper_pg_dir.  In any case,
 * paravirt_pagetable_setup_start() will set up swapper_pg_dir
 * appropriately for the rest of the initialization to work.
 *
 * In general, pagetable_init() assumes that the pagetable may already
 * be partially populated, and so it avoids stomping on any existing
 * mappings.
 */
501
static void __init early_ioremap_page_table_range_init(pgd_t *pgd_base)
502
{
I
Ingo Molnar 已提交
503
	unsigned long vaddr, end;
504

L
Linus Torvalds 已提交
505 506 507 508
	/*
	 * Fixed mappings, only the page table structure has to be
	 * created - mappings will be set by set_fixmap():
	 */
509
	early_ioremap_clear();
L
Linus Torvalds 已提交
510
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
511 512
	end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
	page_table_range_init(vaddr, end, pgd_base);
513
	early_ioremap_reset();
514 515 516 517 518 519
}

static void __init pagetable_init(void)
{
	pgd_t *pgd_base = swapper_pg_dir;

L
Linus Torvalds 已提交
520 521 522
	permanent_kmaps_init(pgd_base);
}

523
#ifdef CONFIG_ACPI_SLEEP
L
Linus Torvalds 已提交
524
/*
525
 * ACPI suspend needs this for resume, because things like the intel-agp
L
Linus Torvalds 已提交
526 527
 * driver might have split up a kernel 4MB mapping.
 */
528
char swsusp_pg_dir[PAGE_SIZE]
I
Ingo Molnar 已提交
529
	__attribute__ ((aligned(PAGE_SIZE)));
L
Linus Torvalds 已提交
530 531 532 533 534

static inline void save_pg_dir(void)
{
	memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
}
535
#else /* !CONFIG_ACPI_SLEEP */
L
Linus Torvalds 已提交
536 537 538
static inline void save_pg_dir(void)
{
}
539
#endif /* !CONFIG_ACPI_SLEEP */
L
Linus Torvalds 已提交
540

I
Ingo Molnar 已提交
541
void zap_low_mappings(void)
L
Linus Torvalds 已提交
542 543 544 545 546 547 548 549 550
{
	int i;

	/*
	 * Zap initial low-memory mappings.
	 *
	 * Note that "pgd_clear()" doesn't do it for
	 * us, because pgd_clear() is a no-op on i386.
	 */
J
Jeremy Fitzhardinge 已提交
551
	for (i = 0; i < KERNEL_PGD_BOUNDARY; i++) {
L
Linus Torvalds 已提交
552 553 554 555 556
#ifdef CONFIG_X86_PAE
		set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
#else
		set_pgd(swapper_pg_dir+i, __pgd(0));
#endif
I
Ingo Molnar 已提交
557
	}
L
Linus Torvalds 已提交
558 559 560
	flush_tlb_all();
}

I
Ingo Molnar 已提交
561
int nx_enabled;
J
Jan Beulich 已提交
562

563
pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL | _PAGE_IOMAP);
564 565
EXPORT_SYMBOL_GPL(__supported_pte_mask);

J
Jan Beulich 已提交
566 567
#ifdef CONFIG_X86_PAE

I
Ingo Molnar 已提交
568
static int disable_nx __initdata;
L
Linus Torvalds 已提交
569 570 571 572 573 574 575 576 577

/*
 * noexec = on|off
 *
 * Control non executable mappings.
 *
 * on      Enable
 * off     Disable
 */
578
static int __init noexec_setup(char *str)
L
Linus Torvalds 已提交
579
{
580 581 582 583 584
	if (!str || !strcmp(str, "on")) {
		if (cpu_has_nx) {
			__supported_pte_mask |= _PAGE_NX;
			disable_nx = 0;
		}
I
Ingo Molnar 已提交
585 586 587 588 589 590 591 592
	} else {
		if (!strcmp(str, "off")) {
			disable_nx = 1;
			__supported_pte_mask &= ~_PAGE_NX;
		} else {
			return -EINVAL;
		}
	}
593 594

	return 0;
L
Linus Torvalds 已提交
595
}
596
early_param("noexec", noexec_setup);
L
Linus Torvalds 已提交
597 598 599 600 601 602 603

static void __init set_nx(void)
{
	unsigned int v[4], l, h;

	if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
		cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
I
Ingo Molnar 已提交
604

L
Linus Torvalds 已提交
605 606 607 608 609 610 611 612 613 614 615
		if ((v[3] & (1 << 20)) && !disable_nx) {
			rdmsr(MSR_EFER, l, h);
			l |= EFER_NX;
			wrmsr(MSR_EFER, l, h);
			nx_enabled = 1;
			__supported_pte_mask |= _PAGE_NX;
		}
	}
}
#endif

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
/* user-defined highmem size */
static unsigned int highmem_pages = -1;

/*
 * highmem=size forces highmem to be exactly 'size' bytes.
 * This works even on boxes that have no highmem otherwise.
 * This also works to reduce highmem size on bigger boxes.
 */
static int __init parse_highmem(char *arg)
{
	if (!arg)
		return -EINVAL;

	highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
	return 0;
}
early_param("highmem", parse_highmem);

/*
 * Determine low and high memory ranges:
 */
637
void __init find_low_pfn_range(void)
638
{
639 640
	/* it could update max_pfn */

Y
Yinghai Lu 已提交
641
	/* max_low_pfn is 0, we already have early_res support */
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704

	max_low_pfn = max_pfn;
	if (max_low_pfn > MAXMEM_PFN) {
		if (highmem_pages == -1)
			highmem_pages = max_pfn - MAXMEM_PFN;
		if (highmem_pages + MAXMEM_PFN < max_pfn)
			max_pfn = MAXMEM_PFN + highmem_pages;
		if (highmem_pages + MAXMEM_PFN > max_pfn) {
			printk(KERN_WARNING "only %luMB highmem pages "
				"available, ignoring highmem size of %uMB.\n",
				pages_to_mb(max_pfn - MAXMEM_PFN),
				pages_to_mb(highmem_pages));
			highmem_pages = 0;
		}
		max_low_pfn = MAXMEM_PFN;
#ifndef CONFIG_HIGHMEM
		/* Maximum memory usable is what is directly addressable */
		printk(KERN_WARNING "Warning only %ldMB will be used.\n",
					MAXMEM>>20);
		if (max_pfn > MAX_NONPAE_PFN)
			printk(KERN_WARNING
				 "Use a HIGHMEM64G enabled kernel.\n");
		else
			printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
		max_pfn = MAXMEM_PFN;
#else /* !CONFIG_HIGHMEM */
#ifndef CONFIG_HIGHMEM64G
		if (max_pfn > MAX_NONPAE_PFN) {
			max_pfn = MAX_NONPAE_PFN;
			printk(KERN_WARNING "Warning only 4GB will be used."
				"Use a HIGHMEM64G enabled kernel.\n");
		}
#endif /* !CONFIG_HIGHMEM64G */
#endif /* !CONFIG_HIGHMEM */
	} else {
		if (highmem_pages == -1)
			highmem_pages = 0;
#ifdef CONFIG_HIGHMEM
		if (highmem_pages >= max_pfn) {
			printk(KERN_ERR "highmem size specified (%uMB) is "
				"bigger than pages available (%luMB)!.\n",
				pages_to_mb(highmem_pages),
				pages_to_mb(max_pfn));
			highmem_pages = 0;
		}
		if (highmem_pages) {
			if (max_low_pfn - highmem_pages <
			    64*1024*1024/PAGE_SIZE){
				printk(KERN_ERR "highmem size %uMB results in "
				"smaller than 64MB lowmem, ignoring it.\n"
					, pages_to_mb(highmem_pages));
				highmem_pages = 0;
			}
			max_low_pfn -= highmem_pages;
		}
#else
		if (highmem_pages)
			printk(KERN_ERR "ignoring highmem size on non-highmem"
					" kernel!\n");
#endif
	}
}

705
#ifndef CONFIG_NEED_MULTIPLE_NODES
706
void __init initmem_init(unsigned long start_pfn,
707 708 709 710 711 712 713
				  unsigned long end_pfn)
{
#ifdef CONFIG_HIGHMEM
	highstart_pfn = highend_pfn = max_pfn;
	if (max_pfn > max_low_pfn)
		highstart_pfn = max_low_pfn;
	memory_present(0, 0, highend_pfn);
714
	e820_register_active_regions(0, 0, highend_pfn);
715 716 717 718 719 720
	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
		pages_to_mb(highend_pfn - highstart_pfn));
	num_physpages = highend_pfn;
	high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
#else
	memory_present(0, 0, max_low_pfn);
721
	e820_register_active_regions(0, 0, max_low_pfn);
722 723 724 725 726 727 728 729 730 731 732
	num_physpages = max_low_pfn;
	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
#endif
#ifdef CONFIG_FLATMEM
	max_mapnr = num_physpages;
#endif
	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
			pages_to_mb(max_low_pfn));

	setup_bootmem_allocator();
}
733
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
734

735
static void __init zone_sizes_init(void)
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
{
	unsigned long max_zone_pfns[MAX_NR_ZONES];
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
	max_zone_pfns[ZONE_DMA] =
		virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
#ifdef CONFIG_HIGHMEM
	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
#endif

	free_area_init_nodes(max_zone_pfns);
}

void __init setup_bootmem_allocator(void)
{
	int i;
	unsigned long bootmap_size, bootmap;
	/*
	 * Initialize the boot-time allocator (with low memory only):
	 */
	bootmap_size = bootmem_bootmap_pages(max_low_pfn)<<PAGE_SHIFT;
	bootmap = find_e820_area(min_low_pfn<<PAGE_SHIFT,
				 max_pfn_mapped<<PAGE_SHIFT, bootmap_size,
				 PAGE_SIZE);
	if (bootmap == -1L)
		panic("Cannot find bootmem map of size %ld\n", bootmap_size);
	reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP");
Y
Yinghai Lu 已提交
763

Y
Yinghai Lu 已提交
764 765 766
	/* don't touch min_low_pfn */
	bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT,
					 min_low_pfn, max_low_pfn);
767 768 769 770 771 772 773 774 775 776
	printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
		 max_pfn_mapped<<PAGE_SHIFT);
	printk(KERN_INFO "  low ram: %08lx - %08lx\n",
		 min_low_pfn<<PAGE_SHIFT, max_low_pfn<<PAGE_SHIFT);
	printk(KERN_INFO "  bootmap %08lx - %08lx\n",
		 bootmap, bootmap + bootmap_size);
	for_each_online_node(i)
		free_bootmem_with_active_regions(i, max_low_pfn);
	early_res_to_bootmem(0, max_low_pfn<<PAGE_SHIFT);

777
	after_init_bootmem = 1;
778 779
}

780
static void __init find_early_table_space(unsigned long end, int use_pse)
781
{
782
	unsigned long puds, pmds, ptes, tables, start;
783 784 785 786 787 788 789

	puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
	tables = PAGE_ALIGN(puds * sizeof(pud_t));

	pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
	tables += PAGE_ALIGN(pmds * sizeof(pmd_t));

790
	if (use_pse) {
791
		unsigned long extra;
792 793 794

		extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
		extra += PMD_SIZE;
795 796 797 798 799
		ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
	} else
		ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;

	tables += PAGE_ALIGN(ptes * sizeof(pte_t));
800

801 802 803
	/* for fixmap */
	tables += PAGE_SIZE * 2;

804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
	/*
	 * RED-PEN putting page tables only on node 0 could
	 * cause a hotspot and fill up ZONE_DMA. The page tables
	 * need roughly 0.5KB per GB.
	 */
	start = 0x7000;
	table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
					tables, PAGE_SIZE);
	if (table_start == -1UL)
		panic("Cannot find space for the kernel page tables");

	table_start >>= PAGE_SHIFT;
	table_end = table_start;
	table_top = table_start + (tables>>PAGE_SHIFT);

	printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
		end, table_start << PAGE_SHIFT,
		(table_start << PAGE_SHIFT) + tables);
}

unsigned long __init_refok init_memory_mapping(unsigned long start,
						unsigned long end)
{
	pgd_t *pgd_base = swapper_pg_dir;
828 829
	unsigned long start_pfn, end_pfn;
	unsigned long big_page_start;
830 831 832 833 834 835 836 837 838 839
#ifdef CONFIG_DEBUG_PAGEALLOC
	/*
	 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
	 * This will simplify cpa(), which otherwise needs to support splitting
	 * large pages into small in interrupt context, etc.
	 */
	int use_pse = 0;
#else
	int use_pse = cpu_has_pse;
#endif
840 841 842 843 844

	/*
	 * Find space for the kernel direct mapping tables.
	 */
	if (!after_init_bootmem)
845
		find_early_table_space(end, use_pse);
846 847 848 849 850 851 852 853 854 855 856 857 858 859

#ifdef CONFIG_X86_PAE
	set_nx();
	if (nx_enabled)
		printk(KERN_INFO "NX (Execute Disable) protection: active\n");
#endif

	/* Enable PSE if available */
	if (cpu_has_pse)
		set_in_cr4(X86_CR4_PSE);

	/* Enable PGE if available */
	if (cpu_has_pge) {
		set_in_cr4(X86_CR4_PGE);
860
		__supported_pte_mask |= _PAGE_GLOBAL;
861 862
	}

863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
	/*
	 * Don't use a large page for the first 2/4MB of memory
	 * because there are often fixed size MTRRs in there
	 * and overlapping MTRRs into large pages can cause
	 * slowdowns.
	 */
	big_page_start = PMD_SIZE;

	if (start < big_page_start) {
		start_pfn = start >> PAGE_SHIFT;
		end_pfn = min(big_page_start>>PAGE_SHIFT, end>>PAGE_SHIFT);
	} else {
		/* head is not big page alignment ? */
		start_pfn = start >> PAGE_SHIFT;
		end_pfn = ((start + (PMD_SIZE - 1))>>PMD_SHIFT)
				 << (PMD_SHIFT - PAGE_SHIFT);
	}
	if (start_pfn < end_pfn)
		kernel_physical_mapping_init(pgd_base, start_pfn, end_pfn, 0);

	/* big page range */
	start_pfn = ((start + (PMD_SIZE - 1))>>PMD_SHIFT)
			 << (PMD_SHIFT - PAGE_SHIFT);
	if (start_pfn < (big_page_start >> PAGE_SHIFT))
		start_pfn =  big_page_start >> PAGE_SHIFT;
	end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
	if (start_pfn < end_pfn)
		kernel_physical_mapping_init(pgd_base, start_pfn, end_pfn,
891
					     use_pse);
892 893 894 895 896 897 898 899 900

	/* tail is not big page alignment ? */
	start_pfn = end_pfn;
	if (start_pfn > (big_page_start>>PAGE_SHIFT)) {
		end_pfn = end >> PAGE_SHIFT;
		if (start_pfn < end_pfn)
			kernel_physical_mapping_init(pgd_base, start_pfn,
							 end_pfn, 0);
	}
901

902 903
	early_ioremap_page_table_range_init(pgd_base);

904 905 906 907 908 909 910 911
	load_cr3(swapper_pg_dir);

	__flush_tlb_all();

	if (!after_init_bootmem)
		reserve_early(table_start << PAGE_SHIFT,
				 table_end << PAGE_SHIFT, "PGTABLE");

912 913 914
	if (!after_init_bootmem)
		early_memtest(start, end);

915 916 917
	return end >> PAGE_SHIFT;
}

918

L
Linus Torvalds 已提交
919 920 921 922 923 924 925 926 927 928 929 930 931 932
/*
 * paging_init() sets up the page tables - note that the first 8MB are
 * already mapped by head.S.
 *
 * This routines also unmaps the page at virtual kernel address 0, so
 * that we can trap those pesky NULL-reference errors in the kernel.
 */
void __init paging_init(void)
{
	pagetable_init();

	__flush_tlb_all();

	kmap_init();
933 934 935 936 937 938

	/*
	 * NOTE: at this point the bootmem allocator is fully available.
	 */
	sparse_init();
	zone_sizes_init();
L
Linus Torvalds 已提交
939 940 941 942
}

/*
 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
D
Dmitri Vorobiev 已提交
943 944 945
 * and also on some strange 486's. All 586+'s are OK. This used to involve
 * black magic jumps to work around some nasty CPU bugs, but fortunately the
 * switch to using exceptions got rid of all that.
L
Linus Torvalds 已提交
946 947 948
 */
static void __init test_wp_bit(void)
{
949 950
	printk(KERN_INFO
  "Checking if this processor honours the WP bit even in supervisor mode...");
L
Linus Torvalds 已提交
951 952 953 954 955 956 957

	/* Any page-aligned address will do, the test is non-destructive */
	__set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
	boot_cpu_data.wp_works_ok = do_test_wp_bit();
	clear_fixmap(FIX_WP_TEST);

	if (!boot_cpu_data.wp_works_ok) {
958
		printk(KERN_CONT "No.\n");
L
Linus Torvalds 已提交
959
#ifdef CONFIG_X86_WP_WORKS_OK
960 961
		panic(
  "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
L
Linus Torvalds 已提交
962 963
#endif
	} else {
964
		printk(KERN_CONT "Ok.\n");
L
Linus Torvalds 已提交
965 966 967
	}
}

I
Ingo Molnar 已提交
968
static struct kcore_list kcore_mem, kcore_vmalloc;
L
Linus Torvalds 已提交
969 970 971 972

void __init mem_init(void)
{
	int codesize, reservedpages, datasize, initsize;
Y
Yinghai Lu 已提交
973
	int tmp;
L
Linus Torvalds 已提交
974

975 976
	pci_iommu_alloc();

977
#ifdef CONFIG_FLATMEM
E
Eric Sesterhenn 已提交
978
	BUG_ON(!mem_map);
L
Linus Torvalds 已提交
979 980 981 982 983 984 985
#endif
	/* this will put all low memory onto the freelists */
	totalram_pages += free_all_bootmem();

	reservedpages = 0;
	for (tmp = 0; tmp < max_low_pfn; tmp++)
		/*
I
Ingo Molnar 已提交
986
		 * Only count reserved RAM pages:
L
Linus Torvalds 已提交
987 988 989 990
		 */
		if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
			reservedpages++;

Y
Yinghai Lu 已提交
991
	set_highmem_pages_init();
L
Linus Torvalds 已提交
992 993 994 995 996

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

I
Ingo Molnar 已提交
997 998
	kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
	kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
L
Linus Torvalds 已提交
999 1000
		   VMALLOC_END-VMALLOC_START);

I
Ingo Molnar 已提交
1001 1002
	printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
			"%dk reserved, %dk data, %dk init, %ldk highmem)\n",
L
Linus Torvalds 已提交
1003 1004 1005 1006 1007 1008 1009 1010 1011
		(unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
		num_physpages << (PAGE_SHIFT-10),
		codesize >> 10,
		reservedpages << (PAGE_SHIFT-10),
		datasize >> 10,
		initsize >> 10,
		(unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))
	       );

1012
	printk(KERN_INFO "virtual kernel memory layout:\n"
I
Ingo Molnar 已提交
1013
		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
1014
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
1015
		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n"
1016
#endif
I
Ingo Molnar 已提交
1017 1018 1019 1020 1021 1022 1023
		"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
		"      .init : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .data : 0x%08lx - 0x%08lx   (%4ld kB)\n"
		"      .text : 0x%08lx - 0x%08lx   (%4ld kB)\n",
		FIXADDR_START, FIXADDR_TOP,
		(FIXADDR_TOP - FIXADDR_START) >> 10,
1024 1025

#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
1026 1027
		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
		(LAST_PKMAP*PAGE_SIZE) >> 10,
1028 1029
#endif

I
Ingo Molnar 已提交
1030 1031
		VMALLOC_START, VMALLOC_END,
		(VMALLOC_END - VMALLOC_START) >> 20,
1032

I
Ingo Molnar 已提交
1033 1034
		(unsigned long)__va(0), (unsigned long)high_memory,
		((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
1035

I
Ingo Molnar 已提交
1036 1037 1038
		(unsigned long)&__init_begin, (unsigned long)&__init_end,
		((unsigned long)&__init_end -
		 (unsigned long)&__init_begin) >> 10,
1039

I
Ingo Molnar 已提交
1040 1041
		(unsigned long)&_etext, (unsigned long)&_edata,
		((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
1042

I
Ingo Molnar 已提交
1043 1044
		(unsigned long)&_text, (unsigned long)&_etext,
		((unsigned long)&_etext - (unsigned long)&_text) >> 10);
1045

1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	/*
	 * Check boundaries twice: Some fundamental inconsistencies can
	 * be detected at build time already.
	 */
#define __FIXADDR_TOP (-PAGE_SIZE)
#ifdef CONFIG_HIGHMEM
	BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUILD_BUG_ON(VMALLOC_END			> PKMAP_BASE);
#endif
#define high_memory (-128UL << 20)
	BUILD_BUG_ON(VMALLOC_START			>= VMALLOC_END);
#undef high_memory
#undef __FIXADDR_TOP

1060
#ifdef CONFIG_HIGHMEM
I
Ingo Molnar 已提交
1061 1062
	BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE	> FIXADDR_START);
	BUG_ON(VMALLOC_END				> PKMAP_BASE);
1063
#endif
1064
	BUG_ON(VMALLOC_START				>= VMALLOC_END);
I
Ingo Molnar 已提交
1065
	BUG_ON((unsigned long)high_memory		> VMALLOC_START);
1066

L
Linus Torvalds 已提交
1067 1068 1069
	if (boot_cpu_data.wp_works_ok < 0)
		test_wp_bit();

1070
	save_pg_dir();
L
Linus Torvalds 已提交
1071 1072 1073
	zap_low_mappings();
}

1074
#ifdef CONFIG_MEMORY_HOTPLUG
1075
int arch_add_memory(int nid, u64 start, u64 size)
1076
{
1077
	struct pglist_data *pgdata = NODE_DATA(nid);
1078
	struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
1079 1080 1081
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

1082
	return __add_pages(nid, zone, start_pfn, nr_pages);
1083
}
1084
#endif
1085

L
Linus Torvalds 已提交
1086 1087 1088 1089
/*
 * This function cannot be __init, since exceptions don't work in that
 * section.  Put this after the callers, so that it cannot be inlined.
 */
I
Ingo Molnar 已提交
1090
static noinline int do_test_wp_bit(void)
L
Linus Torvalds 已提交
1091 1092 1093 1094 1095
{
	char tmp_reg;
	int flag;

	__asm__ __volatile__(
I
Ingo Molnar 已提交
1096 1097 1098
		"	movb %0, %1	\n"
		"1:	movb %1, %0	\n"
		"	xorl %2, %2	\n"
L
Linus Torvalds 已提交
1099
		"2:			\n"
1100
		_ASM_EXTABLE(1b,2b)
L
Linus Torvalds 已提交
1101 1102 1103 1104 1105
		:"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
		 "=q" (tmp_reg),
		 "=r" (flag)
		:"2" (1)
		:"memory");
I
Ingo Molnar 已提交
1106

L
Linus Torvalds 已提交
1107 1108 1109
	return flag;
}

1110
#ifdef CONFIG_DEBUG_RODATA
1111 1112
const int rodata_test_data = 0xC3;
EXPORT_SYMBOL_GPL(rodata_test_data);
1113 1114 1115

void mark_rodata_ro(void)
{
1116 1117
	unsigned long start = PFN_ALIGN(_text);
	unsigned long size = PFN_ALIGN(_etext) - start;
1118

1119 1120
#ifndef CONFIG_DYNAMIC_FTRACE
	/* Dynamic tracing modifies the kernel text section */
1121 1122 1123
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
	printk(KERN_INFO "Write protecting the kernel text: %luk\n",
		size >> 10);
1124 1125

#ifdef CONFIG_CPA_DEBUG
1126 1127 1128
	printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n",
		start, start+size);
	set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
1129

1130 1131
	printk(KERN_INFO "Testing CPA: write protecting again\n");
	set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
1132
#endif
1133 1134
#endif /* CONFIG_DYNAMIC_FTRACE */

1135 1136
	start += size;
	size = (unsigned long)__end_rodata - start;
1137
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
1138 1139
	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
		size >> 10);
1140
	rodata_test();
1141

1142
#ifdef CONFIG_CPA_DEBUG
1143
	printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size);
1144
	set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
1145

1146
	printk(KERN_INFO "Testing CPA: write protecting again\n");
1147
	set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
1148
#endif
1149 1150 1151
}
#endif

G
Gerd Hoffmann 已提交
1152 1153
void free_init_pages(char *what, unsigned long begin, unsigned long end)
{
I
Ingo Molnar 已提交
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
#ifdef CONFIG_DEBUG_PAGEALLOC
	/*
	 * If debugging page accesses then do not free this memory but
	 * mark them not present - any buggy init-section access will
	 * create a kernel page fault:
	 */
	printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n",
		begin, PAGE_ALIGN(end));
	set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
#else
I
Ingo Molnar 已提交
1164 1165
	unsigned long addr;

1166 1167 1168 1169 1170 1171 1172
	/*
	 * We just marked the kernel text read only above, now that
	 * we are going to free part of that, we need to make that
	 * writeable first.
	 */
	set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);

G
Gerd Hoffmann 已提交
1173
	for (addr = begin; addr < end; addr += PAGE_SIZE) {
1174 1175 1176 1177
		ClearPageReserved(virt_to_page(addr));
		init_page_count(virt_to_page(addr));
		memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
		free_page(addr);
G
Gerd Hoffmann 已提交
1178 1179
		totalram_pages++;
	}
1180
	printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
I
Ingo Molnar 已提交
1181
#endif
G
Gerd Hoffmann 已提交
1182 1183 1184 1185 1186
}

void free_initmem(void)
{
	free_init_pages("unused kernel memory",
1187 1188
			(unsigned long)(&__init_begin),
			(unsigned long)(&__init_end));
G
Gerd Hoffmann 已提交
1189
}
1190

L
Linus Torvalds 已提交
1191 1192 1193
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
1194
	free_init_pages("initrd memory", start, end);
L
Linus Torvalds 已提交
1195 1196
}
#endif
1197 1198 1199 1200 1201 1202

int __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
				   int flags)
{
	return reserve_bootmem(phys, len, flags);
}