init_64.c 25.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 *  linux/arch/x86_64/mm/init.c
 *
 *  Copyright (C) 1995  Linus Torvalds
P
Pavel Machek 已提交
5
 *  Copyright (C) 2000  Pavel Machek <pavel@ucw.cz>
L
Linus Torvalds 已提交
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 *  Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
 */

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
T
Thomas Gleixner 已提交
21
#include <linux/initrd.h>
L
Linus Torvalds 已提交
22 23
#include <linux/pagemap.h>
#include <linux/bootmem.h>
24
#include <linux/memblock.h>
L
Linus Torvalds 已提交
25
#include <linux/proc_fs.h>
26
#include <linux/pci.h>
27
#include <linux/pfn.h>
28
#include <linux/poison.h>
29
#include <linux/dma-mapping.h>
30
#include <linux/module.h>
31
#include <linux/memory.h>
32
#include <linux/memory_hotplug.h>
33
#include <linux/nmi.h>
34
#include <linux/gfp.h>
L
Linus Torvalds 已提交
35 36

#include <asm/processor.h>
37
#include <asm/bios_ebda.h>
L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46 47 48
#include <asm/uaccess.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/apic.h>
#include <asm/tlb.h>
#include <asm/mmu_context.h>
#include <asm/proto.h>
#include <asm/smp.h>
49
#include <asm/sections.h>
50
#include <asm/kdebug.h>
51
#include <asm/numa.h>
52
#include <asm/cacheflush.h>
53
#include <asm/init.h>
54
#include <asm/uv/uv.h>
55
#include <asm/setup.h>
L
Linus Torvalds 已提交
56

57 58
#include "mm_internal.h"

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
static void ident_pmd_init(unsigned long pmd_flag, pmd_t *pmd_page,
			   unsigned long addr, unsigned long end)
{
	addr &= PMD_MASK;
	for (; addr < end; addr += PMD_SIZE) {
		pmd_t *pmd = pmd_page + pmd_index(addr);

		if (!pmd_present(*pmd))
			set_pmd(pmd, __pmd(addr | pmd_flag));
	}
}
static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
			  unsigned long addr, unsigned long end)
{
	unsigned long next;

	for (; addr < end; addr = next) {
		pud_t *pud = pud_page + pud_index(addr);
		pmd_t *pmd;

		next = (addr & PUD_MASK) + PUD_SIZE;
		if (next > end)
			next = end;

		if (pud_present(*pud)) {
			pmd = pmd_offset(pud, 0);
			ident_pmd_init(info->pmd_flag, pmd, addr, next);
			continue;
		}
		pmd = (pmd_t *)info->alloc_pgt_page(info->context);
		if (!pmd)
			return -ENOMEM;
		ident_pmd_init(info->pmd_flag, pmd, addr, next);
		set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
	}

	return 0;
}

int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
			      unsigned long addr, unsigned long end)
{
	unsigned long next;
	int result;
	int off = info->kernel_mapping ? pgd_index(__PAGE_OFFSET) : 0;

	for (; addr < end; addr = next) {
		pgd_t *pgd = pgd_page + pgd_index(addr) + off;
		pud_t *pud;

		next = (addr & PGDIR_MASK) + PGDIR_SIZE;
		if (next > end)
			next = end;

		if (pgd_present(*pgd)) {
			pud = pud_offset(pgd, 0);
			result = ident_pud_init(info, pud, addr, next);
			if (result)
				return result;
			continue;
		}

		pud = (pud_t *)info->alloc_pgt_page(info->context);
		if (!pud)
			return -ENOMEM;
		result = ident_pud_init(info, pud, addr, next);
		if (result)
			return result;
		set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
	}

	return 0;
}

I
Ingo Molnar 已提交
133 134 135 136 137 138 139 140 141 142 143 144 145 146
static int __init parse_direct_gbpages_off(char *arg)
{
	direct_gbpages = 0;
	return 0;
}
early_param("nogbpages", parse_direct_gbpages_off);

static int __init parse_direct_gbpages_on(char *arg)
{
	direct_gbpages = 1;
	return 0;
}
early_param("gbpages", parse_direct_gbpages_on);

L
Linus Torvalds 已提交
147 148 149 150 151 152
/*
 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
 * physical space so we can cache the place of the first one and move
 * around without checking the pgd every time.
 */

153
pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP;
154 155 156 157
EXPORT_SYMBOL_GPL(__supported_pte_mask);

int force_personality32;

I
Ingo Molnar 已提交
158 159 160 161 162 163 164 165
/*
 * noexec32=on|off
 * Control non executable heap for 32bit processes.
 * To control the stack too use noexec=off
 *
 * on	PROT_READ does not imply PROT_EXEC for 32-bit processes (default)
 * off	PROT_READ implies PROT_EXEC
 */
166 167 168 169 170 171 172 173 174 175
static int __init nonx32_setup(char *str)
{
	if (!strcmp(str, "on"))
		force_personality32 &= ~READ_IMPLIES_EXEC;
	else if (!strcmp(str, "off"))
		force_personality32 |= READ_IMPLIES_EXEC;
	return 1;
}
__setup("noexec32=", nonx32_setup);

176 177 178 179 180 181
/*
 * When memory was added/removed make sure all the processes MM have
 * suitable PGD entries in the local PGD level page.
 */
void sync_global_pgds(unsigned long start, unsigned long end)
{
182 183 184 185 186 187 188 189 190
	unsigned long address;

	for (address = start; address <= end; address += PGDIR_SIZE) {
		const pgd_t *pgd_ref = pgd_offset_k(address);
		struct page *page;

		if (pgd_none(*pgd_ref))
			continue;

A
Andrea Arcangeli 已提交
191
		spin_lock(&pgd_lock);
192 193
		list_for_each_entry(page, &pgd_list, lru) {
			pgd_t *pgd;
194 195
			spinlock_t *pgt_lock;

196
			pgd = (pgd_t *)page_address(page) + pgd_index(address);
A
Andrea Arcangeli 已提交
197
			/* the pgt_lock only for Xen */
198 199 200
			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
			spin_lock(pgt_lock);

201 202 203 204 205
			if (pgd_none(*pgd))
				set_pgd(pgd, *pgd_ref);
			else
				BUG_ON(pgd_page_vaddr(*pgd)
				       != pgd_page_vaddr(*pgd_ref));
206 207

			spin_unlock(pgt_lock);
208
		}
A
Andrea Arcangeli 已提交
209
		spin_unlock(&pgd_lock);
210
	}
211 212
}

213 214 215 216 217
/*
 * NOTE: This function is marked __ref because it calls __init function
 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.
 */
static __ref void *spp_getpage(void)
T
Thomas Gleixner 已提交
218
{
L
Linus Torvalds 已提交
219
	void *ptr;
T
Thomas Gleixner 已提交
220

L
Linus Torvalds 已提交
221
	if (after_bootmem)
222
		ptr = (void *) get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
L
Linus Torvalds 已提交
223 224
	else
		ptr = alloc_bootmem_pages(PAGE_SIZE);
T
Thomas Gleixner 已提交
225 226 227 228 229

	if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) {
		panic("set_pte_phys: cannot allocate page data %s\n",
			after_bootmem ? "after bootmem" : "");
	}
L
Linus Torvalds 已提交
230

231
	pr_debug("spp_getpage %p\n", ptr);
T
Thomas Gleixner 已提交
232

L
Linus Torvalds 已提交
233
	return ptr;
T
Thomas Gleixner 已提交
234
}
L
Linus Torvalds 已提交
235

236
static pud_t *fill_pud(pgd_t *pgd, unsigned long vaddr)
L
Linus Torvalds 已提交
237
{
238 239 240 241 242 243 244 245 246
	if (pgd_none(*pgd)) {
		pud_t *pud = (pud_t *)spp_getpage();
		pgd_populate(&init_mm, pgd, pud);
		if (pud != pud_offset(pgd, 0))
			printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n",
			       pud, pud_offset(pgd, 0));
	}
	return pud_offset(pgd, vaddr);
}
L
Linus Torvalds 已提交
247

248
static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr)
249
{
L
Linus Torvalds 已提交
250
	if (pud_none(*pud)) {
251
		pmd_t *pmd = (pmd_t *) spp_getpage();
252
		pud_populate(&init_mm, pud, pmd);
253
		if (pmd != pmd_offset(pud, 0))
254
			printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
255
			       pmd, pmd_offset(pud, 0));
L
Linus Torvalds 已提交
256
	}
257 258 259
	return pmd_offset(pud, vaddr);
}

260
static pte_t *fill_pte(pmd_t *pmd, unsigned long vaddr)
261
{
L
Linus Torvalds 已提交
262
	if (pmd_none(*pmd)) {
263
		pte_t *pte = (pte_t *) spp_getpage();
264
		pmd_populate_kernel(&init_mm, pmd, pte);
265
		if (pte != pte_offset_kernel(pmd, 0))
266
			printk(KERN_ERR "PAGETABLE BUG #02!\n");
L
Linus Torvalds 已提交
267
	}
268 269 270 271 272 273 274 275 276 277 278 279
	return pte_offset_kernel(pmd, vaddr);
}

void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte)
{
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pud = pud_page + pud_index(vaddr);
	pmd = fill_pmd(pud, vaddr);
	pte = fill_pte(pmd, vaddr);
L
Linus Torvalds 已提交
280 281 282 283 284 285 286 287 288 289

	set_pte(pte, new_pte);

	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

290
void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
{
	pgd_t *pgd;
	pud_t *pud_page;

	pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval));

	pgd = pgd_offset_k(vaddr);
	if (pgd_none(*pgd)) {
		printk(KERN_ERR
			"PGD FIXMAP MISSING, it should be setup in head.S!\n");
		return;
	}
	pud_page = (pud_t*)pgd_page_vaddr(*pgd);
	set_pte_vaddr_pud(pud_page, vaddr, pteval);
}

307
pmd_t * __init populate_extra_pmd(unsigned long vaddr)
308 309 310 311 312
{
	pgd_t *pgd;
	pud_t *pud;

	pgd = pgd_offset_k(vaddr);
313 314 315 316 317 318 319
	pud = fill_pud(pgd, vaddr);
	return fill_pmd(pud, vaddr);
}

pte_t * __init populate_extra_pte(unsigned long vaddr)
{
	pmd_t *pmd;
320

321 322
	pmd = populate_extra_pmd(vaddr);
	return fill_pte(pmd, vaddr);
323 324
}

325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
/*
 * Create large page table mappings for a range of physical addresses.
 */
static void __init __init_extra_mapping(unsigned long phys, unsigned long size,
						pgprot_t prot)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK));
	for (; size; phys += PMD_SIZE, size -= PMD_SIZE) {
		pgd = pgd_offset_k((unsigned long)__va(phys));
		if (pgd_none(*pgd)) {
			pud = (pud_t *) spp_getpage();
			set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE |
						_PAGE_USER));
		}
		pud = pud_offset(pgd, (unsigned long)__va(phys));
		if (pud_none(*pud)) {
			pmd = (pmd_t *) spp_getpage();
			set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE |
						_PAGE_USER));
		}
		pmd = pmd_offset(pud, phys);
		BUG_ON(!pmd_none(*pmd));
		set_pmd(pmd, __pmd(phys | pgprot_val(prot)));
	}
}

void __init init_extra_mapping_wb(unsigned long phys, unsigned long size)
{
	__init_extra_mapping(phys, size, PAGE_KERNEL_LARGE);
}

void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
{
	__init_extra_mapping(phys, size, PAGE_KERNEL_LARGE_NOCACHE);
}

365
/*
366 367 368
 * The head.S code sets up the kernel high mapping:
 *
 *   from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
369 370 371 372 373
 *
 * phys_addr holds the negative offset to the kernel, which is added
 * to the compile time generated pmds. This results in invalid pmds up
 * to the point where we hit the physaddr 0 mapping.
 *
374 375
 * We limit the mappings to the region from _text to _brk_end.  _brk_end
 * is rounded up to the 2MB boundary. This catches the invalid pmds as
376 377 378 379 380
 * well, as they are located before _text:
 */
void __init cleanup_highmap(void)
{
	unsigned long vaddr = __START_KERNEL_map;
381
	unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE;
382
	unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
383 384
	pmd_t *pmd = level2_kernel_pgt;

385 386 387 388 389 390 391 392
	/*
	 * Native path, max_pfn_mapped is not set yet.
	 * Xen has valid max_pfn_mapped set in
	 *	arch/x86/xen/mmu.c:xen_setup_kernel_pagetable().
	 */
	if (max_pfn_mapped)
		vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);

393
	for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) {
394
		if (pmd_none(*pmd))
395 396 397 398 399 400
			continue;
		if (vaddr < (unsigned long) _text || vaddr > end)
			set_pmd(pmd, __pmd(0));
	}
}

401
static unsigned long __meminit
402 403
phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
	      pgprot_t prot)
404
{
405
	unsigned long pages = 0, next;
406
	unsigned long last_map_addr = end;
407
	int i;
408

409 410
	pte_t *pte = pte_page + pte_index(addr);

411 412
	for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) {
		next = (addr & PAGE_MASK) + PAGE_SIZE;
413
		if (addr >= end) {
414 415 416 417 418
			if (!after_bootmem &&
			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) &&
			    !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN))
				set_pte(pte, __pte(0));
			continue;
419 420
		}

421 422 423 424 425 426
		/*
		 * We will re-use the existing mapping.
		 * Xen for example has some special requirements, like mapping
		 * pagetable pages as RO. So assume someone who pre-setup
		 * these mappings are more intelligent.
		 */
427
		if (pte_val(*pte)) {
J
Jan Beulich 已提交
428 429
			if (!after_bootmem)
				pages++;
430
			continue;
431
		}
432 433 434 435 436

		if (0)
			printk("   pte=%p addr=%lx pte=%016lx\n",
			       pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
		pages++;
437
		set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot));
438
		last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;
439
	}
440

441
	update_page_count(PG_LEVEL_4K, pages);
442 443

	return last_map_addr;
444 445
}

446
static unsigned long __meminit
447
phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
448
	      unsigned long page_size_mask, pgprot_t prot)
449
{
450
	unsigned long pages = 0, next;
451
	unsigned long last_map_addr = end;
452

453
	int i = pmd_index(address);
454

455
	for (; i < PTRS_PER_PMD; i++, address = next) {
456
		pmd_t *pmd = pmd_page + pmd_index(address);
457
		pte_t *pte;
458
		pgprot_t new_prot = prot;
459

460
		next = (address & PMD_MASK) + PMD_SIZE;
461
		if (address >= end) {
462 463 464 465 466
			if (!after_bootmem &&
			    !e820_any_mapped(address & PMD_MASK, next, E820_RAM) &&
			    !e820_any_mapped(address & PMD_MASK, next, E820_RESERVED_KERN))
				set_pmd(pmd, __pmd(0));
			continue;
467
		}
468

469
		if (pmd_val(*pmd)) {
470 471
			if (!pmd_large(*pmd)) {
				spin_lock(&init_mm.page_table_lock);
472
				pte = (pte_t *)pmd_page_vaddr(*pmd);
Y
Yinghai Lu 已提交
473
				last_map_addr = phys_pte_init(pte, address,
474
								end, prot);
475
				spin_unlock(&init_mm.page_table_lock);
476
				continue;
477
			}
478 479 480 481 482 483 484 485 486 487 488 489
			/*
			 * If we are ok with PG_LEVEL_2M mapping, then we will
			 * use the existing mapping,
			 *
			 * Otherwise, we will split the large page mapping but
			 * use the same existing protection bits except for
			 * large page, so that we don't violate Intel's TLB
			 * Application note (317080) which says, while changing
			 * the page sizes, new and old translations should
			 * not differ with respect to page frame and
			 * attributes.
			 */
490
			if (page_size_mask & (1 << PG_LEVEL_2M)) {
J
Jan Beulich 已提交
491 492
				if (!after_bootmem)
					pages++;
493
				last_map_addr = next;
494
				continue;
495
			}
496
			new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd));
497 498
		}

499
		if (page_size_mask & (1<<PG_LEVEL_2M)) {
500
			pages++;
501
			spin_lock(&init_mm.page_table_lock);
502
			set_pte((pte_t *)pmd,
503
				pfn_pte((address & PMD_MASK) >> PAGE_SHIFT,
504
					__pgprot(pgprot_val(prot) | _PAGE_PSE)));
505
			spin_unlock(&init_mm.page_table_lock);
506
			last_map_addr = next;
507
			continue;
508
		}
509

510
		pte = alloc_low_page();
511
		last_map_addr = phys_pte_init(pte, address, end, new_prot);
512

513
		spin_lock(&init_mm.page_table_lock);
514
		pmd_populate_kernel(&init_mm, pmd, pte);
515
		spin_unlock(&init_mm.page_table_lock);
516
	}
517
	update_page_count(PG_LEVEL_2M, pages);
518
	return last_map_addr;
519 520
}

521
static unsigned long __meminit
522 523
phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
			 unsigned long page_size_mask)
T
Thomas Gleixner 已提交
524
{
525
	unsigned long pages = 0, next;
526
	unsigned long last_map_addr = end;
527
	int i = pud_index(addr);
528

529
	for (; i < PTRS_PER_PUD; i++, addr = next) {
530
		pud_t *pud = pud_page + pud_index(addr);
L
Linus Torvalds 已提交
531
		pmd_t *pmd;
532
		pgprot_t prot = PAGE_KERNEL;
L
Linus Torvalds 已提交
533

534
		next = (addr & PUD_MASK) + PUD_SIZE;
535 536 537 538 539
		if (addr >= end) {
			if (!after_bootmem &&
			    !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) &&
			    !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN))
				set_pud(pud, __pud(0));
L
Linus Torvalds 已提交
540
			continue;
T
Thomas Gleixner 已提交
541
		}
L
Linus Torvalds 已提交
542

543
		if (pud_val(*pud)) {
544
			if (!pud_large(*pud)) {
545
				pmd = pmd_offset(pud, 0);
Y
Yinghai Lu 已提交
546
				last_map_addr = phys_pmd_init(pmd, addr, end,
547
							 page_size_mask, prot);
Y
Yinghai Lu 已提交
548
				__flush_tlb_all();
549 550
				continue;
			}
551 552 553 554 555 556 557 558 559 560 561 562
			/*
			 * If we are ok with PG_LEVEL_1G mapping, then we will
			 * use the existing mapping.
			 *
			 * Otherwise, we will split the gbpage mapping but use
			 * the same existing protection  bits except for large
			 * page, so that we don't violate Intel's TLB
			 * Application note (317080) which says, while changing
			 * the page sizes, new and old translations should
			 * not differ with respect to page frame and
			 * attributes.
			 */
563
			if (page_size_mask & (1 << PG_LEVEL_1G)) {
J
Jan Beulich 已提交
564 565
				if (!after_bootmem)
					pages++;
566
				last_map_addr = next;
567
				continue;
568
			}
569
			prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud));
570 571
		}

572
		if (page_size_mask & (1<<PG_LEVEL_1G)) {
573
			pages++;
574
			spin_lock(&init_mm.page_table_lock);
575
			set_pte((pte_t *)pud,
576 577
				pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT,
					PAGE_KERNEL_LARGE));
578
			spin_unlock(&init_mm.page_table_lock);
579
			last_map_addr = next;
580 581 582
			continue;
		}

583
		pmd = alloc_low_page();
584 585
		last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,
					      prot);
586 587

		spin_lock(&init_mm.page_table_lock);
588
		pud_populate(&init_mm, pud, pmd);
589
		spin_unlock(&init_mm.page_table_lock);
L
Linus Torvalds 已提交
590
	}
A
Andi Kleen 已提交
591
	__flush_tlb_all();
592

593
	update_page_count(PG_LEVEL_1G, pages);
594

595
	return last_map_addr;
T
Thomas Gleixner 已提交
596
}
L
Linus Torvalds 已提交
597

598
unsigned long __meminit
599 600 601
kernel_physical_mapping_init(unsigned long start,
			     unsigned long end,
			     unsigned long page_size_mask)
T
Thomas Gleixner 已提交
602
{
603
	bool pgd_changed = false;
604
	unsigned long next, last_map_addr = end;
605
	unsigned long addr;
L
Linus Torvalds 已提交
606 607 608

	start = (unsigned long)__va(start);
	end = (unsigned long)__va(end);
609
	addr = start;
L
Linus Torvalds 已提交
610 611

	for (; start < end; start = next) {
612 613 614
		pgd_t *pgd = pgd_offset_k(start);
		pud_t *pud;

615
		next = (start & PGDIR_MASK) + PGDIR_SIZE;
616 617

		if (pgd_val(*pgd)) {
618
			pud = (pud_t *)pgd_page_vaddr(*pgd);
Y
Yinghai Lu 已提交
619
			last_map_addr = phys_pud_init(pud, __pa(start),
620
						 __pa(end), page_size_mask);
621 622 623
			continue;
		}

624
		pud = alloc_low_page();
625
		last_map_addr = phys_pud_init(pud, __pa(start), __pa(end),
626
						 page_size_mask);
627 628

		spin_lock(&init_mm.page_table_lock);
629
		pgd_populate(&init_mm, pgd, pud);
630
		spin_unlock(&init_mm.page_table_lock);
631
		pgd_changed = true;
T
Thomas Gleixner 已提交
632
	}
633 634 635 636

	if (pgd_changed)
		sync_global_pgds(addr, end);

637
	__flush_tlb_all();
L
Linus Torvalds 已提交
638

639 640
	return last_map_addr;
}
641

642
#ifndef CONFIG_NUMA
643
void __init initmem_init(void)
644
{
T
Tejun Heo 已提交
645
	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
646
}
647
#endif
648

L
Linus Torvalds 已提交
649 650
void __init paging_init(void)
{
651
	sparse_memory_present_with_active_regions(MAX_NUMNODES);
652
	sparse_init();
653 654 655 656 657 658 659

	/*
	 * clear the default setting with node 0
	 * note: don't use nodes_clear here, that is really clearing when
	 *	 numa support is not compiled in, and later node_set_state
	 *	 will not set it back.
	 */
660 661 662
	node_clear_state(0, N_MEMORY);
	if (N_MEMORY != N_NORMAL_MEMORY)
		node_clear_state(0, N_NORMAL_MEMORY);
663

664
	zone_sizes_init();
L
Linus Torvalds 已提交
665 666
}

667 668 669
/*
 * Memory hotplug specific functions
 */
670
#ifdef CONFIG_MEMORY_HOTPLUG
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
/*
 * After memory hotplug the variables max_pfn, max_low_pfn and high_memory need
 * updating.
 */
static void  update_end_of_memory_vars(u64 start, u64 size)
{
	unsigned long end_pfn = PFN_UP(start + size);

	if (end_pfn > max_pfn) {
		max_pfn = end_pfn;
		max_low_pfn = end_pfn;
		high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1;
	}
}

686 687 688 689
/*
 * Memory is added always to NORMAL zone. This means you will never get
 * additional DMA/DMA32 memory.
 */
690
int arch_add_memory(int nid, u64 start, u64 size)
691
{
692
	struct pglist_data *pgdat = NODE_DATA(nid);
693
	struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
694
	unsigned long start_pfn = start >> PAGE_SHIFT;
695 696 697
	unsigned long nr_pages = size >> PAGE_SHIFT;
	int ret;

698
	init_memory_mapping(start, start + size);
699

700
	ret = __add_pages(nid, zone, start_pfn, nr_pages);
701
	WARN_ON_ONCE(ret);
702

703 704 705
	/* update max_pfn, max_low_pfn and high_memory */
	update_end_of_memory_vars(start, size);

706 707
	return ret;
}
708
EXPORT_SYMBOL_GPL(arch_add_memory);
709

710 711
#endif /* CONFIG_MEMORY_HOTPLUG */

712
static struct kcore_list kcore_vsyscall;
L
Linus Torvalds 已提交
713

Y
Yinghai Lu 已提交
714 715 716 717 718 719 720 721 722 723
static void __init register_page_bootmem_info(void)
{
#ifdef CONFIG_NUMA
	int i;

	for_each_online_node(i)
		register_page_bootmem_info_node(NODE_DATA(i));
#endif
}

L
Linus Torvalds 已提交
724 725
void __init mem_init(void)
{
726
	long codesize, reservedpages, datasize, initsize;
727
	unsigned long absent_pages;
L
Linus Torvalds 已提交
728

729
	pci_iommu_alloc();
L
Linus Torvalds 已提交
730

731
	/* clear_bss() already clear the empty_zero_page */
L
Linus Torvalds 已提交
732 733 734 735

	reservedpages = 0;

	/* this will put all low memory onto the freelists */
Y
Yinghai Lu 已提交
736
	register_page_bootmem_info();
737
	totalram_pages = free_all_bootmem();
738 739 740

	absent_pages = absent_pages_in_range(0, max_pfn);
	reservedpages = max_pfn - totalram_pages - absent_pages;
L
Linus Torvalds 已提交
741 742 743 744 745 746 747
	after_bootmem = 1;

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

	/* Register memory areas for /proc/kcore */
T
Thomas Gleixner 已提交
748
	kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
K
KAMEZAWA Hiroyuki 已提交
749
			 VSYSCALL_END - VSYSCALL_START, KCORE_OTHER);
L
Linus Torvalds 已提交
750

751
	printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
752
			 "%ldk absent, %ldk reserved, %ldk data, %ldk init)\n",
753
		nr_free_pages() << (PAGE_SHIFT-10),
Y
Yinghai Lu 已提交
754
		max_pfn << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
755
		codesize >> 10,
756
		absent_pages << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
757 758 759 760 761
		reservedpages << (PAGE_SHIFT-10),
		datasize >> 10,
		initsize >> 10);
}

762
#ifdef CONFIG_DEBUG_RODATA
763 764
const int rodata_test_data = 0xC3;
EXPORT_SYMBOL_GPL(rodata_test_data);
765

766
int kernel_set_to_readonly;
767 768 769

void set_kernel_text_rw(void)
{
770
	unsigned long start = PFN_ALIGN(_text);
771
	unsigned long end = PFN_ALIGN(__stop___ex_table);
772 773 774 775 776 777 778

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read write\n",
		 start, end);

779 780 781 782 783
	/*
	 * Make the kernel identity mapping for text RW. Kernel text
	 * mapping will always be RO. Refer to the comment in
	 * static_protections() in pageattr.c
	 */
784 785 786 787 788
	set_memory_rw(start, (end - start) >> PAGE_SHIFT);
}

void set_kernel_text_ro(void)
{
789
	unsigned long start = PFN_ALIGN(_text);
790
	unsigned long end = PFN_ALIGN(__stop___ex_table);
791 792 793 794 795 796 797

	if (!kernel_set_to_readonly)
		return;

	pr_debug("Set kernel text: %lx - %lx for read only\n",
		 start, end);

798 799 800
	/*
	 * Set the kernel identity mapping for text RO.
	 */
801 802 803
	set_memory_ro(start, (end - start) >> PAGE_SHIFT);
}

804 805
void mark_rodata_ro(void)
{
806
	unsigned long start = PFN_ALIGN(_text);
807 808
	unsigned long rodata_start =
		((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK;
809 810 811 812
	unsigned long end = (unsigned long) &__end_rodata_hpage_align;
	unsigned long text_end = PAGE_ALIGN((unsigned long) &__stop___ex_table);
	unsigned long rodata_end = PAGE_ALIGN((unsigned long) &__end_rodata);
	unsigned long data_start = (unsigned long) &_sdata;
813
	unsigned long all_end = PFN_ALIGN(&_end);
814

815
	printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
816
	       (end - start) >> 10);
817 818
	set_memory_ro(start, (end - start) >> PAGE_SHIFT);

819 820
	kernel_set_to_readonly = 1;

821
	/*
822 823
	 * The rodata/data/bss/brk section (but not the kernel text!)
	 * should also be not-executable.
824
	 */
825
	set_memory_nx(rodata_start, (all_end - rodata_start) >> PAGE_SHIFT);
826

827 828
	rodata_test();

829
#ifdef CONFIG_CPA_DEBUG
830
	printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end);
831
	set_memory_rw(start, (end-start) >> PAGE_SHIFT);
832

833
	printk(KERN_INFO "Testing CPA: again\n");
834
	set_memory_ro(start, (end-start) >> PAGE_SHIFT);
835
#endif
836 837 838 839 840 841 842 843

	free_init_pages("unused kernel memory",
			(unsigned long) page_address(virt_to_page(text_end)),
			(unsigned long)
				 page_address(virt_to_page(rodata_start)));
	free_init_pages("unused kernel memory",
			(unsigned long) page_address(virt_to_page(rodata_end)),
			(unsigned long) page_address(virt_to_page(data_start)));
844
}
845

846 847
#endif

T
Thomas Gleixner 已提交
848 849
int kern_addr_valid(unsigned long addr)
{
L
Linus Torvalds 已提交
850
	unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
T
Thomas Gleixner 已提交
851 852 853 854
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
L
Linus Torvalds 已提交
855 856

	if (above != 0 && above != -1UL)
T
Thomas Gleixner 已提交
857 858
		return 0;

L
Linus Torvalds 已提交
859 860 861 862 863 864
	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
T
Thomas Gleixner 已提交
865
		return 0;
L
Linus Torvalds 已提交
866 867 868 869

	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;
T
Thomas Gleixner 已提交
870

L
Linus Torvalds 已提交
871 872 873 874 875 876
	if (pmd_large(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;
T
Thomas Gleixner 已提交
877

L
Linus Torvalds 已提交
878 879 880
	return pfn_valid(pte_pfn(*pte));
}

T
Thomas Gleixner 已提交
881 882 883 884 885
/*
 * A pseudo VMA to allow ptrace access for the vsyscall page.  This only
 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
 * not need special handling anymore:
 */
L
Linus Torvalds 已提交
886
static struct vm_area_struct gate_vma = {
T
Thomas Gleixner 已提交
887 888 889 890
	.vm_start	= VSYSCALL_START,
	.vm_end		= VSYSCALL_START + (VSYSCALL_MAPPED_PAGES * PAGE_SIZE),
	.vm_page_prot	= PAGE_READONLY_EXEC,
	.vm_flags	= VM_READ | VM_EXEC
L
Linus Torvalds 已提交
891 892
};

893
struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
L
Linus Torvalds 已提交
894 895
{
#ifdef CONFIG_IA32_EMULATION
896
	if (!mm || mm->context.ia32_compat)
897
		return NULL;
L
Linus Torvalds 已提交
898 899 900 901
#endif
	return &gate_vma;
}

902
int in_gate_area(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
903
{
904
	struct vm_area_struct *vma = get_gate_vma(mm);
T
Thomas Gleixner 已提交
905

906 907
	if (!vma)
		return 0;
T
Thomas Gleixner 已提交
908

L
Linus Torvalds 已提交
909 910 911
	return (addr >= vma->vm_start) && (addr < vma->vm_end);
}

T
Thomas Gleixner 已提交
912
/*
913 914 915
 * Use this when you have no reliable mm, typically from interrupt
 * context. It is less reliable than using a task's mm and may give
 * false positives.
L
Linus Torvalds 已提交
916
 */
917
int in_gate_area_no_mm(unsigned long addr)
L
Linus Torvalds 已提交
918
{
919
	return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
L
Linus Torvalds 已提交
920
}
921

922 923 924 925 926 927 928 929
const char *arch_vma_name(struct vm_area_struct *vma)
{
	if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
		return "[vdso]";
	if (vma == &gate_vma)
		return "[vsyscall]";
	return NULL;
}
930

931 932 933 934 935 936 937 938 939 940 941
#ifdef CONFIG_X86_UV
unsigned long memory_block_size_bytes(void)
{
	if (is_uv_system()) {
		printk(KERN_INFO "UV: memory block size 2GB\n");
		return 2UL * 1024 * 1024 * 1024;
	}
	return MIN_MEMORY_BLOCK_SIZE;
}
#endif

942 943 944 945
#ifdef CONFIG_SPARSEMEM_VMEMMAP
/*
 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
 */
946 947 948 949
static long __meminitdata addr_start, addr_end;
static void __meminitdata *p_start, *p_end;
static int __meminitdata node_start;

T
Thomas Gleixner 已提交
950 951
int __meminit
vmemmap_populate(struct page *start_page, unsigned long size, int node)
952 953 954 955 956 957 958 959 960
{
	unsigned long addr = (unsigned long)start_page;
	unsigned long end = (unsigned long)(start_page + size);
	unsigned long next;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	for (; addr < end; addr = next) {
961
		void *p = NULL;
962 963 964 965

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;
T
Thomas Gleixner 已提交
966

967 968 969 970
		pud = vmemmap_pud_populate(pgd, addr, node);
		if (!pud)
			return -ENOMEM;

971 972 973 974 975 976 977 978
		if (!cpu_has_pse) {
			next = (addr + PAGE_SIZE) & PAGE_MASK;
			pmd = vmemmap_pmd_populate(pud, addr, node);

			if (!pmd)
				return -ENOMEM;

			p = vmemmap_pte_populate(pmd, addr, node);
T
Thomas Gleixner 已提交
979

980 981 982
			if (!p)
				return -ENOMEM;

983 984
			addr_end = addr + PAGE_SIZE;
			p_end = p + PAGE_SIZE;
T
Thomas Gleixner 已提交
985
		} else {
986 987 988 989 990 991
			next = pmd_addr_end(addr, end);

			pmd = pmd_offset(pud, addr);
			if (pmd_none(*pmd)) {
				pte_t entry;

992
				p = vmemmap_alloc_block_buf(PMD_SIZE, node);
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
				if (!p)
					return -ENOMEM;

				entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
						PAGE_KERNEL_LARGE);
				set_pmd(pmd, __pmd(pte_val(entry)));

				/* check to see if we have contiguous blocks */
				if (p_end != p || node_start != node) {
					if (p_start)
						printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
						       addr_start, addr_end-1, p_start, p_end-1, node_start);
					addr_start = addr;
					node_start = node;
					p_start = p;
				}
Y
Yinghai Lu 已提交
1009 1010 1011

				addr_end = addr + PMD_SIZE;
				p_end = p + PMD_SIZE;
1012 1013
			} else
				vmemmap_verify((pte_t *)pmd, node, addr, next);
T
Thomas Gleixner 已提交
1014
		}
1015

1016
	}
1017
	sync_global_pgds((unsigned long)start_page, end);
1018 1019
	return 0;
}
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030

void __meminit vmemmap_populate_print_last(void)
{
	if (p_start) {
		printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
			addr_start, addr_end-1, p_start, p_end-1, node_start);
		p_start = NULL;
		p_end = NULL;
		node_start = 0;
	}
}
1031
#endif