init.c 12.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994 - 2000 Ralf Baechle
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
 * Copyright (C) 2000 MIPS Technologies, Inc.  All rights reserved.
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/pagemap.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <linux/swap.h>
26
#include <linux/proc_fs.h>
D
Dave Hansen 已提交
27
#include <linux/pfn.h>
L
Linus Torvalds 已提交
28

R
Ralf Baechle 已提交
29
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
30 31 32 33
#include <asm/bootinfo.h>
#include <asm/cachectl.h>
#include <asm/cpu.h>
#include <asm/dma.h>
34
#include <asm/kmap_types.h>
L
Linus Torvalds 已提交
35 36 37 38 39
#include <asm/mmu_context.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
#include <asm/fixmap.h>

/* Atomicity and interruptability */
#ifdef CONFIG_MIPS_MT_SMTC

#include <asm/mipsmtregs.h>

#define ENTER_CRITICAL(flags) \
	{ \
	unsigned int mvpflags; \
	local_irq_save(flags);\
	mvpflags = dvpe()
#define EXIT_CRITICAL(flags) \
	evpe(mvpflags); \
	local_irq_restore(flags); \
	}
#else

#define ENTER_CRITICAL(flags) local_irq_save(flags)
#define EXIT_CRITICAL(flags) local_irq_restore(flags)

#endif /* CONFIG_MIPS_MT_SMTC */
L
Linus Torvalds 已提交
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78

DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);

/*
 * We have up to 8 empty zeroed pages so we can map one of the right colour
 * when needed.  This is necessary only on R4000 / R4400 SC and MC versions
 * where we have to avoid VCED / VECI exceptions for good performance at
 * any price.  Since page is never written to after the initialization we
 * don't have to care about aliases on other CPUs.
 */
unsigned long empty_zero_page, zero_page_mask;

/*
 * Not static inline because used by IP27 special magic initialization code
 */
unsigned long setup_zero_pages(void)
{
N
Nick Piggin 已提交
79 80
	unsigned int order;
	unsigned long size;
L
Linus Torvalds 已提交
81 82 83 84 85 86 87 88 89 90 91
	struct page *page;

	if (cpu_has_vce)
		order = 3;
	else
		order = 0;

	empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
	if (!empty_zero_page)
		panic("Oh boy, that early out of memory?");

92
	page = virt_to_page((void *)empty_zero_page);
N
Nick Piggin 已提交
93
	split_page(page, order);
94
	while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {
95
		SetPageReserved(page);
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104
		page++;
	}

	size = PAGE_SIZE << order;
	zero_page_mask = (size - 1) & PAGE_MASK;

	return 1UL << order;
}

105 106 107 108
/*
 * These are almost like kmap_atomic / kunmap_atmic except they take an
 * additional address argument as the hint.
 */
L
Linus Torvalds 已提交
109 110

#define kmap_get_fixmap_pte(vaddr)					\
111
	pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
L
Linus Torvalds 已提交
112

113 114 115 116 117 118 119 120 121 122 123 124 125 126
#ifdef CONFIG_MIPS_MT_SMTC
static pte_t *kmap_coherent_pte;
static void __init kmap_coherent_init(void)
{
	unsigned long vaddr;

	/* cache the first coherent kmap pte */
	vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
	kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
}
#else
static inline void kmap_coherent_init(void) {}
#endif

127
void *kmap_coherent(struct page *page, unsigned long addr)
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
{
	enum fixed_addresses idx;
	unsigned long vaddr, flags, entrylo;
	unsigned long old_ctx;
	pte_t pte;
	int tlbidx;

	inc_preempt_count();
	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
#ifdef CONFIG_MIPS_MT_SMTC
	idx += FIX_N_COLOURS * smp_processor_id();
#endif
	vaddr = __fix_to_virt(FIX_CMAP_END - idx);
	pte = mk_pte(page, PAGE_KERNEL);
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
	entrylo = pte.pte_high;
#else
	entrylo = pte_val(pte) >> 6;
#endif

	ENTER_CRITICAL(flags);
	old_ctx = read_c0_entryhi();
	write_c0_entryhi(vaddr & (PAGE_MASK << 1));
	write_c0_entrylo0(entrylo);
	write_c0_entrylo1(entrylo);
#ifdef CONFIG_MIPS_MT_SMTC
	set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
	/* preload TLB instead of local_flush_tlb_one() */
	mtc0_tlbw_hazard();
	tlb_probe();
	tlb_probe_hazard();
	tlbidx = read_c0_index();
	mtc0_tlbw_hazard();
	if (tlbidx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();
#else
	tlbidx = read_c0_wired();
	write_c0_wired(tlbidx + 1);
	write_c0_index(tlbidx);
	mtc0_tlbw_hazard();
	tlb_write_indexed();
#endif
	tlbw_use_hazard();
	write_c0_entryhi(old_ctx);
	EXIT_CRITICAL(flags);

	return (void*) vaddr;
}

#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))

181
void kunmap_coherent(void)
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
{
#ifndef CONFIG_MIPS_MT_SMTC
	unsigned int wired;
	unsigned long flags, old_ctx;

	ENTER_CRITICAL(flags);
	old_ctx = read_c0_entryhi();
	wired = read_c0_wired() - 1;
	write_c0_wired(wired);
	write_c0_index(wired);
	write_c0_entryhi(UNIQUE_ENTRYHI(wired));
	write_c0_entrylo0(0);
	write_c0_entrylo1(0);
	mtc0_tlbw_hazard();
	tlb_write_indexed();
	tlbw_use_hazard();
	write_c0_entryhi(old_ctx);
	EXIT_CRITICAL(flags);
#endif
	dec_preempt_count();
	preempt_check_resched();
}

205 206 207 208 209 210 211 212 213
void copy_user_highpage(struct page *to, struct page *from,
	unsigned long vaddr, struct vm_area_struct *vma)
{
	void *vfrom, *vto;

	vto = kmap_atomic(to, KM_USER1);
	if (cpu_has_dc_aliases) {
		vfrom = kmap_coherent(from, vaddr);
		copy_page(vto, vfrom);
214
		kunmap_coherent();
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
	} else {
		vfrom = kmap_atomic(from, KM_USER0);
		copy_page(vto, vfrom);
		kunmap_atomic(vfrom, KM_USER0);
	}
	if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
	    pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
		flush_data_cache_page((unsigned long)vto);
	kunmap_atomic(vto, KM_USER1);
	/* Make sure this page is cleared on other CPU's too before using it */
	smp_wmb();
}

EXPORT_SYMBOL(copy_user_highpage);

230 231 232 233 234 235 236
void copy_to_user_page(struct vm_area_struct *vma,
	struct page *page, unsigned long vaddr, void *dst, const void *src,
	unsigned long len)
{
	if (cpu_has_dc_aliases) {
		void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
		memcpy(vto, src, len);
237
		kunmap_coherent();
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
	} else
		memcpy(dst, src, len);
	if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
		flush_cache_page(vma, vaddr, page_to_pfn(page));
}

EXPORT_SYMBOL(copy_to_user_page);

void copy_from_user_page(struct vm_area_struct *vma,
	struct page *page, unsigned long vaddr, void *dst, const void *src,
	unsigned long len)
{
	if (cpu_has_dc_aliases) {
		void *vfrom =
			kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
		memcpy(dst, vfrom, len);
254
		kunmap_coherent();
255 256 257 258 259 260 261 262
	} else
		memcpy(dst, src, len);
}

EXPORT_SYMBOL(copy_from_user_page);


#ifdef CONFIG_HIGHMEM
263 264
unsigned long highstart_pfn, highend_pfn;

265 266 267
pte_t *kmap_pte;
pgprot_t kmap_prot;

L
Linus Torvalds 已提交
268 269 270 271 272 273 274 275 276 277
static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

	/* cache the first kmap pte */
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);

	kmap_prot = PAGE_KERNEL;
}
278
#endif /* CONFIG_HIGHMEM */
L
Linus Torvalds 已提交
279

280
void __init fixrange_init(unsigned long start, unsigned long end,
L
Linus Torvalds 已提交
281 282
	pgd_t *pgd_base)
{
283
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
L
Linus Torvalds 已提交
284
	pgd_t *pgd;
285
	pud_t *pud;
L
Linus Torvalds 已提交
286 287
	pmd_t *pmd;
	pte_t *pte;
288
	int i, j, k;
L
Linus Torvalds 已提交
289 290 291 292
	unsigned long vaddr;

	vaddr = start;
	i = __pgd_offset(vaddr);
293 294
	j = __pud_offset(vaddr);
	k = __pmd_offset(vaddr);
L
Linus Torvalds 已提交
295 296 297
	pgd = pgd_base + i;

	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
298 299 300 301 302 303
		pud = (pud_t *)pgd;
		for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
			pmd = (pmd_t *)pud;
			for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
				if (pmd_none(*pmd)) {
					pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
304
					set_pmd(pmd, __pmd((unsigned long)pte));
305 306 307 308
					if (pte != pte_offset_kernel(pmd, 0))
						BUG();
				}
				vaddr += PMD_SIZE;
L
Linus Torvalds 已提交
309
			}
310
			k = 0;
L
Linus Torvalds 已提交
311 312 313
		}
		j = 0;
	}
314
#endif
L
Linus Torvalds 已提交
315 316
}

317
#ifndef CONFIG_NEED_MULTIPLE_NODES
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
static int __init page_is_ram(unsigned long pagenr)
{
	int i;

	for (i = 0; i < boot_mem_map.nr_map; i++) {
		unsigned long addr, end;

		if (boot_mem_map.map[i].type != BOOT_MEM_RAM)
			/* not usable memory */
			continue;

		addr = PFN_UP(boot_mem_map.map[i].addr);
		end = PFN_DOWN(boot_mem_map.map[i].addr +
			       boot_mem_map.map[i].size);

		if (pagenr >= addr && pagenr < end)
			return 1;
	}

	return 0;
}

L
Linus Torvalds 已提交
340 341
void __init paging_init(void)
{
342
	unsigned long zones_size[MAX_NR_ZONES] = { 0, };
343
#ifndef CONFIG_FLATMEM
344
	unsigned long zholes_size[MAX_NR_ZONES] = { 0, };
345 346
	unsigned long i, j, pfn;
#endif
L
Linus Torvalds 已提交
347 348 349 350 351 352

	pagetable_init();

#ifdef CONFIG_HIGHMEM
	kmap_init();
#endif
353
	kmap_coherent_init();
L
Linus Torvalds 已提交
354

355 356 357 358 359 360
#ifdef CONFIG_ZONE_DMA
	if (min_low_pfn < MAX_DMA_PFN && MAX_DMA_PFN <= max_low_pfn) {
		zones_size[ZONE_DMA] = MAX_DMA_PFN - min_low_pfn;
		zones_size[ZONE_NORMAL] = max_low_pfn - MAX_DMA_PFN;
	} else if (max_low_pfn < MAX_DMA_PFN)
		zones_size[ZONE_DMA] = max_low_pfn - min_low_pfn;
361
	else
L
Linus Torvalds 已提交
362
#endif
363
	zones_size[ZONE_NORMAL] = max_low_pfn - min_low_pfn;
364

L
Linus Torvalds 已提交
365
#ifdef CONFIG_HIGHMEM
366 367 368 369 370 371 372
	zones_size[ZONE_HIGHMEM] = highend_pfn - highstart_pfn;

	if (cpu_has_dc_aliases && zones_size[ZONE_HIGHMEM]) {
		printk(KERN_WARNING "This processor doesn't support highmem."
		       " %ldk highmem ignored\n", zones_size[ZONE_HIGHMEM]);
		zones_size[ZONE_HIGHMEM] = 0;
	}
L
Linus Torvalds 已提交
373 374
#endif

375
#ifdef CONFIG_FLATMEM
L
Linus Torvalds 已提交
376
	free_area_init(zones_size);
377
#else
378
	pfn = min_low_pfn;
379 380 381 382 383 384
	for (i = 0; i < MAX_NR_ZONES; i++)
		for (j = 0; j < zones_size[i]; j++, pfn++)
			if (!page_is_ram(pfn))
				zholes_size[i]++;
	free_area_init_node(0, NODE_DATA(0), zones_size, 0, zholes_size);
#endif
L
Linus Torvalds 已提交
385 386
}

387 388 389 390 391
static struct kcore_list kcore_mem, kcore_vmalloc;
#ifdef CONFIG_64BIT
static struct kcore_list kcore_kseg0;
#endif

L
Linus Torvalds 已提交
392 393 394 395 396 397 398 399 400
void __init mem_init(void)
{
	unsigned long codesize, reservedpages, datasize, initsize;
	unsigned long tmp, ram;

#ifdef CONFIG_HIGHMEM
#ifdef CONFIG_DISCONTIGMEM
#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
#endif
401
	max_mapnr = highend_pfn;
L
Linus Torvalds 已提交
402
#else
403
	max_mapnr = max_low_pfn;
L
Linus Torvalds 已提交
404 405 406 407 408 409 410 411 412 413
#endif
	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);

	totalram_pages += free_all_bootmem();
	totalram_pages -= setup_zero_pages();	/* Setup zeroed pages.  */

	reservedpages = ram = 0;
	for (tmp = 0; tmp < max_low_pfn; tmp++)
		if (page_is_ram(tmp)) {
			ram++;
C
Chad Reese 已提交
414
			if (PageReserved(pfn_to_page(tmp)))
L
Linus Torvalds 已提交
415 416
				reservedpages++;
		}
417
	num_physpages = ram;
L
Linus Torvalds 已提交
418 419 420 421 422 423 424 425 426 427

#ifdef CONFIG_HIGHMEM
	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
		struct page *page = mem_map + tmp;

		if (!page_is_ram(tmp)) {
			SetPageReserved(page);
			continue;
		}
		ClearPageReserved(page);
428
		init_page_count(page);
L
Linus Torvalds 已提交
429 430 431 432
		__free_page(page);
		totalhigh_pages++;
	}
	totalram_pages += totalhigh_pages;
433
	num_physpages += totalhigh_pages;
L
Linus Torvalds 已提交
434 435 436 437 438 439
#endif

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

440 441 442 443 444 445 446 447 448 449
#ifdef CONFIG_64BIT
	if ((unsigned long) &_text > (unsigned long) CKSEG0)
		/* The -4 is a hack so that user tools don't have to handle
		   the overflow.  */
		kclist_add(&kcore_kseg0, (void *) CKSEG0, 0x80000000 - 4);
#endif
	kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
	kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
		   VMALLOC_END-VMALLOC_START);

L
Linus Torvalds 已提交
450 451 452 453 454 455 456 457 458 459
	printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
	       "%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
	       (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
	       ram << (PAGE_SHIFT-10),
	       codesize >> 10,
	       reservedpages << (PAGE_SHIFT-10),
	       datasize >> 10,
	       initsize >> 10,
	       (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
}
460
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
L
Linus Torvalds 已提交
461

462
void free_init_pages(const char *what, unsigned long begin, unsigned long end)
463
{
464
	unsigned long pfn;
465

466 467 468 469 470 471 472 473
	for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
		struct page *page = pfn_to_page(pfn);
		void *addr = phys_to_virt(PFN_PHYS(pfn));

		ClearPageReserved(page);
		init_page_count(page);
		memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
		__free_page(page);
474 475 476 477 478
		totalram_pages++;
	}
	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
}

L
Linus Torvalds 已提交
479 480 481
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
482 483 484
	free_init_pages("initrd memory",
			virt_to_phys((void *)start),
			virt_to_phys((void *)end));
L
Linus Torvalds 已提交
485 486 487
}
#endif

488
void __init_refok free_initmem(void)
L
Linus Torvalds 已提交
489
{
490
	prom_free_prom_memory();
491 492 493
	free_init_pages("unused kernel memory",
			__pa_symbol(&__init_begin),
			__pa_symbol(&__init_end));
L
Linus Torvalds 已提交
494
}
495 496 497 498 499 500 501

unsigned long pgd_current[NR_CPUS];
/*
 * On 64-bit we've got three-level pagetables with a slightly
 * different layout ...
 */
#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
R
Ralf Baechle 已提交
502 503 504 505 506 507 508

/*
 * gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER
 * are constants.  So we use the variants from asm-offset.h until that gcc
 * will officially be retired.
 */
pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
509 510 511 512 513 514 515
#ifdef CONFIG_64BIT
#ifdef MODULE_START
pgd_t module_pg_dir[PTRS_PER_PGD] __page_aligned(PGD_ORDER);
#endif
pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
#endif
pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);