init.c 12.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994 - 2000 Ralf Baechle
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
 * Copyright (C) 2000 MIPS Technologies, Inc.  All rights reserved.
 */
11
#include <linux/bug.h>
L
Linus Torvalds 已提交
12 13 14 15
#include <linux/init.h>
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/sched.h>
16
#include <linux/smp.h>
L
Linus Torvalds 已提交
17 18 19 20 21 22 23 24 25 26 27
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/pagemap.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <linux/swap.h>
28
#include <linux/proc_fs.h>
D
Dave Hansen 已提交
29
#include <linux/pfn.h>
30
#include <linux/hardirq.h>
L
Linus Torvalds 已提交
31

R
Ralf Baechle 已提交
32
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
33 34 35 36
#include <asm/bootinfo.h>
#include <asm/cachectl.h>
#include <asm/cpu.h>
#include <asm/dma.h>
37
#include <asm/kmap_types.h>
L
Linus Torvalds 已提交
38 39 40 41 42
#include <asm/mmu_context.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
#include <asm/fixmap.h>

/* Atomicity and interruptability */
#ifdef CONFIG_MIPS_MT_SMTC

#include <asm/mipsmtregs.h>

#define ENTER_CRITICAL(flags) \
	{ \
	unsigned int mvpflags; \
	local_irq_save(flags);\
	mvpflags = dvpe()
#define EXIT_CRITICAL(flags) \
	evpe(mvpflags); \
	local_irq_restore(flags); \
	}
#else

#define ENTER_CRITICAL(flags) local_irq_save(flags)
#define EXIT_CRITICAL(flags) local_irq_restore(flags)

#endif /* CONFIG_MIPS_MT_SMTC */
L
Linus Torvalds 已提交
65 66 67 68 69 70 71 72 73 74 75

DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);

/*
 * We have up to 8 empty zeroed pages so we can map one of the right colour
 * when needed.  This is necessary only on R4000 / R4400 SC and MC versions
 * where we have to avoid VCED / VECI exceptions for good performance at
 * any price.  Since page is never written to after the initialization we
 * don't have to care about aliases on other CPUs.
 */
unsigned long empty_zero_page, zero_page_mask;
76
EXPORT_SYMBOL_GPL(empty_zero_page);
L
Linus Torvalds 已提交
77 78 79 80 81 82

/*
 * Not static inline because used by IP27 special magic initialization code
 */
unsigned long setup_zero_pages(void)
{
N
Nick Piggin 已提交
83 84
	unsigned int order;
	unsigned long size;
L
Linus Torvalds 已提交
85 86 87 88 89 90 91 92 93 94 95
	struct page *page;

	if (cpu_has_vce)
		order = 3;
	else
		order = 0;

	empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
	if (!empty_zero_page)
		panic("Oh boy, that early out of memory?");

96
	page = virt_to_page((void *)empty_zero_page);
N
Nick Piggin 已提交
97
	split_page(page, order);
98
	while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {
99
		SetPageReserved(page);
L
Linus Torvalds 已提交
100 101 102 103 104 105 106 107 108
		page++;
	}

	size = PAGE_SIZE << order;
	zero_page_mask = (size - 1) & PAGE_MASK;

	return 1UL << order;
}

109 110 111 112 113 114 115 116 117 118 119 120 121 122
#ifdef CONFIG_MIPS_MT_SMTC
static pte_t *kmap_coherent_pte;
static void __init kmap_coherent_init(void)
{
	unsigned long vaddr;

	/* cache the first coherent kmap pte */
	vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
	kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
}
#else
static inline void kmap_coherent_init(void) {}
#endif

123
void *kmap_coherent(struct page *page, unsigned long addr)
124 125 126 127 128 129 130
{
	enum fixed_addresses idx;
	unsigned long vaddr, flags, entrylo;
	unsigned long old_ctx;
	pte_t pte;
	int tlbidx;

131 132
	BUG_ON(Page_dcache_dirty(page));

133 134 135
	inc_preempt_count();
	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
#ifdef CONFIG_MIPS_MT_SMTC
136 137 138 139
	idx += FIX_N_COLOURS * smp_processor_id() +
		(in_interrupt() ? (FIX_N_COLOURS * NR_CPUS) : 0);
#else
	idx += in_interrupt() ? FIX_N_COLOURS : 0;
140 141 142
#endif
	vaddr = __fix_to_virt(FIX_CMAP_END - idx);
	pte = mk_pte(page, PAGE_KERNEL);
143
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
	entrylo = pte.pte_high;
#else
	entrylo = pte_val(pte) >> 6;
#endif

	ENTER_CRITICAL(flags);
	old_ctx = read_c0_entryhi();
	write_c0_entryhi(vaddr & (PAGE_MASK << 1));
	write_c0_entrylo0(entrylo);
	write_c0_entrylo1(entrylo);
#ifdef CONFIG_MIPS_MT_SMTC
	set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
	/* preload TLB instead of local_flush_tlb_one() */
	mtc0_tlbw_hazard();
	tlb_probe();
	tlb_probe_hazard();
	tlbidx = read_c0_index();
	mtc0_tlbw_hazard();
	if (tlbidx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();
#else
	tlbidx = read_c0_wired();
	write_c0_wired(tlbidx + 1);
	write_c0_index(tlbidx);
	mtc0_tlbw_hazard();
	tlb_write_indexed();
#endif
	tlbw_use_hazard();
	write_c0_entryhi(old_ctx);
	EXIT_CRITICAL(flags);

	return (void*) vaddr;
}

#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))

182
void kunmap_coherent(void)
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
{
#ifndef CONFIG_MIPS_MT_SMTC
	unsigned int wired;
	unsigned long flags, old_ctx;

	ENTER_CRITICAL(flags);
	old_ctx = read_c0_entryhi();
	wired = read_c0_wired() - 1;
	write_c0_wired(wired);
	write_c0_index(wired);
	write_c0_entryhi(UNIQUE_ENTRYHI(wired));
	write_c0_entrylo0(0);
	write_c0_entrylo1(0);
	mtc0_tlbw_hazard();
	tlb_write_indexed();
	tlbw_use_hazard();
	write_c0_entryhi(old_ctx);
	EXIT_CRITICAL(flags);
#endif
	dec_preempt_count();
	preempt_check_resched();
}

206 207 208 209 210 211
void copy_user_highpage(struct page *to, struct page *from,
	unsigned long vaddr, struct vm_area_struct *vma)
{
	void *vfrom, *vto;

	vto = kmap_atomic(to, KM_USER1);
212 213
	if (cpu_has_dc_aliases &&
	    page_mapped(from) && !Page_dcache_dirty(from)) {
214 215
		vfrom = kmap_coherent(from, vaddr);
		copy_page(vto, vfrom);
216
		kunmap_coherent();
217 218 219 220 221
	} else {
		vfrom = kmap_atomic(from, KM_USER0);
		copy_page(vto, vfrom);
		kunmap_atomic(vfrom, KM_USER0);
	}
222
	if ((!cpu_has_ic_fills_f_dc) ||
223 224 225 226 227 228 229
	    pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
		flush_data_cache_page((unsigned long)vto);
	kunmap_atomic(vto, KM_USER1);
	/* Make sure this page is cleared on other CPU's too before using it */
	smp_wmb();
}

230 231 232 233
void copy_to_user_page(struct vm_area_struct *vma,
	struct page *page, unsigned long vaddr, void *dst, const void *src,
	unsigned long len)
{
234 235
	if (cpu_has_dc_aliases &&
	    page_mapped(page) && !Page_dcache_dirty(page)) {
236 237
		void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
		memcpy(vto, src, len);
238
		kunmap_coherent();
239
	} else {
240
		memcpy(dst, src, len);
241 242 243
		if (cpu_has_dc_aliases)
			SetPageDcacheDirty(page);
	}
244 245 246 247 248 249 250 251
	if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
		flush_cache_page(vma, vaddr, page_to_pfn(page));
}

void copy_from_user_page(struct vm_area_struct *vma,
	struct page *page, unsigned long vaddr, void *dst, const void *src,
	unsigned long len)
{
252 253
	if (cpu_has_dc_aliases &&
	    page_mapped(page) && !Page_dcache_dirty(page)) {
254
		void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
255
		memcpy(dst, vfrom, len);
256
		kunmap_coherent();
257
	} else {
258
		memcpy(dst, src, len);
259 260 261
		if (cpu_has_dc_aliases)
			SetPageDcacheDirty(page);
	}
262 263
}

264
void __init fixrange_init(unsigned long start, unsigned long end,
L
Linus Torvalds 已提交
265 266
	pgd_t *pgd_base)
{
267
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
L
Linus Torvalds 已提交
268
	pgd_t *pgd;
269
	pud_t *pud;
L
Linus Torvalds 已提交
270 271
	pmd_t *pmd;
	pte_t *pte;
272
	int i, j, k;
L
Linus Torvalds 已提交
273 274 275 276
	unsigned long vaddr;

	vaddr = start;
	i = __pgd_offset(vaddr);
277 278
	j = __pud_offset(vaddr);
	k = __pmd_offset(vaddr);
L
Linus Torvalds 已提交
279 280 281
	pgd = pgd_base + i;

	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
282 283 284 285 286 287
		pud = (pud_t *)pgd;
		for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
			pmd = (pmd_t *)pud;
			for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
				if (pmd_none(*pmd)) {
					pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
288
					set_pmd(pmd, __pmd((unsigned long)pte));
289
					BUG_ON(pte != pte_offset_kernel(pmd, 0));
290 291
				}
				vaddr += PMD_SIZE;
L
Linus Torvalds 已提交
292
			}
293
			k = 0;
L
Linus Torvalds 已提交
294 295 296
		}
		j = 0;
	}
297
#endif
L
Linus Torvalds 已提交
298 299
}

300
#ifndef CONFIG_NEED_MULTIPLE_NODES
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
static int __init page_is_ram(unsigned long pagenr)
{
	int i;

	for (i = 0; i < boot_mem_map.nr_map; i++) {
		unsigned long addr, end;

		if (boot_mem_map.map[i].type != BOOT_MEM_RAM)
			/* not usable memory */
			continue;

		addr = PFN_UP(boot_mem_map.map[i].addr);
		end = PFN_DOWN(boot_mem_map.map[i].addr +
			       boot_mem_map.map[i].size);

		if (pagenr >= addr && pagenr < end)
			return 1;
	}

	return 0;
}

L
Linus Torvalds 已提交
323 324
void __init paging_init(void)
{
325 326
	unsigned long max_zone_pfns[MAX_NR_ZONES];
	unsigned long lastpfn;
L
Linus Torvalds 已提交
327 328 329 330 331 332

	pagetable_init();

#ifdef CONFIG_HIGHMEM
	kmap_init();
#endif
333
	kmap_coherent_init();
L
Linus Torvalds 已提交
334

335
#ifdef CONFIG_ZONE_DMA
336
	max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
L
Linus Torvalds 已提交
337
#endif
338 339 340 341 342
#ifdef CONFIG_ZONE_DMA32
	max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
#endif
	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
	lastpfn = max_low_pfn;
L
Linus Torvalds 已提交
343
#ifdef CONFIG_HIGHMEM
344 345
	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
	lastpfn = highend_pfn;
346

347
	if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
348
		printk(KERN_WARNING "This processor doesn't support highmem."
349 350 351 352
		       " %ldk highmem ignored\n",
		       (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
		max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
		lastpfn = max_low_pfn;
353
	}
L
Linus Torvalds 已提交
354 355
#endif

356
	free_area_init_nodes(max_zone_pfns);
L
Linus Torvalds 已提交
357 358
}

359 360 361 362
#ifdef CONFIG_64BIT
static struct kcore_list kcore_kseg0;
#endif

L
Linus Torvalds 已提交
363 364 365 366 367 368 369 370 371
void __init mem_init(void)
{
	unsigned long codesize, reservedpages, datasize, initsize;
	unsigned long tmp, ram;

#ifdef CONFIG_HIGHMEM
#ifdef CONFIG_DISCONTIGMEM
#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
#endif
372
	max_mapnr = highend_pfn;
L
Linus Torvalds 已提交
373
#else
374
	max_mapnr = max_low_pfn;
L
Linus Torvalds 已提交
375 376 377 378 379 380 381 382 383 384
#endif
	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);

	totalram_pages += free_all_bootmem();
	totalram_pages -= setup_zero_pages();	/* Setup zeroed pages.  */

	reservedpages = ram = 0;
	for (tmp = 0; tmp < max_low_pfn; tmp++)
		if (page_is_ram(tmp)) {
			ram++;
C
Chad Reese 已提交
385
			if (PageReserved(pfn_to_page(tmp)))
L
Linus Torvalds 已提交
386 387
				reservedpages++;
		}
388
	num_physpages = ram;
L
Linus Torvalds 已提交
389 390 391

#ifdef CONFIG_HIGHMEM
	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
392
		struct page *page = pfn_to_page(tmp);
L
Linus Torvalds 已提交
393 394 395 396 397 398

		if (!page_is_ram(tmp)) {
			SetPageReserved(page);
			continue;
		}
		ClearPageReserved(page);
399
		init_page_count(page);
L
Linus Torvalds 已提交
400 401 402 403
		__free_page(page);
		totalhigh_pages++;
	}
	totalram_pages += totalhigh_pages;
404
	num_physpages += totalhigh_pages;
L
Linus Torvalds 已提交
405 406 407 408 409 410
#endif

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

411 412 413 414
#ifdef CONFIG_64BIT
	if ((unsigned long) &_text > (unsigned long) CKSEG0)
		/* The -4 is a hack so that user tools don't have to handle
		   the overflow.  */
K
KAMEZAWA Hiroyuki 已提交
415 416
		kclist_add(&kcore_kseg0, (void *) CKSEG0,
				0x80000000 - 4, KCORE_TEXT);
417 418
#endif

L
Linus Torvalds 已提交
419 420
	printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
	       "%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
421
	       nr_free_pages() << (PAGE_SHIFT-10),
L
Linus Torvalds 已提交
422 423 424 425 426
	       ram << (PAGE_SHIFT-10),
	       codesize >> 10,
	       reservedpages << (PAGE_SHIFT-10),
	       datasize >> 10,
	       initsize >> 10,
427
	       totalhigh_pages << (PAGE_SHIFT-10));
L
Linus Torvalds 已提交
428
}
429
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
L
Linus Torvalds 已提交
430

431
void free_init_pages(const char *what, unsigned long begin, unsigned long end)
432
{
433
	unsigned long pfn;
434

435 436 437 438 439 440 441 442
	for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
		struct page *page = pfn_to_page(pfn);
		void *addr = phys_to_virt(PFN_PHYS(pfn));

		ClearPageReserved(page);
		init_page_count(page);
		memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
		__free_page(page);
443 444 445 446 447
		totalram_pages++;
	}
	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
}

L
Linus Torvalds 已提交
448 449 450
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
451 452 453
	free_init_pages("initrd memory",
			virt_to_phys((void *)start),
			virt_to_phys((void *)end));
L
Linus Torvalds 已提交
454 455 456
}
#endif

457
void __init_refok free_initmem(void)
L
Linus Torvalds 已提交
458
{
459
	prom_free_prom_memory();
460 461 462
	free_init_pages("unused kernel memory",
			__pa_symbol(&__init_begin),
			__pa_symbol(&__init_end));
L
Linus Torvalds 已提交
463
}
464

465
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
466
unsigned long pgd_current[NR_CPUS];
467
#endif
468 469 470 471 472
/*
 * On 64-bit we've got three-level pagetables with a slightly
 * different layout ...
 */
#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
R
Ralf Baechle 已提交
473 474 475 476 477 478 479

/*
 * gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER
 * are constants.  So we use the variants from asm-offset.h until that gcc
 * will officially be retired.
 */
pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
480
#ifndef __PAGETABLE_PMD_FOLDED
481 482 483
pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
#endif
pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);