init.c 46.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*  $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
 *  arch/sparc64/mm/init.c
 *
 *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
 *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
 */
 
8
#include <linux/module.h>
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/initrd.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
20
#include <linux/poison.h>
L
Linus Torvalds 已提交
21 22
#include <linux/fs.h>
#include <linux/seq_file.h>
23
#include <linux/kprobes.h>
24
#include <linux/cache.h>
25
#include <linux/sort.h>
26
#include <linux/percpu.h>
27
#include <linux/lmb.h>
L
Linus Torvalds 已提交
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44

#include <asm/head.h>
#include <asm/system.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/oplib.h>
#include <asm/iommu.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
#include <asm/dma.h>
#include <asm/starfire.h>
#include <asm/tlb.h>
#include <asm/spitfire.h>
#include <asm/sections.h>
45
#include <asm/tsb.h>
46
#include <asm/hypervisor.h>
47
#include <asm/prom.h>
48
#include <asm/sstate.h>
49
#include <asm/mdesc.h>
50
#include <asm/cpudata.h>
L
Linus Torvalds 已提交
51

52 53 54 55 56 57 58 59 60 61 62 63 64
#define MAX_PHYS_ADDRESS	(1UL << 42UL)
#define KPTE_BITMAP_CHUNK_SZ	(256UL * 1024UL * 1024UL)
#define KPTE_BITMAP_BYTES	\
	((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)

unsigned long kern_linear_pte_xor[2] __read_mostly;

/* A bitmap, one bit for every 256MB of physical memory.  If the bit
 * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else
 * if set we should use a 256MB page (via kern_linear_pte_xor[1]).
 */
unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];

65
#ifndef CONFIG_DEBUG_PAGEALLOC
66 67 68 69 70
/* A special kernel TSB for 4MB and 256MB linear mappings.
 * Space is allocated for this right after the trap table
 * in arch/sparc64/kernel/head.S
 */
extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
71
#endif
72

73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
#define MAX_BANKS	32

static struct linux_prom64_registers pavail[MAX_BANKS] __initdata;
static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
static int pavail_ents __initdata;
static int pavail_rescan_ents __initdata;

static int cmp_p64(const void *a, const void *b)
{
	const struct linux_prom64_registers *x = a, *y = b;

	if (x->phys_addr > y->phys_addr)
		return 1;
	if (x->phys_addr < y->phys_addr)
		return -1;
	return 0;
}

static void __init read_obp_memory(const char *property,
				   struct linux_prom64_registers *regs,
				   int *num_ents)
{
	int node = prom_finddevice("/memory");
	int prop_size = prom_getproplen(node, property);
	int ents, ret, i;

	ents = prop_size / sizeof(struct linux_prom64_registers);
	if (ents > MAX_BANKS) {
		prom_printf("The machine has more %s property entries than "
			    "this kernel can support (%d).\n",
			    property, MAX_BANKS);
		prom_halt();
	}

	ret = prom_getproperty(node, property, (char *) regs, prop_size);
	if (ret == -1) {
		prom_printf("Couldn't get %s property from /memory.\n");
		prom_halt();
	}

	/* Sanitize what we got from the firmware, by page aligning
	 * everything.
	 */
	for (i = 0; i < ents; i++) {
		unsigned long base, size;

		base = regs[i].phys_addr;
		size = regs[i].reg_size;
121

122 123 124 125 126 127 128 129 130
		size &= PAGE_MASK;
		if (base & ~PAGE_MASK) {
			unsigned long new_base = PAGE_ALIGN(base);

			size -= new_base - base;
			if ((long) size < 0L)
				size = 0UL;
			base = new_base;
		}
131 132 133 134 135 136 137
		if (size == 0UL) {
			/* If it is empty, simply get rid of it.
			 * This simplifies the logic of the other
			 * functions that process these arrays.
			 */
			memmove(&regs[i], &regs[i + 1],
				(ents - i - 1) * sizeof(regs[0]));
138
			i--;
139 140
			ents--;
			continue;
141
		}
142 143
		regs[i].phys_addr = base;
		regs[i].reg_size = size;
144 145 146 147
	}

	*num_ents = ents;

148
	sort(regs, ents, sizeof(struct linux_prom64_registers),
149 150
	     cmp_p64, NULL);
}
L
Linus Torvalds 已提交
151

152
unsigned long *sparc64_valid_addr_bitmap __read_mostly;
L
Linus Torvalds 已提交
153

154
/* Kernel physical address base and size in bytes.  */
155 156
unsigned long kern_base __read_mostly;
unsigned long kern_size __read_mostly;
L
Linus Torvalds 已提交
157 158 159 160 161 162

/* Initial ramdisk setup */
extern unsigned long sparc_ramdisk_image64;
extern unsigned int sparc_ramdisk_image;
extern unsigned int sparc_ramdisk_size;

163
struct page *mem_map_zero __read_mostly;
L
Linus Torvalds 已提交
164

165 166 167 168 169 170
unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;

unsigned long sparc64_kern_pri_context __read_mostly;
unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
unsigned long sparc64_kern_sec_context __read_mostly;

171
int num_kernel_image_mappings;
L
Linus Torvalds 已提交
172 173 174 175 176 177 178 179

#ifdef CONFIG_DEBUG_DCFLUSH
atomic_t dcpage_flushes = ATOMIC_INIT(0);
#ifdef CONFIG_SMP
atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
#endif
#endif

180
inline void flush_dcache_page_impl(struct page *page)
L
Linus Torvalds 已提交
181
{
182
	BUG_ON(tlb_type == hypervisor);
L
Linus Torvalds 已提交
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
#ifdef CONFIG_DEBUG_DCFLUSH
	atomic_inc(&dcpage_flushes);
#endif

#ifdef DCACHE_ALIASING_POSSIBLE
	__flush_dcache_page(page_address(page),
			    ((tlb_type == spitfire) &&
			     page_mapping(page) != NULL));
#else
	if (page_mapping(page) != NULL &&
	    tlb_type == spitfire)
		__flush_icache_page(__pa(page_address(page)));
#endif
}

#define PG_dcache_dirty		PG_arch_1
199 200 201
#define PG_dcache_cpu_shift	32UL
#define PG_dcache_cpu_mask	\
	((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
L
Linus Torvalds 已提交
202 203

#define dcache_dirty_cpu(page) \
204
	(((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
L
Linus Torvalds 已提交
205

D
David S. Miller 已提交
206
static inline void set_dcache_dirty(struct page *page, int this_cpu)
L
Linus Torvalds 已提交
207 208
{
	unsigned long mask = this_cpu;
209 210 211 212 213
	unsigned long non_cpu_bits;

	non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
	mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);

L
Linus Torvalds 已提交
214 215 216 217 218 219
	__asm__ __volatile__("1:\n\t"
			     "ldx	[%2], %%g7\n\t"
			     "and	%%g7, %1, %%g1\n\t"
			     "or	%%g1, %0, %%g1\n\t"
			     "casx	[%2], %%g7, %%g1\n\t"
			     "cmp	%%g7, %%g1\n\t"
220
			     "membar	#StoreLoad | #StoreStore\n\t"
L
Linus Torvalds 已提交
221
			     "bne,pn	%%xcc, 1b\n\t"
222
			     " nop"
L
Linus Torvalds 已提交
223 224 225 226 227
			     : /* no outputs */
			     : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
			     : "g1", "g7");
}

D
David S. Miller 已提交
228
static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
L
Linus Torvalds 已提交
229 230 231 232 233 234
{
	unsigned long mask = (1UL << PG_dcache_dirty);

	__asm__ __volatile__("! test_and_clear_dcache_dirty\n"
			     "1:\n\t"
			     "ldx	[%2], %%g7\n\t"
235
			     "srlx	%%g7, %4, %%g1\n\t"
L
Linus Torvalds 已提交
236 237 238 239 240 241
			     "and	%%g1, %3, %%g1\n\t"
			     "cmp	%%g1, %0\n\t"
			     "bne,pn	%%icc, 2f\n\t"
			     " andn	%%g7, %1, %%g1\n\t"
			     "casx	[%2], %%g7, %%g1\n\t"
			     "cmp	%%g7, %%g1\n\t"
242
			     "membar	#StoreLoad | #StoreStore\n\t"
L
Linus Torvalds 已提交
243
			     "bne,pn	%%xcc, 1b\n\t"
244
			     " nop\n"
L
Linus Torvalds 已提交
245 246 247
			     "2:"
			     : /* no outputs */
			     : "r" (cpu), "r" (mask), "r" (&page->flags),
248 249
			       "i" (PG_dcache_cpu_mask),
			       "i" (PG_dcache_cpu_shift)
L
Linus Torvalds 已提交
250 251 252
			     : "g1", "g7");
}

253 254 255 256
static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
{
	unsigned long tsb_addr = (unsigned long) ent;

257
	if (tlb_type == cheetah_plus || tlb_type == hypervisor)
258 259 260 261 262
		tsb_addr = __pa(tsb_addr);

	__tsb_insert(tsb_addr, tag, pte);
}

263 264 265
unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
unsigned long _PAGE_SZBITS __read_mostly;

L
Linus Torvalds 已提交
266 267
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
268
	struct mm_struct *mm;
269
	struct tsb *tsb;
270
	unsigned long tag, flags;
271
	unsigned long tsb_index, tsb_hash_shift;
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296

	if (tlb_type != hypervisor) {
		unsigned long pfn = pte_pfn(pte);
		unsigned long pg_flags;
		struct page *page;

		if (pfn_valid(pfn) &&
		    (page = pfn_to_page(pfn), page_mapping(page)) &&
		    ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
			int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
				   PG_dcache_cpu_mask);
			int this_cpu = get_cpu();

			/* This is just to optimize away some function calls
			 * in the SMP case.
			 */
			if (cpu == this_cpu)
				flush_dcache_page_impl(page);
			else
				smp_flush_dcache_page_impl(page, cpu);

			clear_dcache_dirty_cpu(page, cpu);

			put_cpu();
		}
L
Linus Torvalds 已提交
297
	}
298 299

	mm = vma->vm_mm;
300

301 302 303
	tsb_index = MM_TSB_BASE;
	tsb_hash_shift = PAGE_SHIFT;

304 305
	spin_lock_irqsave(&mm->context.lock, flags);

306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
#ifdef CONFIG_HUGETLB_PAGE
	if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
		if ((tlb_type == hypervisor &&
		     (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
		    (tlb_type != hypervisor &&
		     (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
			tsb_index = MM_TSB_HUGE;
			tsb_hash_shift = HPAGE_SHIFT;
		}
	}
#endif

	tsb = mm->context.tsb_block[tsb_index].tsb;
	tsb += ((address >> tsb_hash_shift) &
		(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
321 322
	tag = (address >> 22UL);
	tsb_insert(tsb, tag, pte_val(pte));
323 324

	spin_unlock_irqrestore(&mm->context.lock, flags);
L
Linus Torvalds 已提交
325 326 327 328
}

void flush_dcache_page(struct page *page)
{
329 330
	struct address_space *mapping;
	int this_cpu;
L
Linus Torvalds 已提交
331

332 333 334
	if (tlb_type == hypervisor)
		return;

335 336 337 338 339 340 341 342 343 344
	/* Do not bother with the expensive D-cache flush if it
	 * is merely the zero page.  The 'bigcore' testcase in GDB
	 * causes this case to run millions of times.
	 */
	if (page == ZERO_PAGE(0))
		return;

	this_cpu = get_cpu();

	mapping = page_mapping(page);
L
Linus Torvalds 已提交
345
	if (mapping && !mapping_mapped(mapping)) {
346
		int dirty = test_bit(PG_dcache_dirty, &page->flags);
L
Linus Torvalds 已提交
347
		if (dirty) {
348 349
			int dirty_cpu = dcache_dirty_cpu(page);

L
Linus Torvalds 已提交
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
			if (dirty_cpu == this_cpu)
				goto out;
			smp_flush_dcache_page_impl(page, dirty_cpu);
		}
		set_dcache_dirty(page, this_cpu);
	} else {
		/* We could delay the flush for the !page_mapping
		 * case too.  But that case is for exec env/arg
		 * pages and those are %99 certainly going to get
		 * faulted into the tlb (and thus flushed) anyways.
		 */
		flush_dcache_page_impl(page);
	}

out:
	put_cpu();
}

368
void __kprobes flush_icache_range(unsigned long start, unsigned long end)
L
Linus Torvalds 已提交
369
{
370
	/* Cheetah and Hypervisor platform cpus have coherent I-cache. */
L
Linus Torvalds 已提交
371 372 373
	if (tlb_type == spitfire) {
		unsigned long kaddr;

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
		/* This code only runs on Spitfire cpus so this is
		 * why we can assume _PAGE_PADDR_4U.
		 */
		for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
			unsigned long paddr, mask = _PAGE_PADDR_4U;

			if (kaddr >= PAGE_OFFSET)
				paddr = kaddr & mask;
			else {
				pgd_t *pgdp = pgd_offset_k(kaddr);
				pud_t *pudp = pud_offset(pgdp, kaddr);
				pmd_t *pmdp = pmd_offset(pudp, kaddr);
				pte_t *ptep = pte_offset_kernel(pmdp, kaddr);

				paddr = pte_val(*ptep) & mask;
			}
			__flush_icache_page(paddr);
		}
L
Linus Torvalds 已提交
392 393 394 395 396
	}
}

void show_mem(void)
{
397 398 399 400
	unsigned long total = 0, reserved = 0;
	unsigned long shared = 0, cached = 0;
	pg_data_t *pgdat;

401
	printk(KERN_INFO "Mem-info:\n");
L
Linus Torvalds 已提交
402
	show_free_areas();
403
	printk(KERN_INFO "Free swap:       %6ldkB\n",
L
Linus Torvalds 已提交
404
	       nr_swap_pages << (PAGE_SHIFT-10));
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
	for_each_online_pgdat(pgdat) {
		unsigned long i, flags;

		pgdat_resize_lock(pgdat, &flags);
		for (i = 0; i < pgdat->node_spanned_pages; i++) {
			struct page *page = pgdat_page_nr(pgdat, i);
			total++;
			if (PageReserved(page))
				reserved++;
			else if (PageSwapCache(page))
				cached++;
			else if (page_count(page))
				shared += page_count(page) - 1;
		}
		pgdat_resize_unlock(pgdat, &flags);
	}

	printk(KERN_INFO "%lu pages of RAM\n", total);
	printk(KERN_INFO "%lu reserved pages\n", reserved);
	printk(KERN_INFO "%lu pages shared\n", shared);
	printk(KERN_INFO "%lu pages swap cached\n", cached);

	printk(KERN_INFO "%lu pages dirty\n",
	       global_page_state(NR_FILE_DIRTY));
	printk(KERN_INFO "%lu pages writeback\n",
	       global_page_state(NR_WRITEBACK));
	printk(KERN_INFO "%lu pages mapped\n",
	       global_page_state(NR_FILE_MAPPED));
	printk(KERN_INFO "%lu pages slab\n",
		global_page_state(NR_SLAB_RECLAIMABLE) +
		global_page_state(NR_SLAB_UNRECLAIMABLE));
	printk(KERN_INFO "%lu pages pagetables\n",
	       global_page_state(NR_PAGETABLE));
L
Linus Torvalds 已提交
438 439 440 441 442 443 444 445 446 447
}

void mmu_info(struct seq_file *m)
{
	if (tlb_type == cheetah)
		seq_printf(m, "MMU Type\t: Cheetah\n");
	else if (tlb_type == cheetah_plus)
		seq_printf(m, "MMU Type\t: Cheetah+\n");
	else if (tlb_type == spitfire)
		seq_printf(m, "MMU Type\t: Spitfire\n");
448 449
	else if (tlb_type == hypervisor)
		seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
L
Linus Torvalds 已提交
450 451 452 453 454 455 456 457 458 459 460 461 462
	else
		seq_printf(m, "MMU Type\t: ???\n");

#ifdef CONFIG_DEBUG_DCFLUSH
	seq_printf(m, "DCPageFlushes\t: %d\n",
		   atomic_read(&dcpage_flushes));
#ifdef CONFIG_SMP
	seq_printf(m, "DCPageFlushesXC\t: %d\n",
		   atomic_read(&dcpage_flushes_xcall));
#endif /* CONFIG_SMP */
#endif /* CONFIG_DEBUG_DCFLUSH */
}

463 464 465 466 467 468 469 470 471 472
struct linux_prom_translation {
	unsigned long virt;
	unsigned long size;
	unsigned long data;
};

/* Exported for kernel TLB miss handling in ktlb.S */
struct linux_prom_translation prom_trans[512] __read_mostly;
unsigned int prom_trans_ents __read_mostly;

L
Linus Torvalds 已提交
473 474 475
/* Exported for SMP bootup purposes. */
unsigned long kern_locked_tte_data;

476 477
/* The obp translations are saved based on 8k pagesize, since obp can
 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
478
 * HI_OBP_ADDRESS range are handled in ktlb.S.
479
 */
480 481 482 483 484 485
static inline int in_obp_range(unsigned long vaddr)
{
	return (vaddr >= LOW_OBP_ADDRESS &&
		vaddr < HI_OBP_ADDRESS);
}

486
static int cmp_ptrans(const void *a, const void *b)
487
{
488
	const struct linux_prom_translation *x = a, *y = b;
489

490 491 492 493 494
	if (x->virt > y->virt)
		return 1;
	if (x->virt < y->virt)
		return -1;
	return 0;
495 496
}

497
/* Read OBP translations property into 'prom_trans[]'.  */
498
static void __init read_obp_translations(void)
499
{
500
	int n, node, ents, first, last, i;
L
Linus Torvalds 已提交
501 502 503

	node = prom_finddevice("/virtual-memory");
	n = prom_getproplen(node, "translations");
504
	if (unlikely(n == 0 || n == -1)) {
505
		prom_printf("prom_mappings: Couldn't get size.\n");
L
Linus Torvalds 已提交
506 507
		prom_halt();
	}
508 509
	if (unlikely(n > sizeof(prom_trans))) {
		prom_printf("prom_mappings: Size %Zd is too big.\n", n);
L
Linus Torvalds 已提交
510 511
		prom_halt();
	}
512

513
	if ((n = prom_getproperty(node, "translations",
514 515
				  (char *)&prom_trans[0],
				  sizeof(prom_trans))) == -1) {
516
		prom_printf("prom_mappings: Couldn't get property.\n");
L
Linus Torvalds 已提交
517 518
		prom_halt();
	}
519

520
	n = n / sizeof(struct linux_prom_translation);
521

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
	ents = n;

	sort(prom_trans, ents, sizeof(struct linux_prom_translation),
	     cmp_ptrans, NULL);

	/* Now kick out all the non-OBP entries.  */
	for (i = 0; i < ents; i++) {
		if (in_obp_range(prom_trans[i].virt))
			break;
	}
	first = i;
	for (; i < ents; i++) {
		if (!in_obp_range(prom_trans[i].virt))
			break;
	}
	last = i;

	for (i = 0; i < (last - first); i++) {
		struct linux_prom_translation *src = &prom_trans[i + first];
		struct linux_prom_translation *dest = &prom_trans[i];

		*dest = *src;
	}
	for (; i < ents; i++) {
		struct linux_prom_translation *dest = &prom_trans[i];
		dest->virt = dest->size = dest->data = 0x0UL;
	}

	prom_trans_ents = last - first;

	if (tlb_type == spitfire) {
		/* Clear diag TTE bits. */
		for (i = 0; i < prom_trans_ents; i++)
			prom_trans[i].data &= ~0x0003fe0000000000UL;
	}
557
}
L
Linus Torvalds 已提交
558

559 560 561 562
static void __init hypervisor_tlb_lock(unsigned long vaddr,
				       unsigned long pte,
				       unsigned long mmu)
{
563 564 565
	unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);

	if (ret != 0) {
566
		prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
567
			    "errors with %lx\n", vaddr, 0, pte, mmu, ret);
568 569
		prom_halt();
	}
570 571
}

572 573
static unsigned long kern_large_tte(unsigned long paddr);

574
static void __init remap_kernel(void)
575 576
{
	unsigned long phys_page, tte_vaddr, tte_data;
577
	int i, tlb_ent = sparc64_highest_locked_tlbent();
578

L
Linus Torvalds 已提交
579
	tte_vaddr = (unsigned long) KERNBASE;
580
	phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
581
	tte_data = kern_large_tte(phys_page);
L
Linus Torvalds 已提交
582 583 584

	kern_locked_tte_data = tte_data;

585 586
	/* Now lock us into the TLBs via Hypervisor or OBP. */
	if (tlb_type == hypervisor) {
587
		for (i = 0; i < num_kernel_image_mappings; i++) {
588 589
			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
590 591
			tte_vaddr += 0x400000;
			tte_data += 0x400000;
592 593
		}
	} else {
594 595 596 597 598
		for (i = 0; i < num_kernel_image_mappings; i++) {
			prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
			prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
			tte_vaddr += 0x400000;
			tte_data += 0x400000;
599
		}
600
		sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
L
Linus Torvalds 已提交
601
	}
602 603 604 605 606 607
	if (tlb_type == cheetah_plus) {
		sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
					    CTX_CHEETAH_PLUS_NUC);
		sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
		sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
	}
608
}
L
Linus Torvalds 已提交
609

610

611
static void __init inherit_prom_mappings(void)
612 613
{
	read_obp_translations();
614 615

	/* Now fixup OBP's idea about where we really are mapped. */
616
	printk("Remapping the kernel... ");
617
	remap_kernel();
618
	printk("done.\n");
L
Linus Torvalds 已提交
619 620 621 622 623 624 625
}

void prom_world(int enter)
{
	if (!enter)
		set_fs((mm_segment_t) { get_thread_current_ds() });

626
	__asm__ __volatile__("flushw");
L
Linus Torvalds 已提交
627 628 629 630 631 632 633 634 635 636 637 638 639 640
}

void __flush_dcache_range(unsigned long start, unsigned long end)
{
	unsigned long va;

	if (tlb_type == spitfire) {
		int n = 0;

		for (va = start; va < end; va += 32) {
			spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
			if (++n >= 512)
				break;
		}
641
	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
L
Linus Torvalds 已提交
642 643 644 645 646 647 648 649 650 651 652
		start = __pa(start);
		end = __pa(end);
		for (va = start; va < end; va += 32)
			__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
					     "membar #Sync"
					     : /* no outputs */
					     : "r" (va),
					       "i" (ASI_DCACHE_INVALIDATE));
	}
}

653 654 655 656 657 658 659
/* get_new_mmu_context() uses "cache + 1".  */
DEFINE_SPINLOCK(ctx_alloc_lock);
unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
#define MAX_CTX_NR	(1UL << CTX_NR_BITS)
#define CTX_BMAP_SLOTS	BITS_TO_LONGS(MAX_CTX_NR)
DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);

L
Linus Torvalds 已提交
660 661 662 663 664 665 666
/* Caller does TLB context flushing on local CPU if necessary.
 * The caller also ensures that CTX_VALID(mm->context) is false.
 *
 * We must be careful about boundary cases so that we never
 * let the user have CTX 0 (nucleus) or we ever use a CTX
 * version of zero (and thus NO_CONTEXT would not be caught
 * by version mis-match tests in mmu_context.h).
667 668
 *
 * Always invoked with interrupts disabled.
L
Linus Torvalds 已提交
669 670 671 672 673
 */
void get_new_mmu_context(struct mm_struct *mm)
{
	unsigned long ctx, new_ctx;
	unsigned long orig_pgsz_bits;
674
	unsigned long flags;
675
	int new_version;
L
Linus Torvalds 已提交
676

677
	spin_lock_irqsave(&ctx_alloc_lock, flags);
L
Linus Torvalds 已提交
678 679 680
	orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
	ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
	new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
681
	new_version = 0;
L
Linus Torvalds 已提交
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
	if (new_ctx >= (1 << CTX_NR_BITS)) {
		new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
		if (new_ctx >= ctx) {
			int i;
			new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
				CTX_FIRST_VERSION;
			if (new_ctx == 1)
				new_ctx = CTX_FIRST_VERSION;

			/* Don't call memset, for 16 entries that's just
			 * plain silly...
			 */
			mmu_context_bmap[0] = 3;
			mmu_context_bmap[1] = 0;
			mmu_context_bmap[2] = 0;
			mmu_context_bmap[3] = 0;
			for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
				mmu_context_bmap[i + 0] = 0;
				mmu_context_bmap[i + 1] = 0;
				mmu_context_bmap[i + 2] = 0;
				mmu_context_bmap[i + 3] = 0;
			}
704
			new_version = 1;
L
Linus Torvalds 已提交
705 706 707 708 709 710 711 712
			goto out;
		}
	}
	mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
	new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
out:
	tlb_context_cache = new_ctx;
	mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
713
	spin_unlock_irqrestore(&ctx_alloc_lock, flags);
714 715 716

	if (unlikely(new_version))
		smp_new_mmu_context_version();
L
Linus Torvalds 已提交
717 718
}

719 720 721 722 723
/* Find a free area for the bootmem map, avoiding the kernel image
 * and the initial ramdisk.
 */
static unsigned long __init choose_bootmap_pfn(unsigned long start_pfn,
					       unsigned long end_pfn)
L
Linus Torvalds 已提交
724
{
725
	unsigned long bootmap_size;
726

727 728
	bootmap_size = bootmem_bootmap_pages(end_pfn - start_pfn);
	bootmap_size <<= PAGE_SHIFT;
729

730
	return lmb_alloc(bootmap_size, PAGE_SIZE) >> PAGE_SHIFT;
731 732
}

733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
static void __init find_ramdisk(unsigned long phys_base)
{
#ifdef CONFIG_BLK_DEV_INITRD
	if (sparc_ramdisk_image || sparc_ramdisk_image64) {
		unsigned long ramdisk_image;

		/* Older versions of the bootloader only supported a
		 * 32-bit physical address for the ramdisk image
		 * location, stored at sparc_ramdisk_image.  Newer
		 * SILO versions set sparc_ramdisk_image to zero and
		 * provide a full 64-bit physical address at
		 * sparc_ramdisk_image64.
		 */
		ramdisk_image = sparc_ramdisk_image;
		if (!ramdisk_image)
			ramdisk_image = sparc_ramdisk_image64;

		/* Another bootloader quirk.  The bootloader normalizes
		 * the physical address to KERNBASE, so we have to
		 * factor that back out and add in the lowest valid
		 * physical page address to get the true physical address.
		 */
		ramdisk_image -= KERNBASE;
		ramdisk_image += phys_base;

		initrd_start = ramdisk_image;
		initrd_end = ramdisk_image + sparc_ramdisk_size;
760 761

		lmb_reserve(initrd_start, initrd_end);
762 763 764 765
	}
#endif
}

766 767 768 769 770 771 772 773 774 775 776 777 778 779
/* About pages_avail, this is the value we will use to calculate
 * the zholes_size[] argument given to free_area_init_node().  The
 * page allocator uses this to calculate nr_kernel_pages,
 * nr_all_pages and zone->present_pages.  On NUMA it is used
 * to calculate zone->min_unmapped_pages and zone->min_slab_pages.
 *
 * So this number should really be set to what the page allocator
 * actually ends up with.  This means:
 * 1) It should include bootmem map pages, we'll release those.
 * 2) It should not include the kernel image, except for the
 *    __init sections which we will also release.
 * 3) It should include the initrd image, since we'll release
 *    that too.
 */
780 781 782
static unsigned long __init bootmem_init(unsigned long *pages_avail,
					 unsigned long phys_base)
{
783
	unsigned long end_pfn;
L
Linus Torvalds 已提交
784 785
	int i;

786 787
	*pages_avail = lmb_phys_mem_size() >> PAGE_SHIFT;
	end_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
788

L
Linus Torvalds 已提交
789 790
	/* Initialize the boot-time allocator. */
	max_pfn = max_low_pfn = end_pfn;
791 792
	min_low_pfn = (phys_base >> PAGE_SHIFT);

793 794 795
	init_bootmem_node(NODE_DATA(0),
			  choose_bootmap_pfn(min_low_pfn, end_pfn),
			  min_low_pfn, end_pfn);
L
Linus Torvalds 已提交
796 797 798 799

	/* Now register the available physical memory with the
	 * allocator.
	 */
800 801 802
	for (i = 0; i < lmb.memory.cnt; i++)
		free_bootmem(lmb.memory.region[i].base,
			     lmb_size_bytes(&lmb.memory, i));
L
Linus Torvalds 已提交
803

804 805 806 807
	for (i = 0; i < lmb.reserved.cnt; i++)
		reserve_bootmem(lmb.reserved.region[i].base,
				lmb_size_bytes(&lmb.reserved, i),
				BOOTMEM_DEFAULT);
L
Linus Torvalds 已提交
808 809 810

	*pages_avail -= PAGE_ALIGN(kern_size) >> PAGE_SHIFT;

811 812
	for (i = 0; i < lmb.memory.cnt; ++i) {
		unsigned long start_pfn, end_pfn, pages;
L
Linus Torvalds 已提交
813

814 815 816
		pages = lmb_size_pages(&lmb.memory, i);
		start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
		end_pfn = start_pfn + pages;
817 818 819 820 821 822

		memory_present(0, start_pfn, end_pfn);
	}

	sparse_init();

L
Linus Torvalds 已提交
823 824 825
	return end_pfn;
}

826 827 828
static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
static int pall_ents __initdata;

829
#ifdef CONFIG_DEBUG_PAGEALLOC
830 831
static unsigned long __ref kernel_map_range(unsigned long pstart,
					    unsigned long pend, pgprot_t prot)
832 833 834 835 836 837
{
	unsigned long vstart = PAGE_OFFSET + pstart;
	unsigned long vend = PAGE_OFFSET + pend;
	unsigned long alloc_bytes = 0UL;

	if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
838
		prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
			    vstart, vend);
		prom_halt();
	}

	while (vstart < vend) {
		unsigned long this_end, paddr = __pa(vstart);
		pgd_t *pgd = pgd_offset_k(vstart);
		pud_t *pud;
		pmd_t *pmd;
		pte_t *pte;

		pud = pud_offset(pgd, vstart);
		if (pud_none(*pud)) {
			pmd_t *new;

			new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
			alloc_bytes += PAGE_SIZE;
			pud_populate(&init_mm, pud, new);
		}

		pmd = pmd_offset(pud, vstart);
		if (!pmd_present(*pmd)) {
			pte_t *new;

			new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
			alloc_bytes += PAGE_SIZE;
			pmd_populate_kernel(&init_mm, pmd, new);
		}

		pte = pte_offset_kernel(pmd, vstart);
		this_end = (vstart + PMD_SIZE) & PMD_MASK;
		if (this_end > vend)
			this_end = vend;

		while (vstart < this_end) {
			pte_val(*pte) = (paddr | pgprot_val(prot));

			vstart += PAGE_SIZE;
			paddr += PAGE_SIZE;
			pte++;
		}
	}

	return alloc_bytes;
}

extern unsigned int kvmap_linear_patch[1];
886 887 888 889 890 891 892 893 894 895 896
#endif /* CONFIG_DEBUG_PAGEALLOC */

static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
{
	const unsigned long shift_256MB = 28;
	const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL);
	const unsigned long size_256MB = (1UL << shift_256MB);

	while (start < end) {
		long remains;

897 898 899 900
		remains = end - start;
		if (remains < size_256MB)
			break;

901 902 903 904 905 906 907 908 909 910 911 912 913 914 915
		if (start & mask_256MB) {
			start = (start + size_256MB) & ~mask_256MB;
			continue;
		}

		while (remains >= size_256MB) {
			unsigned long index = start >> shift_256MB;

			__set_bit(index, kpte_linear_bitmap);

			start += size_256MB;
			remains -= size_256MB;
		}
	}
}
916

917
static void __init init_kpte_bitmap(void)
918
{
919
	unsigned long i;
920 921

	for (i = 0; i < pall_ents; i++) {
922 923
		unsigned long phys_start, phys_end;

924 925
		phys_start = pall[i].phys_addr;
		phys_end = phys_start + pall[i].reg_size;
926 927

		mark_kpte_bitmap(phys_start, phys_end);
928 929
	}
}
930

931 932
static void __init kernel_physical_mapping_init(void)
{
933
#ifdef CONFIG_DEBUG_PAGEALLOC
934 935 936 937 938 939 940 941
	unsigned long i, mem_alloced = 0UL;

	for (i = 0; i < pall_ents; i++) {
		unsigned long phys_start, phys_end;

		phys_start = pall[i].phys_addr;
		phys_end = phys_start + pall[i].reg_size;

942 943 944 945 946 947 948 949 950 951 952
		mem_alloced += kernel_map_range(phys_start, phys_end,
						PAGE_KERNEL);
	}

	printk("Allocated %ld bytes for kernel page tables.\n",
	       mem_alloced);

	kvmap_linear_patch[0] = 0x01000000; /* nop */
	flushi(&kvmap_linear_patch[0]);

	__flush_tlb_all();
953
#endif
954 955
}

956
#ifdef CONFIG_DEBUG_PAGEALLOC
957 958 959 960 961 962 963 964
void kernel_map_pages(struct page *page, int numpages, int enable)
{
	unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
	unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);

	kernel_map_range(phys_start, phys_end,
			 (enable ? PAGE_KERNEL : __pgprot(0)));

965 966 967
	flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
			       PAGE_OFFSET + phys_end);

968 969 970 971 972 973 974 975
	/* we should perform an IPI and flush all tlbs,
	 * but that can deadlock->flush only current cpu.
	 */
	__flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
				 PAGE_OFFSET + phys_end);
}
#endif

976 977
unsigned long __init find_ecache_flush_span(unsigned long size)
{
978 979
	int i;

980 981 982
	for (i = 0; i < pavail_ents; i++) {
		if (pavail[i].reg_size >= size)
			return pavail[i].phys_addr;
983 984
	}

985
	return ~0UL;
986 987
}

988 989
static void __init tsb_phys_patch(void)
{
990
	struct tsb_ldquad_phys_patch_entry *pquad;
991 992
	struct tsb_phys_patch_entry *p;

993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
	pquad = &__tsb_ldquad_phys_patch;
	while (pquad < &__tsb_ldquad_phys_patch_end) {
		unsigned long addr = pquad->addr;

		if (tlb_type == hypervisor)
			*(unsigned int *) addr = pquad->sun4v_insn;
		else
			*(unsigned int *) addr = pquad->sun4u_insn;
		wmb();
		__asm__ __volatile__("flush	%0"
				     : /* no outputs */
				     : "r" (addr));

		pquad++;
	}

1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
	p = &__tsb_phys_patch;
	while (p < &__tsb_phys_patch_end) {
		unsigned long addr = p->addr;

		*(unsigned int *) addr = p->insn;
		wmb();
		__asm__ __volatile__("flush	%0"
				     : /* no outputs */
				     : "r" (addr));

		p++;
	}
}

1023
/* Don't mark as init, we give this to the Hypervisor.  */
1024 1025 1026 1027 1028 1029
#ifndef CONFIG_DEBUG_PAGEALLOC
#define NUM_KTSB_DESCR	2
#else
#define NUM_KTSB_DESCR	1
#endif
static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1030 1031 1032 1033 1034 1035
extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];

static void __init sun4v_ktsb_init(void)
{
	unsigned long ktsb_pa;

1036
	/* First KTSB for PAGE_SIZE mappings.  */
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
	ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);

	switch (PAGE_SIZE) {
	case 8 * 1024:
	default:
		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
		break;

	case 64 * 1024:
		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
		break;

	case 512 * 1024:
		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
		break;

	case 4 * 1024 * 1024:
		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
		break;
	};

1062
	ktsb_descr[0].assoc = 1;
1063 1064 1065 1066 1067
	ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
	ktsb_descr[0].ctx_idx = 0;
	ktsb_descr[0].tsb_base = ktsb_pa;
	ktsb_descr[0].resv = 0;

1068
#ifndef CONFIG_DEBUG_PAGEALLOC
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
	/* Second KTSB for 4MB/256MB mappings.  */
	ktsb_pa = (kern_base +
		   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));

	ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
	ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB |
				   HV_PGSZ_MASK_256MB);
	ktsb_descr[1].assoc = 1;
	ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
	ktsb_descr[1].ctx_idx = 0;
	ktsb_descr[1].tsb_base = ktsb_pa;
	ktsb_descr[1].resv = 0;
1081
#endif
1082 1083 1084 1085
}

void __cpuinit sun4v_ktsb_register(void)
{
1086
	unsigned long pa, ret;
1087 1088 1089

	pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);

1090 1091 1092 1093 1094 1095
	ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
	if (ret != 0) {
		prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
			    "errors with %lx\n", pa, ret);
		prom_halt();
	}
1096 1097
}

L
Linus Torvalds 已提交
1098 1099
/* paging_init() sets up the page tables */

1100 1101
extern void central_probe(void);

L
Linus Torvalds 已提交
1102
static unsigned long last_valid_pfn;
1103
pgd_t swapper_pg_dir[2048];
L
Linus Torvalds 已提交
1104

1105 1106 1107
static void sun4u_pgprot_init(void);
static void sun4v_pgprot_init(void);

T
travis@sgi.com 已提交
1108 1109 1110 1111 1112
/* Dummy function */
void __init setup_per_cpu_areas(void)
{
}

L
Linus Torvalds 已提交
1113 1114
void __init paging_init(void)
{
1115
	unsigned long end_pfn, pages_avail, shift, phys_base;
1116 1117
	unsigned long real_end, i;

1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
	/* These build time checkes make sure that the dcache_dirty_cpu()
	 * page->flags usage will work.
	 *
	 * When a page gets marked as dcache-dirty, we store the
	 * cpu number starting at bit 32 in the page->flags.  Also,
	 * functions like clear_dcache_dirty_cpu use the cpu mask
	 * in 13-bit signed-immediate instruction fields.
	 */
	BUILD_BUG_ON(FLAGS_RESERVED != 32);
	BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
		     ilog2(roundup_pow_of_two(NR_CPUS)) > FLAGS_RESERVED);
	BUILD_BUG_ON(NR_CPUS > 4096);

1131 1132 1133
	kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
	kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;

1134 1135
	sstate_booting();

1136
	/* Invalidate both kernel TSBs.  */
1137
	memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
1138
#ifndef CONFIG_DEBUG_PAGEALLOC
1139
	memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
1140
#endif
1141

1142 1143 1144 1145 1146
	if (tlb_type == hypervisor)
		sun4v_pgprot_init();
	else
		sun4u_pgprot_init();

1147 1148
	if (tlb_type == cheetah_plus ||
	    tlb_type == hypervisor)
1149 1150
		tsb_phys_patch();

1151
	if (tlb_type == hypervisor) {
1152
		sun4v_patch_tlb_handlers();
1153 1154
		sun4v_ktsb_init();
	}
1155

1156 1157
	lmb_init();

1158 1159
	/* Find available physical memory... */
	read_obp_memory("available", &pavail[0], &pavail_ents);
1160 1161

	phys_base = 0xffffffffffffffffUL;
1162
	for (i = 0; i < pavail_ents; i++) {
1163
		phys_base = min(phys_base, pavail[i].phys_addr);
1164 1165 1166 1167
		lmb_add(pavail[i].phys_addr, pavail[i].reg_size);
	}

	lmb_reserve(kern_base, kern_size);
1168

1169 1170
	find_ramdisk(phys_base);

1171 1172 1173
	if (cmdline_memory_size)
		lmb_enforce_memory_limit(phys_base + cmdline_memory_size);

1174 1175 1176
	lmb_analyze();
	lmb_dump_all();

L
Linus Torvalds 已提交
1177 1178
	set_bit(0, mmu_context_bmap);

1179 1180
	shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);

L
Linus Torvalds 已提交
1181
	real_end = (unsigned long)_end;
1182 1183 1184
	num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22);
	printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
	       num_kernel_image_mappings);
1185 1186

	/* Set kernel pgd to upper alias so physical page computations
L
Linus Torvalds 已提交
1187 1188 1189 1190
	 * work.
	 */
	init_mm.pgd += ((shift) / (sizeof(pgd_t)));
	
1191
	memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
L
Linus Torvalds 已提交
1192 1193 1194

	/* Now can init the kernel/bad page tables. */
	pud_set(pud_offset(&swapper_pg_dir[0], 0),
1195
		swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
L
Linus Torvalds 已提交
1196
	
1197
	inherit_prom_mappings();
1198
	
1199 1200 1201 1202
	read_obp_memory("reg", &pall[0], &pall_ents);

	init_kpte_bitmap();

1203 1204
	/* Ok, we can use our TLB miss and window trap handlers safely.  */
	setup_tba();
L
Linus Torvalds 已提交
1205

1206
	__flush_tlb_all();
1207

1208 1209 1210
	if (tlb_type == hypervisor)
		sun4v_ktsb_register();

1211 1212
	/* Setup bootmem... */
	pages_avail = 0;
1213 1214
	last_valid_pfn = end_pfn = bootmem_init(&pages_avail, phys_base);

1215
	max_mapnr = last_valid_pfn;
1216

1217 1218
	kernel_physical_mapping_init();

1219 1220
	real_setup_per_cpu_areas();

1221 1222
	prom_build_devicetree();

1223 1224 1225
	if (tlb_type == hypervisor)
		sun4v_mdesc_init();

L
Linus Torvalds 已提交
1226 1227 1228 1229 1230 1231 1232 1233
	{
		unsigned long zones_size[MAX_NR_ZONES];
		unsigned long zholes_size[MAX_NR_ZONES];
		int znum;

		for (znum = 0; znum < MAX_NR_ZONES; znum++)
			zones_size[znum] = zholes_size[znum] = 0;

1234 1235
		zones_size[ZONE_NORMAL] = end_pfn;
		zholes_size[ZONE_NORMAL] = end_pfn - pages_avail;
L
Linus Torvalds 已提交
1236 1237

		free_area_init_node(0, &contig_page_data, zones_size,
1238 1239
				    __pa(PAGE_OFFSET) >> PAGE_SHIFT,
				    zholes_size);
L
Linus Torvalds 已提交
1240 1241
	}

1242
	printk("Booting Linux...\n");
1243 1244 1245

	central_probe();
	cpu_probe();
L
Linus Torvalds 已提交
1246 1247 1248 1249 1250 1251
}

static void __init taint_real_pages(void)
{
	int i;

1252
	read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
L
Linus Torvalds 已提交
1253

1254
	/* Find changes discovered in the physmem available rescan and
L
Linus Torvalds 已提交
1255 1256
	 * reserve the lost portions in the bootmem maps.
	 */
1257
	for (i = 0; i < pavail_ents; i++) {
L
Linus Torvalds 已提交
1258 1259
		unsigned long old_start, old_end;

1260
		old_start = pavail[i].phys_addr;
L
Linus Torvalds 已提交
1261
		old_end = old_start +
1262
			pavail[i].reg_size;
L
Linus Torvalds 已提交
1263 1264 1265
		while (old_start < old_end) {
			int n;

1266
			for (n = 0; n < pavail_rescan_ents; n++) {
L
Linus Torvalds 已提交
1267 1268
				unsigned long new_start, new_end;

1269 1270 1271
				new_start = pavail_rescan[n].phys_addr;
				new_end = new_start +
					pavail_rescan[n].reg_size;
L
Linus Torvalds 已提交
1272 1273 1274

				if (new_start <= old_start &&
				    new_end >= (old_start + PAGE_SIZE)) {
1275 1276
					set_bit(old_start >> 22,
						sparc64_valid_addr_bitmap);
L
Linus Torvalds 已提交
1277 1278 1279
					goto do_next_page;
				}
			}
1280
			reserve_bootmem(old_start, PAGE_SIZE, BOOTMEM_DEFAULT);
L
Linus Torvalds 已提交
1281 1282 1283 1284 1285 1286 1287

		do_next_page:
			old_start += PAGE_SIZE;
		}
	}
}

1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
int __init page_in_phys_avail(unsigned long paddr)
{
	int i;

	paddr &= PAGE_MASK;

	for (i = 0; i < pavail_rescan_ents; i++) {
		unsigned long start, end;

		start = pavail_rescan[i].phys_addr;
		end = start + pavail_rescan[i].reg_size;

		if (paddr >= start && paddr < end)
			return 1;
	}
	if (paddr >= kern_base && paddr < (kern_base + kern_size))
		return 1;
#ifdef CONFIG_BLK_DEV_INITRD
	if (paddr >= __pa(initrd_start) &&
	    paddr < __pa(PAGE_ALIGN(initrd_end)))
		return 1;
#endif

	return 0;
}

L
Linus Torvalds 已提交
1314 1315 1316 1317 1318 1319 1320 1321
void __init mem_init(void)
{
	unsigned long codepages, datapages, initpages;
	unsigned long addr, last;
	int i;

	i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
	i += 1;
1322
	sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3);
L
Linus Torvalds 已提交
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
	if (sparc64_valid_addr_bitmap == NULL) {
		prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
		prom_halt();
	}
	memset(sparc64_valid_addr_bitmap, 0, i << 3);

	addr = PAGE_OFFSET + kern_base;
	last = PAGE_ALIGN(kern_size) + addr;
	while (addr < last) {
		set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
		addr += PAGE_SIZE;
	}

	taint_real_pages();

	high_memory = __va(last_valid_pfn << PAGE_SHIFT);

1340 1341 1342
	/* We subtract one to account for the mem_map_zero page
	 * allocated below.
	 */
L
Linus Torvalds 已提交
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362
	totalram_pages = num_physpages = free_all_bootmem() - 1;

	/*
	 * Set up the zero page, mark it reserved, so that page count
	 * is not manipulated when freeing the page from user ptes.
	 */
	mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
	if (mem_map_zero == NULL) {
		prom_printf("paging_init: Cannot alloc zero page.\n");
		prom_halt();
	}
	SetPageReserved(mem_map_zero);

	codepages = (((unsigned long) _etext) - ((unsigned long) _start));
	codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
	datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
	datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
	initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
	initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;

C
Christoph Lameter 已提交
1363
	printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
L
Linus Torvalds 已提交
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
	       nr_free_pages() << (PAGE_SHIFT-10),
	       codepages << (PAGE_SHIFT-10),
	       datapages << (PAGE_SHIFT-10), 
	       initpages << (PAGE_SHIFT-10), 
	       PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));

	if (tlb_type == cheetah || tlb_type == cheetah_plus)
		cheetah_ecache_flush_init();
}

1374
void free_initmem(void)
L
Linus Torvalds 已提交
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
{
	unsigned long addr, initend;

	/*
	 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
	 */
	addr = PAGE_ALIGN((unsigned long)(__init_begin));
	initend = (unsigned long)(__init_end) & PAGE_MASK;
	for (; addr < initend; addr += PAGE_SIZE) {
		unsigned long page;
		struct page *p;

		page = (addr +
			((unsigned long) __va(kern_base)) -
			((unsigned long) KERNBASE));
1390
		memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
L
Linus Torvalds 已提交
1391 1392 1393
		p = virt_to_page(page);

		ClearPageReserved(p);
1394
		init_page_count(p);
L
Linus Torvalds 已提交
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
		__free_page(p);
		num_physpages++;
		totalram_pages++;
	}
}

#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
	if (start < end)
		printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
	for (; start < end; start += PAGE_SIZE) {
		struct page *p = virt_to_page(start);

		ClearPageReserved(p);
1410
		init_page_count(p);
L
Linus Torvalds 已提交
1411 1412 1413 1414 1415 1416
		__free_page(p);
		num_physpages++;
		totalram_pages++;
	}
}
#endif
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429

#define _PAGE_CACHE_4U	(_PAGE_CP_4U | _PAGE_CV_4U)
#define _PAGE_CACHE_4V	(_PAGE_CP_4V | _PAGE_CV_4V)
#define __DIRTY_BITS_4U	 (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
#define __DIRTY_BITS_4V	 (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)

pgprot_t PAGE_KERNEL __read_mostly;
EXPORT_SYMBOL(PAGE_KERNEL);

pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
pgprot_t PAGE_COPY __read_mostly;
1430 1431 1432 1433

pgprot_t PAGE_SHARED __read_mostly;
EXPORT_SYMBOL(PAGE_SHARED);

1434 1435 1436 1437
pgprot_t PAGE_EXEC __read_mostly;
unsigned long pg_iobits __read_mostly;

unsigned long _PAGE_IE __read_mostly;
1438
EXPORT_SYMBOL(_PAGE_IE);
1439

1440
unsigned long _PAGE_E __read_mostly;
1441 1442
EXPORT_SYMBOL(_PAGE_E);

1443
unsigned long _PAGE_CACHE __read_mostly;
1444
EXPORT_SYMBOL(_PAGE_CACHE);
1445

D
David Miller 已提交
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
#ifdef CONFIG_SPARSEMEM_VMEMMAP

#define VMEMMAP_CHUNK_SHIFT	22
#define VMEMMAP_CHUNK		(1UL << VMEMMAP_CHUNK_SHIFT)
#define VMEMMAP_CHUNK_MASK	~(VMEMMAP_CHUNK - 1UL)
#define VMEMMAP_ALIGN(x)	(((x)+VMEMMAP_CHUNK-1UL)&VMEMMAP_CHUNK_MASK)

#define VMEMMAP_SIZE	((((1UL << MAX_PHYSADDR_BITS) >> PAGE_SHIFT) * \
			  sizeof(struct page *)) >> VMEMMAP_CHUNK_SHIFT)
unsigned long vmemmap_table[VMEMMAP_SIZE];

int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
{
	unsigned long vstart = (unsigned long) start;
	unsigned long vend = (unsigned long) (start + nr);
	unsigned long phys_start = (vstart - VMEMMAP_BASE);
	unsigned long phys_end = (vend - VMEMMAP_BASE);
	unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
	unsigned long end = VMEMMAP_ALIGN(phys_end);
	unsigned long pte_base;

	pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
		    _PAGE_CP_4U | _PAGE_CV_4U |
		    _PAGE_P_4U | _PAGE_W_4U);
	if (tlb_type == hypervisor)
		pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
			    _PAGE_CP_4V | _PAGE_CV_4V |
			    _PAGE_P_4V | _PAGE_W_4V);

	for (; addr < end; addr += VMEMMAP_CHUNK) {
		unsigned long *vmem_pp =
			vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);
		void *block;

		if (!(*vmem_pp & _PAGE_VALID)) {
			block = vmemmap_alloc_block(1UL << 22, node);
			if (!block)
				return -ENOMEM;

			*vmem_pp = pte_base | __pa(block);

			printk(KERN_INFO "[%p-%p] page_structs=%lu "
			       "node=%d entry=%lu/%lu\n", start, block, nr,
			       node,
			       addr >> VMEMMAP_CHUNK_SHIFT,
			       VMEMMAP_SIZE >> VMEMMAP_CHUNK_SHIFT);
		}
	}
	return 0;
}
#endif /* CONFIG_SPARSEMEM_VMEMMAP */

1498 1499 1500 1501 1502 1503 1504
static void prot_init_common(unsigned long page_none,
			     unsigned long page_shared,
			     unsigned long page_copy,
			     unsigned long page_readonly,
			     unsigned long page_exec_bit)
{
	PAGE_COPY = __pgprot(page_copy);
1505
	PAGE_SHARED = __pgprot(page_shared);
1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546

	protection_map[0x0] = __pgprot(page_none);
	protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
	protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
	protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
	protection_map[0x4] = __pgprot(page_readonly);
	protection_map[0x5] = __pgprot(page_readonly);
	protection_map[0x6] = __pgprot(page_copy);
	protection_map[0x7] = __pgprot(page_copy);
	protection_map[0x8] = __pgprot(page_none);
	protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
	protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
	protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
	protection_map[0xc] = __pgprot(page_readonly);
	protection_map[0xd] = __pgprot(page_readonly);
	protection_map[0xe] = __pgprot(page_shared);
	protection_map[0xf] = __pgprot(page_shared);
}

static void __init sun4u_pgprot_init(void)
{
	unsigned long page_none, page_shared, page_copy, page_readonly;
	unsigned long page_exec_bit;

	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
				_PAGE_CACHE_4U | _PAGE_P_4U |
				__ACCESS_BITS_4U | __DIRTY_BITS_4U |
				_PAGE_EXEC_4U);
	PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
				       _PAGE_CACHE_4U | _PAGE_P_4U |
				       __ACCESS_BITS_4U | __DIRTY_BITS_4U |
				       _PAGE_EXEC_4U | _PAGE_L_4U);
	PAGE_EXEC = __pgprot(_PAGE_EXEC_4U);

	_PAGE_IE = _PAGE_IE_4U;
	_PAGE_E = _PAGE_E_4U;
	_PAGE_CACHE = _PAGE_CACHE_4U;

	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
		     __ACCESS_BITS_4U | _PAGE_E_4U);

1547 1548 1549 1550
#ifdef CONFIG_DEBUG_PAGEALLOC
	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^
		0xfffff80000000000;
#else
1551
	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
1552
		0xfffff80000000000;
1553
#endif
1554 1555 1556 1557 1558
	kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
				   _PAGE_P_4U | _PAGE_W_4U);

	/* XXX Should use 256MB on Panther. XXX */
	kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595

	_PAGE_SZBITS = _PAGE_SZBITS_4U;
	_PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
			      _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
			      _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);


	page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
		       __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
		       __ACCESS_BITS_4U | _PAGE_EXEC_4U);
	page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
			   __ACCESS_BITS_4U | _PAGE_EXEC_4U);

	page_exec_bit = _PAGE_EXEC_4U;

	prot_init_common(page_none, page_shared, page_copy, page_readonly,
			 page_exec_bit);
}

static void __init sun4v_pgprot_init(void)
{
	unsigned long page_none, page_shared, page_copy, page_readonly;
	unsigned long page_exec_bit;

	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
				_PAGE_CACHE_4V | _PAGE_P_4V |
				__ACCESS_BITS_4V | __DIRTY_BITS_4V |
				_PAGE_EXEC_4V);
	PAGE_KERNEL_LOCKED = PAGE_KERNEL;
	PAGE_EXEC = __pgprot(_PAGE_EXEC_4V);

	_PAGE_IE = _PAGE_IE_4V;
	_PAGE_E = _PAGE_E_4V;
	_PAGE_CACHE = _PAGE_CACHE_4V;

1596 1597 1598 1599
#ifdef CONFIG_DEBUG_PAGEALLOC
	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
		0xfffff80000000000;
#else
1600 1601
	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
		0xfffff80000000000;
1602
#endif
1603 1604 1605
	kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
				   _PAGE_P_4V | _PAGE_W_4V);

1606 1607 1608 1609
#ifdef CONFIG_DEBUG_PAGEALLOC
	kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^
		0xfffff80000000000;
#else
1610
	kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
1611
		0xfffff80000000000;
1612
#endif
1613 1614
	kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
				   _PAGE_P_4V | _PAGE_W_4V);
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670

	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
		     __ACCESS_BITS_4V | _PAGE_E_4V);

	_PAGE_SZBITS = _PAGE_SZBITS_4V;
	_PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
			     _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
			     _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
			     _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);

	page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;
	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
		       __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
		       __ACCESS_BITS_4V | _PAGE_EXEC_4V);
	page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
			 __ACCESS_BITS_4V | _PAGE_EXEC_4V);

	page_exec_bit = _PAGE_EXEC_4V;

	prot_init_common(page_none, page_shared, page_copy, page_readonly,
			 page_exec_bit);
}

unsigned long pte_sz_bits(unsigned long sz)
{
	if (tlb_type == hypervisor) {
		switch (sz) {
		case 8 * 1024:
		default:
			return _PAGE_SZ8K_4V;
		case 64 * 1024:
			return _PAGE_SZ64K_4V;
		case 512 * 1024:
			return _PAGE_SZ512K_4V;
		case 4 * 1024 * 1024:
			return _PAGE_SZ4MB_4V;
		};
	} else {
		switch (sz) {
		case 8 * 1024:
		default:
			return _PAGE_SZ8K_4U;
		case 64 * 1024:
			return _PAGE_SZ64K_4U;
		case 512 * 1024:
			return _PAGE_SZ512K_4U;
		case 4 * 1024 * 1024:
			return _PAGE_SZ4MB_4U;
		};
	}
}

pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
{
	pte_t pte;
1671 1672

	pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
1673 1674 1675
	pte_val(pte) |= (((unsigned long)space) << 32);
	pte_val(pte) |= pte_sz_bits(page_size);

1676
	return pte;
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704
}

static unsigned long kern_large_tte(unsigned long paddr)
{
	unsigned long val;

	val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
	       _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
	       _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
	if (tlb_type == hypervisor)
		val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
		       _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |
		       _PAGE_EXEC_4V | _PAGE_W_4V);

	return val | paddr;
}

/* If not locked, zap it. */
void __flush_tlb_all(void)
{
	unsigned long pstate;
	int i;

	__asm__ __volatile__("flushw\n\t"
			     "rdpr	%%pstate, %0\n\t"
			     "wrpr	%0, %1, %%pstate"
			     : "=r" (pstate)
			     : "i" (PSTATE_IE));
1705 1706 1707
	if (tlb_type == hypervisor) {
		sun4v_mmu_demap_all();
	} else if (tlb_type == spitfire) {
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751
		for (i = 0; i < 64; i++) {
			/* Spitfire Errata #32 workaround */
			/* NOTE: Always runs on spitfire, so no
			 *       cheetah+ page size encodings.
			 */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

			if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				spitfire_put_dtlb_data(i, 0x0UL);
			}

			/* Spitfire Errata #32 workaround */
			/* NOTE: Always runs on spitfire, so no
			 *       cheetah+ page size encodings.
			 */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

			if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
				spitfire_put_itlb_data(i, 0x0UL);
			}
		}
	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
		cheetah_flush_dtlb_all();
		cheetah_flush_itlb_all();
	}
	__asm__ __volatile__("wrpr	%0, 0, %%pstate"
			     : : "r" (pstate));
}
1752 1753 1754 1755 1756 1757

#ifdef CONFIG_MEMORY_HOTPLUG

void online_page(struct page *page)
{
	ClearPageReserved(page);
1758 1759
	init_page_count(page);
	__free_page(page);
1760 1761 1762 1763 1764
	totalram_pages++;
	num_physpages++;
}

#endif /* CONFIG_MEMORY_HOTPLUG */