hash_utils_64.c 32.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * PowerPC64 port by Mike Corrigan and Dave Engebretsen
 *   {mikejc|engebret}@us.ibm.com
 *
 *    Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
 *
 * SMP scalability work:
 *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
 * 
 *    Module name: htab.c
 *
 *    Description:
 *      PowerPC Hashed Page Table functions
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#undef DEBUG
22
#undef DEBUG_LOW
L
Linus Torvalds 已提交
23 24 25 26 27 28 29 30 31 32 33

#include <linux/spinlock.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/sysctl.h>
#include <linux/ctype.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/signal.h>
34
#include <linux/lmb.h>
L
Linus Torvalds 已提交
35 36 37 38 39 40 41 42 43 44

#include <asm/processor.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
#include <asm/page.h>
#include <asm/types.h>
#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/machdep.h>
45
#include <asm/prom.h>
L
Linus Torvalds 已提交
46 47 48 49 50 51 52 53
#include <asm/abs_addr.h>
#include <asm/tlbflush.h>
#include <asm/io.h>
#include <asm/eeh.h>
#include <asm/tlb.h>
#include <asm/cacheflush.h>
#include <asm/cputable.h>
#include <asm/sections.h>
54
#include <asm/spu.h>
55
#include <asm/udbg.h>
L
Linus Torvalds 已提交
56 57 58 59 60 61 62

#ifdef DEBUG
#define DBG(fmt...) udbg_printf(fmt)
#else
#define DBG(fmt...)
#endif

63 64 65 66 67 68 69 70
#ifdef DEBUG_LOW
#define DBG_LOW(fmt...) udbg_printf(fmt)
#else
#define DBG_LOW(fmt...)
#endif

#define KB (1024)
#define MB (1024*KB)
71
#define GB (1024L*MB)
72

L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
/*
 * Note:  pte   --> Linux PTE
 *        HPTE  --> PowerPC Hashed Page Table Entry
 *
 * Execution context:
 *   htab_initialize is called with the MMU off (of course), but
 *   the kernel has been copied down to zero so it can directly
 *   reference global data.  At this point it is very difficult
 *   to print debug info.
 *
 */

#ifdef CONFIG_U3_DART
extern unsigned long dart_tablebase;
#endif /* CONFIG_U3_DART */

89 90 91
static unsigned long _SDR1;
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];

92
struct hash_pte *htab_address;
93
unsigned long htab_size_bytes;
94
unsigned long htab_hash_mask;
A
Alexander Graf 已提交
95
EXPORT_SYMBOL_GPL(htab_hash_mask);
96 97
int mmu_linear_psize = MMU_PAGE_4K;
int mmu_virtual_psize = MMU_PAGE_4K;
98
int mmu_vmalloc_psize = MMU_PAGE_4K;
99 100 101
#ifdef CONFIG_SPARSEMEM_VMEMMAP
int mmu_vmemmap_psize = MMU_PAGE_4K;
#endif
102
int mmu_io_psize = MMU_PAGE_4K;
P
Paul Mackerras 已提交
103 104
int mmu_kernel_ssize = MMU_SEGSIZE_256M;
int mmu_highuser_ssize = MMU_SEGSIZE_256M;
105
u16 mmu_slb_size = 64;
A
Alexander Graf 已提交
106
EXPORT_SYMBOL_GPL(mmu_slb_size);
107 108 109
#ifdef CONFIG_HUGETLB_PAGE
unsigned int HPAGE_SHIFT;
#endif
110 111 112
#ifdef CONFIG_PPC_64K_PAGES
int mmu_ci_restrictions;
#endif
113 114 115
#ifdef CONFIG_DEBUG_PAGEALLOC
static u8 *linear_map_hash_slots;
static unsigned long linear_map_hash_count;
116
static DEFINE_SPINLOCK(linear_map_hash_lock);
117
#endif /* CONFIG_DEBUG_PAGEALLOC */
L
Linus Torvalds 已提交
118

119 120 121
/* There are definitions of page sizes arrays to be used when none
 * is provided by the firmware.
 */
L
Linus Torvalds 已提交
122

123 124
/* Pre-POWER4 CPUs (4k pages only)
 */
125
static struct mmu_psize_def mmu_psize_defaults_old[] = {
126 127 128 129 130 131 132 133 134 135 136 137 138
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.sllp	= 0,
		.penc	= 0,
		.avpnm	= 0,
		.tlbiel = 0,
	},
};

/* POWER4, GPUL, POWER5
 *
 * Support for 16Mb large pages
 */
139
static struct mmu_psize_def mmu_psize_defaults_gp[] = {
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.sllp	= 0,
		.penc	= 0,
		.avpnm	= 0,
		.tlbiel = 1,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.sllp	= SLB_VSID_L,
		.penc	= 0,
		.avpnm	= 0x1UL,
		.tlbiel = 0,
	},
};

156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
static unsigned long htab_convert_pte_flags(unsigned long pteflags)
{
	unsigned long rflags = pteflags & 0x1fa;

	/* _PAGE_EXEC -> NOEXEC */
	if ((pteflags & _PAGE_EXEC) == 0)
		rflags |= HPTE_R_N;

	/* PP bits. PAGE_USER is already PP bit 0x2, so we only
	 * need to add in 0x1 if it's a read-only user page
	 */
	if ((pteflags & _PAGE_USER) && !((pteflags & _PAGE_RW) &&
					 (pteflags & _PAGE_DIRTY)))
		rflags |= 1;

	/* Always add C */
	return rflags | HPTE_R_C;
}
174 175

int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
176
		      unsigned long pstart, unsigned long prot,
P
Paul Mackerras 已提交
177
		      int psize, int ssize)
L
Linus Torvalds 已提交
178
{
179 180 181
	unsigned long vaddr, paddr;
	unsigned int step, shift;
	int ret = 0;
L
Linus Torvalds 已提交
182

183 184
	shift = mmu_psize_defs[psize].shift;
	step = 1 << shift;
L
Linus Torvalds 已提交
185

186 187 188 189 190
	prot = htab_convert_pte_flags(prot);

	DBG("htab_bolt_mapping(%lx..%lx -> %lx (%lx,%d,%d)\n",
	    vstart, vend, pstart, prot, psize, ssize);

191 192
	for (vaddr = vstart, paddr = pstart; vaddr < vend;
	     vaddr += step, paddr += step) {
193
		unsigned long hash, hpteg;
P
Paul Mackerras 已提交
194 195
		unsigned long vsid = get_kernel_vsid(vaddr, ssize);
		unsigned long va = hpt_va(vaddr, vsid, ssize);
196 197 198
		unsigned long tprot = prot;

		/* Make kernel text executable */
199
		if (overlaps_kernel_text(vaddr, vaddr + step))
200
			tprot &= ~HPTE_R_N;
L
Linus Torvalds 已提交
201

P
Paul Mackerras 已提交
202
		hash = hpt_hash(va, shift, ssize);
L
Linus Torvalds 已提交
203 204
		hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);

205
		BUG_ON(!ppc_md.hpte_insert);
206
		ret = ppc_md.hpte_insert(hpteg, va, paddr, tprot,
207
					 HPTE_V_BOLTED, psize, ssize);
208

209 210
		if (ret < 0)
			break;
211 212 213 214
#ifdef CONFIG_DEBUG_PAGEALLOC
		if ((paddr >> PAGE_SHIFT) < linear_map_hash_count)
			linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80;
#endif /* CONFIG_DEBUG_PAGEALLOC */
215 216 217
	}
	return ret < 0 ? ret : 0;
}
L
Linus Torvalds 已提交
218

219
#ifdef CONFIG_MEMORY_HOTPLUG
220
static int htab_remove_mapping(unsigned long vstart, unsigned long vend,
221 222 223 224 225 226 227 228 229
		      int psize, int ssize)
{
	unsigned long vaddr;
	unsigned int step, shift;

	shift = mmu_psize_defs[psize].shift;
	step = 1 << shift;

	if (!ppc_md.hpte_removebolted) {
230 231 232
		printk(KERN_WARNING "Platform doesn't implement "
				"hpte_removebolted\n");
		return -EINVAL;
233 234 235 236
	}

	for (vaddr = vstart; vaddr < vend; vaddr += step)
		ppc_md.hpte_removebolted(vaddr, psize, ssize);
237 238

	return 0;
239
}
240
#endif /* CONFIG_MEMORY_HOTPLUG */
241

P
Paul Mackerras 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
static int __init htab_dt_scan_seg_sizes(unsigned long node,
					 const char *uname, int depth,
					 void *data)
{
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	u32 *prop;
	unsigned long size = 0;

	/* We are scanning "cpu" nodes only */
	if (type == NULL || strcmp(type, "cpu") != 0)
		return 0;

	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,processor-segment-sizes",
					  &size);
	if (prop == NULL)
		return 0;
	for (; size >= 4; size -= 4, ++prop) {
		if (prop[0] == 40) {
			DBG("1T segment support detected\n");
			cur_cpu_spec->cpu_features |= CPU_FTR_1T_SEGMENT;
262
			return 1;
P
Paul Mackerras 已提交
263 264
		}
	}
265
	cur_cpu_spec->cpu_features &= ~CPU_FTR_NO_SLBIE_B;
P
Paul Mackerras 已提交
266 267 268 269 270 271 272 273
	return 0;
}

static void __init htab_init_seg_sizes(void)
{
	of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL);
}

274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
static int __init htab_dt_scan_page_sizes(unsigned long node,
					  const char *uname, int depth,
					  void *data)
{
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	u32 *prop;
	unsigned long size = 0;

	/* We are scanning "cpu" nodes only */
	if (type == NULL || strcmp(type, "cpu") != 0)
		return 0;

	prop = (u32 *)of_get_flat_dt_prop(node,
					  "ibm,segment-page-sizes", &size);
	if (prop != NULL) {
		DBG("Page sizes from device-tree:\n");
		size /= 4;
		cur_cpu_spec->cpu_features &= ~(CPU_FTR_16M_PAGE);
		while(size > 0) {
			unsigned int shift = prop[0];
			unsigned int slbenc = prop[1];
			unsigned int lpnum = prop[2];
			unsigned int lpenc = 0;
			struct mmu_psize_def *def;
			int idx = -1;

			size -= 3; prop += 3;
			while(size > 0 && lpnum) {
				if (prop[0] == shift)
					lpenc = prop[1];
				prop += 2; size -= 2;
				lpnum--;
			}
			switch(shift) {
			case 0xc:
				idx = MMU_PAGE_4K;
				break;
			case 0x10:
				idx = MMU_PAGE_64K;
				break;
			case 0x14:
				idx = MMU_PAGE_1M;
				break;
			case 0x18:
				idx = MMU_PAGE_16M;
				cur_cpu_spec->cpu_features |= CPU_FTR_16M_PAGE;
				break;
			case 0x22:
				idx = MMU_PAGE_16G;
				break;
			}
			if (idx < 0)
				continue;
			def = &mmu_psize_defs[idx];
			def->shift = shift;
			if (shift <= 23)
				def->avpnm = 0;
			else
				def->avpnm = (1 << (shift - 23)) - 1;
			def->sllp = slbenc;
			def->penc = lpenc;
			/* We don't know for sure what's up with tlbiel, so
			 * for now we only set it for 4K and 64K pages
			 */
			if (idx == MMU_PAGE_4K || idx == MMU_PAGE_64K)
				def->tlbiel = 1;
			else
				def->tlbiel = 0;

			DBG(" %d: shift=%02x, sllp=%04x, avpnm=%08x, "
			    "tlbiel=%d, penc=%d\n",
			    idx, shift, def->sllp, def->avpnm, def->tlbiel,
			    def->penc);
L
Linus Torvalds 已提交
347
		}
348 349 350 351 352
		return 1;
	}
	return 0;
}

353
#ifdef CONFIG_HUGETLB_PAGE
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
/* Scan for 16G memory blocks that have been set aside for huge pages
 * and reserve those blocks for 16G huge pages.
 */
static int __init htab_dt_scan_hugepage_blocks(unsigned long node,
					const char *uname, int depth,
					void *data) {
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	unsigned long *addr_prop;
	u32 *page_count_prop;
	unsigned int expected_pages;
	long unsigned int phys_addr;
	long unsigned int block_size;

	/* We are scanning "memory" nodes only */
	if (type == NULL || strcmp(type, "memory") != 0)
		return 0;

	/* This property is the log base 2 of the number of virtual pages that
	 * will represent this memory block. */
	page_count_prop = of_get_flat_dt_prop(node, "ibm,expected#pages", NULL);
	if (page_count_prop == NULL)
		return 0;
	expected_pages = (1 << page_count_prop[0]);
	addr_prop = of_get_flat_dt_prop(node, "reg", NULL);
	if (addr_prop == NULL)
		return 0;
	phys_addr = addr_prop[0];
	block_size = addr_prop[1];
	if (block_size != (16 * GB))
		return 0;
	printk(KERN_INFO "Huge page(16GB) memory: "
			"addr = 0x%lX size = 0x%lX pages = %d\n",
			phys_addr, block_size, expected_pages);
387 388 389 390
	if (phys_addr + (16 * GB) <= lmb_end_of_DRAM()) {
		lmb_reserve(phys_addr, block_size * expected_pages);
		add_gpage(phys_addr, block_size, expected_pages);
	}
391 392
	return 0;
}
393
#endif /* CONFIG_HUGETLB_PAGE */
394

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
static void __init htab_init_page_sizes(void)
{
	int rc;

	/* Default to 4K pages only */
	memcpy(mmu_psize_defs, mmu_psize_defaults_old,
	       sizeof(mmu_psize_defaults_old));

	/*
	 * Try to find the available page sizes in the device-tree
	 */
	rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL);
	if (rc != 0)  /* Found */
		goto found;

	/*
	 * Not in the device-tree, let's fallback on known size
	 * list for 16M capable GP & GR
	 */
414
	if (cpu_has_feature(CPU_FTR_16M_PAGE))
415 416 417
		memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
		       sizeof(mmu_psize_defaults_gp));
 found:
418
#ifndef CONFIG_DEBUG_PAGEALLOC
419 420 421 422 423 424 425 426
	/*
	 * Pick a size for the linear mapping. Currently, we only support
	 * 16M, 1M and 4K which is the default
	 */
	if (mmu_psize_defs[MMU_PAGE_16M].shift)
		mmu_linear_psize = MMU_PAGE_16M;
	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
		mmu_linear_psize = MMU_PAGE_1M;
427
#endif /* CONFIG_DEBUG_PAGEALLOC */
428

429
#ifdef CONFIG_PPC_64K_PAGES
430 431
	/*
	 * Pick a size for the ordinary pages. Default is 4K, we support
432 433 434 435 436 437
	 * 64K for user mappings and vmalloc if supported by the processor.
	 * We only use 64k for ioremap if the processor
	 * (and firmware) support cache-inhibited large pages.
	 * If not, we use 4k and set mmu_ci_restrictions so that
	 * hash_page knows to switch processes that use cache-inhibited
	 * mappings to 4k pages.
438
	 */
439
	if (mmu_psize_defs[MMU_PAGE_64K].shift) {
440
		mmu_virtual_psize = MMU_PAGE_64K;
441
		mmu_vmalloc_psize = MMU_PAGE_64K;
442 443
		if (mmu_linear_psize == MMU_PAGE_4K)
			mmu_linear_psize = MMU_PAGE_64K;
444 445 446 447 448 449 450 451
		if (cpu_has_feature(CPU_FTR_CI_LARGE_PAGE)) {
			/*
			 * Don't use 64k pages for ioremap on pSeries, since
			 * that would stop us accessing the HEA ethernet.
			 */
			if (!machine_is(pseries))
				mmu_io_psize = MMU_PAGE_64K;
		} else
452 453
			mmu_ci_restrictions = 1;
	}
454
#endif /* CONFIG_PPC_64K_PAGES */
455

456 457 458 459 460 461 462 463 464 465 466 467 468
#ifdef CONFIG_SPARSEMEM_VMEMMAP
	/* We try to use 16M pages for vmemmap if that is supported
	 * and we have at least 1G of RAM at boot
	 */
	if (mmu_psize_defs[MMU_PAGE_16M].shift &&
	    lmb_phys_mem_size() >= 0x40000000)
		mmu_vmemmap_psize = MMU_PAGE_16M;
	else if (mmu_psize_defs[MMU_PAGE_64K].shift)
		mmu_vmemmap_psize = MMU_PAGE_64K;
	else
		mmu_vmemmap_psize = MMU_PAGE_4K;
#endif /* CONFIG_SPARSEMEM_VMEMMAP */

469
	printk(KERN_DEBUG "Page orders: linear mapping = %d, "
470 471 472 473 474
	       "virtual = %d, io = %d"
#ifdef CONFIG_SPARSEMEM_VMEMMAP
	       ", vmemmap = %d"
#endif
	       "\n",
475
	       mmu_psize_defs[mmu_linear_psize].shift,
476
	       mmu_psize_defs[mmu_virtual_psize].shift,
477 478 479 480 481
	       mmu_psize_defs[mmu_io_psize].shift
#ifdef CONFIG_SPARSEMEM_VMEMMAP
	       ,mmu_psize_defs[mmu_vmemmap_psize].shift
#endif
	       );
482 483

#ifdef CONFIG_HUGETLB_PAGE
484 485
	/* Reserve 16G huge page memory sections for huge pages */
	of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
#endif /* CONFIG_HUGETLB_PAGE */
}

static int __init htab_dt_scan_pftsize(unsigned long node,
				       const char *uname, int depth,
				       void *data)
{
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	u32 *prop;

	/* We are scanning "cpu" nodes only */
	if (type == NULL || strcmp(type, "cpu") != 0)
		return 0;

	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,pft-size", NULL);
	if (prop != NULL) {
		/* pft_size[0] is the NUMA CEC cookie */
		ppc64_pft_size = prop[1];
		return 1;
L
Linus Torvalds 已提交
505
	}
506
	return 0;
L
Linus Torvalds 已提交
507 508
}

509
static unsigned long __init htab_get_table_size(void)
510
{
511
	unsigned long mem_size, rnd_mem_size, pteg_count, psize;
512

513
	/* If hash size isn't already provided by the platform, we try to
A
Adrian Bunk 已提交
514
	 * retrieve it from the device-tree. If it's not there neither, we
515
	 * calculate it now based on the total RAM size
516
	 */
517 518
	if (ppc64_pft_size == 0)
		of_scan_flat_dt(htab_dt_scan_pftsize, NULL);
519 520 521 522
	if (ppc64_pft_size)
		return 1UL << ppc64_pft_size;

	/* round mem_size up to next power of 2 */
523 524 525
	mem_size = lmb_phys_mem_size();
	rnd_mem_size = 1UL << __ilog2(mem_size);
	if (rnd_mem_size < mem_size)
526 527 528
		rnd_mem_size <<= 1;

	/* # pages / 2 */
529 530
	psize = mmu_psize_defs[mmu_virtual_psize].shift;
	pteg_count = max(rnd_mem_size >> (psize + 1), 1UL << 11);
531 532 533 534

	return pteg_count << 7;
}

535 536 537
#ifdef CONFIG_MEMORY_HOTPLUG
void create_section_mapping(unsigned long start, unsigned long end)
{
538
	BUG_ON(htab_bolt_mapping(start, end, __pa(start),
539
				 pgprot_val(PAGE_KERNEL), mmu_linear_psize,
540
				 mmu_kernel_ssize));
541
}
542

543
int remove_section_mapping(unsigned long start, unsigned long end)
544
{
545 546
	return htab_remove_mapping(start, end, mmu_linear_psize,
			mmu_kernel_ssize);
547
}
548 549
#endif /* CONFIG_MEMORY_HOTPLUG */

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
static inline void make_bl(unsigned int *insn_addr, void *func)
{
	unsigned long funcp = *((unsigned long *)func);
	int offset = funcp - (unsigned long)insn_addr;

	*insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc));
	flush_icache_range((unsigned long)insn_addr, 4+
			   (unsigned long)insn_addr);
}

static void __init htab_finish_init(void)
{
	extern unsigned int *htab_call_hpte_insert1;
	extern unsigned int *htab_call_hpte_insert2;
	extern unsigned int *htab_call_hpte_remove;
	extern unsigned int *htab_call_hpte_updatepp;

567
#ifdef CONFIG_PPC_HAS_HASH_64K
568 569 570 571 572 573 574 575 576
	extern unsigned int *ht64_call_hpte_insert1;
	extern unsigned int *ht64_call_hpte_insert2;
	extern unsigned int *ht64_call_hpte_remove;
	extern unsigned int *ht64_call_hpte_updatepp;

	make_bl(ht64_call_hpte_insert1, ppc_md.hpte_insert);
	make_bl(ht64_call_hpte_insert2, ppc_md.hpte_insert);
	make_bl(ht64_call_hpte_remove, ppc_md.hpte_remove);
	make_bl(ht64_call_hpte_updatepp, ppc_md.hpte_updatepp);
J
Jon Tollefson 已提交
577
#endif /* CONFIG_PPC_HAS_HASH_64K */
578 579 580 581 582 583 584

	make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert);
	make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert);
	make_bl(htab_call_hpte_remove, ppc_md.hpte_remove);
	make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
}

585
static void __init htab_initialize(void)
L
Linus Torvalds 已提交
586
{
587
	unsigned long table;
L
Linus Torvalds 已提交
588
	unsigned long pteg_count;
589
	unsigned long prot;
590
	unsigned long base = 0, size = 0, limit;
591 592
	int i;

L
Linus Torvalds 已提交
593 594
	DBG(" -> htab_initialize()\n");

P
Paul Mackerras 已提交
595 596 597
	/* Initialize segment sizes */
	htab_init_seg_sizes();

598 599 600
	/* Initialize page sizes */
	htab_init_page_sizes();

P
Paul Mackerras 已提交
601 602 603 604 605 606
	if (cpu_has_feature(CPU_FTR_1T_SEGMENT)) {
		mmu_kernel_ssize = MMU_SEGSIZE_1T;
		mmu_highuser_ssize = MMU_SEGSIZE_1T;
		printk(KERN_INFO "Using 1TB segments\n");
	}

L
Linus Torvalds 已提交
607 608 609 610
	/*
	 * Calculate the required size of the htab.  We want the number of
	 * PTEGs to equal one half the number of real pages.
	 */ 
611
	htab_size_bytes = htab_get_table_size();
L
Linus Torvalds 已提交
612 613 614 615
	pteg_count = htab_size_bytes >> 7;

	htab_hash_mask = pteg_count - 1;

616
	if (firmware_has_feature(FW_FEATURE_LPAR)) {
L
Linus Torvalds 已提交
617 618 619 620 621
		/* Using a hypervisor which owns the htab */
		htab_address = NULL;
		_SDR1 = 0; 
	} else {
		/* Find storage for the HPT.  Must be contiguous in
622
		 * the absolute address space. On cell we want it to be
623
		 * in the first 2 Gig so we can use it for IOMMU hacks.
L
Linus Torvalds 已提交
624
		 */
625
		if (machine_is(cell))
626
			limit = 0x80000000;
627 628 629 630
		else
			limit = 0;

		table = lmb_alloc_base(htab_size_bytes, htab_size_bytes, limit);
L
Linus Torvalds 已提交
631 632 633 634 635 636 637 638 639 640 641

		DBG("Hash table allocated at %lx, size: %lx\n", table,
		    htab_size_bytes);

		htab_address = abs_to_virt(table);

		/* htab absolute addr + encoded htabsize */
		_SDR1 = table + __ilog2(pteg_count) - 11;

		/* Initialize the HPT with no entries */
		memset((void *)table, 0, htab_size_bytes);
642 643 644

		/* Set SDR1 */
		mtspr(SPRN_SDR1, _SDR1);
L
Linus Torvalds 已提交
645 646
	}

647
	prot = pgprot_val(PAGE_KERNEL);
L
Linus Torvalds 已提交
648

649 650 651 652 653 654 655
#ifdef CONFIG_DEBUG_PAGEALLOC
	linear_map_hash_count = lmb_end_of_DRAM() >> PAGE_SHIFT;
	linear_map_hash_slots = __va(lmb_alloc_base(linear_map_hash_count,
						    1, lmb.rmo_size));
	memset(linear_map_hash_slots, 0, linear_map_hash_count);
#endif /* CONFIG_DEBUG_PAGEALLOC */

L
Linus Torvalds 已提交
656 657 658 659 660 661 662
	/* On U3 based machines, we need to reserve the DART area and
	 * _NOT_ map it to avoid cache paradoxes as it's remapped non
	 * cacheable later on
	 */

	/* create bolted the linear mapping in the hash table */
	for (i=0; i < lmb.memory.cnt; i++) {
663
		base = (unsigned long)__va(lmb.memory.region[i].base);
L
Linus Torvalds 已提交
664 665
		size = lmb.memory.region[i].size;

666
		DBG("creating mapping for region: %lx..%lx (prot: %x)\n",
667
		    base, size, prot);
L
Linus Torvalds 已提交
668 669 670

#ifdef CONFIG_U3_DART
		/* Do not map the DART space. Fortunately, it will be aligned
671 672 673 674 675
		 * in such a way that it will not cross two lmb regions and
		 * will fit within a single 16Mb page.
		 * The DART space is assumed to be a full 16Mb region even if
		 * we only use 2Mb of that space. We will use more of it later
		 * for AGP GART. We have to use a full 16Mb large page.
L
Linus Torvalds 已提交
676 677 678 679 680
		 */
		DBG("DART base: %lx\n", dart_tablebase);

		if (dart_tablebase != 0 && dart_tablebase >= base
		    && dart_tablebase < (base + size)) {
681
			unsigned long dart_table_end = dart_tablebase + 16 * MB;
L
Linus Torvalds 已提交
682
			if (base != dart_tablebase)
683
				BUG_ON(htab_bolt_mapping(base, dart_tablebase,
684
							__pa(base), prot,
P
Paul Mackerras 已提交
685 686
							mmu_linear_psize,
							mmu_kernel_ssize));
687
			if ((base + size) > dart_table_end)
688
				BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
689 690
							base + size,
							__pa(dart_table_end),
691
							 prot,
P
Paul Mackerras 已提交
692 693
							 mmu_linear_psize,
							 mmu_kernel_ssize));
L
Linus Torvalds 已提交
694 695 696
			continue;
		}
#endif /* CONFIG_U3_DART */
697
		BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
698
				prot, mmu_linear_psize, mmu_kernel_ssize));
699
       }
L
Linus Torvalds 已提交
700 701 702 703 704 705 706 707 708

	/*
	 * If we have a memory_limit and we've allocated TCEs then we need to
	 * explicitly map the TCE area at the top of RAM. We also cope with the
	 * case that the TCEs start below memory_limit.
	 * tce_alloc_start/end are 16MB aligned so the mapping should work
	 * for either 4K or 16MB pages.
	 */
	if (tce_alloc_start) {
709 710
		tce_alloc_start = (unsigned long)__va(tce_alloc_start);
		tce_alloc_end = (unsigned long)__va(tce_alloc_end);
L
Linus Torvalds 已提交
711 712 713 714

		if (base + size >= tce_alloc_start)
			tce_alloc_start = base + size + 1;

715
		BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
716
					 __pa(tce_alloc_start), prot,
P
Paul Mackerras 已提交
717
					 mmu_linear_psize, mmu_kernel_ssize));
L
Linus Torvalds 已提交
718 719
	}

720 721
	htab_finish_init();

L
Linus Torvalds 已提交
722 723 724 725 726
	DBG(" <- htab_initialize()\n");
}
#undef KB
#undef MB

727
void __init early_init_mmu(void)
728
{
729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	/* Setup initial STAB address in the PACA */
	get_paca()->stab_real = __pa((u64)&initial_stab);
	get_paca()->stab_addr = (u64)&initial_stab;

	/* Initialize the MMU Hash table and create the linear mapping
	 * of memory. Has to be done before stab/slb initialization as
	 * this is currently where the page size encoding is obtained
	 */
	htab_initialize();

	/* Initialize stab / SLB management except on iSeries
	 */
	if (cpu_has_feature(CPU_FTR_SLB))
		slb_initialize();
	else if (!firmware_has_feature(FW_FEATURE_ISERIES))
		stab_initialize(get_paca()->stab_real);
}

#ifdef CONFIG_SMP
748
void __cpuinit early_init_mmu_secondary(void)
749 750
{
	/* Initialize hash table for that CPU */
751
	if (!firmware_has_feature(FW_FEATURE_LPAR))
752
		mtspr(SPRN_SDR1, _SDR1);
753 754 755 756 757 758 759 760 761

	/* Initialize STAB/SLB. We use a virtual address as it works
	 * in real mode on pSeries and we want a virutal address on
	 * iSeries anyway
	 */
	if (cpu_has_feature(CPU_FTR_SLB))
		slb_initialize();
	else
		stab_initialize(get_paca()->stab_addr);
762
}
763
#endif /* CONFIG_SMP */
764

L
Linus Torvalds 已提交
765 766 767 768 769 770 771
/*
 * Called by asm hashtable.S for doing lazy icache flush
 */
unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
{
	struct page *page;

772 773 774
	if (!pfn_valid(pte_pfn(pte)))
		return pp;

L
Linus Torvalds 已提交
775 776 777 778 779
	page = pte_page(pte);

	/* page is dirty */
	if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
		if (trap == 0x400) {
780
			flush_dcache_icache_page(page);
L
Linus Torvalds 已提交
781 782
			set_bit(PG_arch_1, &page->flags);
		} else
783
			pp |= HPTE_R_N;
L
Linus Torvalds 已提交
784 785 786 787
	}
	return pp;
}

788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
#ifdef CONFIG_PPC_MM_SLICES
unsigned int get_paca_psize(unsigned long addr)
{
	unsigned long index, slices;

	if (addr < SLICE_LOW_TOP) {
		slices = get_paca()->context.low_slices_psize;
		index = GET_LOW_SLICE_INDEX(addr);
	} else {
		slices = get_paca()->context.high_slices_psize;
		index = GET_HIGH_SLICE_INDEX(addr);
	}
	return (slices >> (index * 4)) & 0xF;
}

#else
unsigned int get_paca_psize(unsigned long addr)
{
	return get_paca()->context.user_psize;
}
#endif

810 811 812 813 814
/*
 * Demote a segment to using 4k pages.
 * For now this makes the whole process use 4k pages.
 */
#ifdef CONFIG_PPC_64K_PAGES
815
void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
816
{
817
	if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
818
		return;
819
	slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K);
820
#ifdef CONFIG_SPU_BASE
821 822
	spu_flush_all_slbs(mm);
#endif
823
	if (get_paca_psize(addr) != MMU_PAGE_4K) {
824 825 826
		get_paca()->context = mm->context;
		slb_flush_and_rebolt();
	}
827
}
828
#endif /* CONFIG_PPC_64K_PAGES */
829

830 831 832 833 834 835 836 837
#ifdef CONFIG_PPC_SUBPAGE_PROT
/*
 * This looks up a 2-bit protection code for a 4k subpage of a 64k page.
 * Userspace sets the subpage permissions using the subpage_prot system call.
 *
 * Result is 0: full permissions, _PAGE_RW: read-only,
 * _PAGE_USER or _PAGE_USER|_PAGE_RW: no access.
 */
838
static int subpage_protection(pgd_t *pgdir, unsigned long ea)
839
{
840
	struct subpage_prot_table *spt = pgd_subpage_prot(pgdir);
841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
	u32 spp = 0;
	u32 **sbpm, *sbpp;

	if (ea >= spt->maxaddr)
		return 0;
	if (ea < 0x100000000) {
		/* addresses below 4GB use spt->low_prot */
		sbpm = spt->low_prot;
	} else {
		sbpm = spt->protptrs[ea >> SBP_L3_SHIFT];
		if (!sbpm)
			return 0;
	}
	sbpp = sbpm[(ea >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)];
	if (!sbpp)
		return 0;
	spp = sbpp[(ea >> PAGE_SHIFT) & (SBP_L1_COUNT - 1)];

	/* extract 2-bit bitfield for this 4k subpage */
	spp >>= 30 - 2 * ((ea >> 12) & 0xf);

	/* turn 0,1,2,3 into combination of _PAGE_USER and _PAGE_RW */
	spp = ((spp & 2) ? _PAGE_USER : 0) | ((spp & 1) ? _PAGE_RW : 0);
	return spp;
}

#else /* CONFIG_PPC_SUBPAGE_PROT */
868
static inline int subpage_protection(pgd_t *pgdir, unsigned long ea)
869 870 871 872 873
{
	return 0;
}
#endif

L
Linus Torvalds 已提交
874 875 876 877
/* Result code is:
 *  0 - handled
 *  1 - normal page fault
 * -1 - critical hash insertion error
878
 * -2 - access not permitted by subpage protection mechanism
L
Linus Torvalds 已提交
879 880 881 882 883 884 885
 */
int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
{
	void *pgdir;
	unsigned long vsid;
	struct mm_struct *mm;
	pte_t *ptep;
886
	unsigned hugeshift;
887
	const struct cpumask *tmp;
888
	int rc, user_region = 0, local = 0;
P
Paul Mackerras 已提交
889
	int psize, ssize;
L
Linus Torvalds 已提交
890

891 892
	DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
		ea, access, trap);
893

894 895 896 897 898 899
	if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) {
		DBG_LOW(" out of pgtable range !\n");
 		return 1;
	}

	/* Get region & vsid */
L
Linus Torvalds 已提交
900 901 902 903
 	switch (REGION_ID(ea)) {
	case USER_REGION_ID:
		user_region = 1;
		mm = current->mm;
904 905
		if (! mm) {
			DBG_LOW(" user region with no mm !\n");
L
Linus Torvalds 已提交
906
			return 1;
907
		}
908
		psize = get_slice_psize(mm, ea);
P
Paul Mackerras 已提交
909 910
		ssize = user_segment_size(ea);
		vsid = get_vsid(mm->context.id, ea, ssize);
L
Linus Torvalds 已提交
911 912 913
		break;
	case VMALLOC_REGION_ID:
		mm = &init_mm;
P
Paul Mackerras 已提交
914
		vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
915 916 917 918
		if (ea < VMALLOC_END)
			psize = mmu_vmalloc_psize;
		else
			psize = mmu_io_psize;
P
Paul Mackerras 已提交
919
		ssize = mmu_kernel_ssize;
L
Linus Torvalds 已提交
920 921 922 923 924 925 926
		break;
	default:
		/* Not a valid range
		 * Send the problem up to do_page_fault 
		 */
		return 1;
	}
927
	DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
L
Linus Torvalds 已提交
928

929
	/* Get pgdir */
L
Linus Torvalds 已提交
930 931 932 933
	pgdir = mm->pgd;
	if (pgdir == NULL)
		return 1;

934
	/* Check CPU locality */
935 936
	tmp = cpumask_of(smp_processor_id());
	if (user_region && cpumask_equal(mm_cpumask(mm), tmp))
L
Linus Torvalds 已提交
937 938
		local = 1;

939
#ifndef CONFIG_PPC_64K_PAGES
940 941 942 943 944 945
	/* If we use 4K pages and our psize is not 4K, then we might
	 * be hitting a special driver mapping, and need to align the
	 * address before we fetch the PTE.
	 *
	 * It could also be a hugepage mapping, in which case this is
	 * not necessary, but it's not harmful, either.
946 947 948 949 950
	 */
	if (psize != MMU_PAGE_4K)
		ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
#endif /* CONFIG_PPC_64K_PAGES */

951
	/* Get PTE and page size from page tables */
952
	ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugeshift);
953 954 955 956 957
	if (ptep == NULL || !pte_present(*ptep)) {
		DBG_LOW(" no PTE !\n");
		return 1;
	}

958 959 960 961 962 963
#ifdef CONFIG_HUGETLB_PAGE
	if (hugeshift)
		return __hash_page_huge(ea, access, vsid, ptep, trap, local,
					ssize, hugeshift, psize);
#endif /* CONFIG_HUGETLB_PAGE */

964 965 966 967 968 969 970 971 972 973 974 975
#ifndef CONFIG_PPC_64K_PAGES
	DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
#else
	DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
		pte_val(*(ptep + PTRS_PER_PTE)));
#endif
	/* Pre-check access permissions (will be re-checked atomically
	 * in __hash_page_XX but this pre-check is a fast path
	 */
	if (access & ~pte_val(*ptep)) {
		DBG_LOW(" no access !\n");
		return 1;
L
Linus Torvalds 已提交
976 977
	}

978
	/* Do actual hashing */
979
#ifdef CONFIG_PPC_64K_PAGES
980
	/* If _PAGE_4K_PFN is set, make sure this is a 4k segment */
981
	if ((pte_val(*ptep) & _PAGE_4K_PFN) && psize == MMU_PAGE_64K) {
982 983 984 985
		demote_segment_4k(mm, ea);
		psize = MMU_PAGE_4K;
	}

986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
	/* If this PTE is non-cacheable and we have restrictions on
	 * using non cacheable large pages, then we switch to 4k
	 */
	if (mmu_ci_restrictions && psize == MMU_PAGE_64K &&
	    (pte_val(*ptep) & _PAGE_NO_CACHE)) {
		if (user_region) {
			demote_segment_4k(mm, ea);
			psize = MMU_PAGE_4K;
		} else if (ea < VMALLOC_END) {
			/*
			 * some driver did a non-cacheable mapping
			 * in vmalloc space, so switch vmalloc
			 * to 4k pages
			 */
			printk(KERN_ALERT "Reducing vmalloc segment "
			       "to 4kB pages because of "
			       "non-cacheable mapping\n");
			psize = mmu_vmalloc_psize = MMU_PAGE_4K;
1004
#ifdef CONFIG_SPU_BASE
1005 1006
			spu_flush_all_slbs(mm);
#endif
1007
		}
1008 1009
	}
	if (user_region) {
1010
		if (psize != get_paca_psize(ea)) {
1011
			get_paca()->context = mm->context;
1012 1013
			slb_flush_and_rebolt();
		}
1014 1015 1016 1017
	} else if (get_paca()->vmalloc_sllp !=
		   mmu_psize_defs[mmu_vmalloc_psize].sllp) {
		get_paca()->vmalloc_sllp =
			mmu_psize_defs[mmu_vmalloc_psize].sllp;
1018
		slb_vmalloc_update();
1019
	}
1020
#endif /* CONFIG_PPC_64K_PAGES */
1021

1022
#ifdef CONFIG_PPC_HAS_HASH_64K
1023
	if (psize == MMU_PAGE_64K)
P
Paul Mackerras 已提交
1024
		rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
1025
	else
1026
#endif /* CONFIG_PPC_HAS_HASH_64K */
1027 1028 1029 1030 1031 1032 1033 1034
	{
		int spp = subpage_protection(pgdir, ea);
		if (access & spp)
			rc = -2;
		else
			rc = __hash_page_4K(ea, access, vsid, ptep, trap,
					    local, ssize, spp);
	}
1035 1036 1037 1038 1039 1040 1041 1042 1043

#ifndef CONFIG_PPC_64K_PAGES
	DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
#else
	DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
		pte_val(*(ptep + PTRS_PER_PTE)));
#endif
	DBG_LOW(" -> rc=%d\n", rc);
	return rc;
L
Linus Torvalds 已提交
1044
}
1045
EXPORT_SYMBOL_GPL(hash_page);
L
Linus Torvalds 已提交
1046

1047 1048
void hash_preload(struct mm_struct *mm, unsigned long ea,
		  unsigned long access, unsigned long trap)
L
Linus Torvalds 已提交
1049
{
1050 1051 1052 1053 1054
	unsigned long vsid;
	void *pgdir;
	pte_t *ptep;
	unsigned long flags;
	int local = 0;
P
Paul Mackerras 已提交
1055
	int ssize;
1056

1057 1058 1059 1060
	BUG_ON(REGION_ID(ea) != USER_REGION_ID);

#ifdef CONFIG_PPC_MM_SLICES
	/* We only prefault standard pages for now */
1061
	if (unlikely(get_slice_psize(mm, ea) != mm->context.user_psize))
1062
		return;
1063
#endif
1064 1065 1066

	DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
		" trap=%lx\n", mm, mm->pgd, ea, access, trap);
L
Linus Torvalds 已提交
1067

1068
	/* Get Linux PTE if available */
1069 1070 1071 1072 1073 1074
	pgdir = mm->pgd;
	if (pgdir == NULL)
		return;
	ptep = find_linux_pte(pgdir, ea);
	if (!ptep)
		return;
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087

#ifdef CONFIG_PPC_64K_PAGES
	/* If either _PAGE_4K_PFN or _PAGE_NO_CACHE is set (and we are on
	 * a 64K kernel), then we don't preload, hash_page() will take
	 * care of it once we actually try to access the page.
	 * That way we don't have to duplicate all of the logic for segment
	 * page size demotion here
	 */
	if (pte_val(*ptep) & (_PAGE_4K_PFN | _PAGE_NO_CACHE))
		return;
#endif /* CONFIG_PPC_64K_PAGES */

	/* Get VSID */
P
Paul Mackerras 已提交
1088 1089
	ssize = user_segment_size(ea);
	vsid = get_vsid(mm->context.id, ea, ssize);
1090

1091
	/* Hash doesn't like irqs */
1092
	local_irq_save(flags);
1093 1094

	/* Is that local to this CPU ? */
1095
	if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
1096
		local = 1;
1097 1098 1099

	/* Hash it in */
#ifdef CONFIG_PPC_HAS_HASH_64K
1100
	if (mm->context.user_psize == MMU_PAGE_64K)
P
Paul Mackerras 已提交
1101
		__hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
L
Linus Torvalds 已提交
1102
	else
J
Jon Tollefson 已提交
1103
#endif /* CONFIG_PPC_HAS_HASH_64K */
1104 1105
		__hash_page_4K(ea, access, vsid, ptep, trap, local, ssize,
			       subpage_protection(pgdir, ea));
1106

1107 1108 1109
	local_irq_restore(flags);
}

1110 1111 1112
/* WARNING: This is called from hash_low_64.S, if you change this prototype,
 *          do not forget to update the assembly call site !
 */
P
Paul Mackerras 已提交
1113 1114
void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int ssize,
		     int local)
1115 1116 1117 1118 1119
{
	unsigned long hash, index, shift, hidx, slot;

	DBG_LOW("flush_hash_page(va=%016x)\n", va);
	pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
P
Paul Mackerras 已提交
1120
		hash = hpt_hash(va, shift, ssize);
1121 1122 1123 1124 1125 1126
		hidx = __rpte_to_hidx(pte, index);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;
		DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx);
P
Paul Mackerras 已提交
1127
		ppc_md.hpte_invalidate(slot, va, psize, ssize, local);
1128
	} pte_iterate_hashed_end();
L
Linus Torvalds 已提交
1129 1130
}

1131
void flush_hash_range(unsigned long number, int local)
L
Linus Torvalds 已提交
1132
{
1133
	if (ppc_md.flush_hash_range)
1134
		ppc_md.flush_hash_range(number, local);
1135
	else {
L
Linus Torvalds 已提交
1136
		int i;
1137 1138
		struct ppc64_tlb_batch *batch =
			&__get_cpu_var(ppc64_tlb_batch);
L
Linus Torvalds 已提交
1139 1140

		for (i = 0; i < number; i++)
1141
			flush_hash_page(batch->vaddr[i], batch->pte[i],
P
Paul Mackerras 已提交
1142
					batch->psize, batch->ssize, local);
L
Linus Torvalds 已提交
1143 1144 1145 1146 1147 1148 1149
	}
}

/*
 * low_hash_fault is called when we the low level hash code failed
 * to instert a PTE due to an hypervisor error
 */
1150
void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
L
Linus Torvalds 已提交
1151 1152
{
	if (user_mode(regs)) {
1153 1154 1155 1156 1157 1158 1159 1160
#ifdef CONFIG_PPC_SUBPAGE_PROT
		if (rc == -2)
			_exception(SIGSEGV, regs, SEGV_ACCERR, address);
		else
#endif
			_exception(SIGBUS, regs, BUS_ADRERR, address);
	} else
		bad_page_fault(regs, address, SIGBUS);
L
Linus Torvalds 已提交
1161
}
1162 1163 1164 1165

#ifdef CONFIG_DEBUG_PAGEALLOC
static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi)
{
P
Paul Mackerras 已提交
1166 1167 1168
	unsigned long hash, hpteg;
	unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
	unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
1169
	unsigned long mode = htab_convert_pte_flags(PAGE_KERNEL);
1170 1171
	int ret;

P
Paul Mackerras 已提交
1172
	hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
1173 1174 1175
	hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);

	ret = ppc_md.hpte_insert(hpteg, va, __pa(vaddr),
P
Paul Mackerras 已提交
1176 1177
				 mode, HPTE_V_BOLTED,
				 mmu_linear_psize, mmu_kernel_ssize);
1178 1179 1180 1181 1182 1183 1184 1185 1186
	BUG_ON (ret < 0);
	spin_lock(&linear_map_hash_lock);
	BUG_ON(linear_map_hash_slots[lmi] & 0x80);
	linear_map_hash_slots[lmi] = ret | 0x80;
	spin_unlock(&linear_map_hash_lock);
}

static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
{
P
Paul Mackerras 已提交
1187 1188 1189
	unsigned long hash, hidx, slot;
	unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
	unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
1190

P
Paul Mackerras 已提交
1191
	hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
1192 1193 1194 1195 1196 1197 1198 1199 1200
	spin_lock(&linear_map_hash_lock);
	BUG_ON(!(linear_map_hash_slots[lmi] & 0x80));
	hidx = linear_map_hash_slots[lmi] & 0x7f;
	linear_map_hash_slots[lmi] = 0;
	spin_unlock(&linear_map_hash_lock);
	if (hidx & _PTEIDX_SECONDARY)
		hash = ~hash;
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	slot += hidx & _PTEIDX_GROUP_IX;
P
Paul Mackerras 已提交
1201
	ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, mmu_kernel_ssize, 0);
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
}

void kernel_map_pages(struct page *page, int numpages, int enable)
{
	unsigned long flags, vaddr, lmi;
	int i;

	local_irq_save(flags);
	for (i = 0; i < numpages; i++, page++) {
		vaddr = (unsigned long)page_address(page);
		lmi = __pa(vaddr) >> PAGE_SHIFT;
		if (lmi >= linear_map_hash_count)
			continue;
		if (enable)
			kernel_map_linear_page(vaddr, lmi);
		else
			kernel_unmap_linear_page(vaddr, lmi);
	}
	local_irq_restore(flags);
}
#endif /* CONFIG_DEBUG_PAGEALLOC */