hash_utils_64.c 19.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * PowerPC64 port by Mike Corrigan and Dave Engebretsen
 *   {mikejc|engebret}@us.ibm.com
 *
 *    Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
 *
 * SMP scalability work:
 *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
 * 
 *    Module name: htab.c
 *
 *    Description:
 *      PowerPC Hashed Page Table functions
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#undef DEBUG
22
#undef DEBUG_LOW
L
Linus Torvalds 已提交
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

#include <linux/config.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/sysctl.h>
#include <linux/ctype.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/signal.h>

#include <asm/processor.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
#include <asm/page.h>
#include <asm/types.h>
#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/machdep.h>
#include <asm/lmb.h>
#include <asm/abs_addr.h>
#include <asm/tlbflush.h>
#include <asm/io.h>
#include <asm/eeh.h>
#include <asm/tlb.h>
#include <asm/cacheflush.h>
#include <asm/cputable.h>
#include <asm/abs_addr.h>
#include <asm/sections.h>

#ifdef DEBUG
#define DBG(fmt...) udbg_printf(fmt)
#else
#define DBG(fmt...)
#endif

62 63 64 65 66 67 68 69 70
#ifdef DEBUG_LOW
#define DBG_LOW(fmt...) udbg_printf(fmt)
#else
#define DBG_LOW(fmt...)
#endif

#define KB (1024)
#define MB (1024*KB)

L
Linus Torvalds 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * Note:  pte   --> Linux PTE
 *        HPTE  --> PowerPC Hashed Page Table Entry
 *
 * Execution context:
 *   htab_initialize is called with the MMU off (of course), but
 *   the kernel has been copied down to zero so it can directly
 *   reference global data.  At this point it is very difficult
 *   to print debug info.
 *
 */

#ifdef CONFIG_U3_DART
extern unsigned long dart_tablebase;
#endif /* CONFIG_U3_DART */

87 88 89
static unsigned long _SDR1;
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];

90
hpte_t *htab_address;
91
unsigned long htab_size_bytes;
92
unsigned long htab_hash_mask;
93 94 95 96 97 98
int mmu_linear_psize = MMU_PAGE_4K;
int mmu_virtual_psize = MMU_PAGE_4K;
#ifdef CONFIG_HUGETLB_PAGE
int mmu_huge_psize = MMU_PAGE_16M;
unsigned int HPAGE_SHIFT;
#endif
L
Linus Torvalds 已提交
99

100 101 102
/* There are definitions of page sizes arrays to be used when none
 * is provided by the firmware.
 */
L
Linus Torvalds 已提交
103

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
/* Pre-POWER4 CPUs (4k pages only)
 */
struct mmu_psize_def mmu_psize_defaults_old[] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.sllp	= 0,
		.penc	= 0,
		.avpnm	= 0,
		.tlbiel = 0,
	},
};

/* POWER4, GPUL, POWER5
 *
 * Support for 16Mb large pages
 */
struct mmu_psize_def mmu_psize_defaults_gp[] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.sllp	= 0,
		.penc	= 0,
		.avpnm	= 0,
		.tlbiel = 1,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.sllp	= SLB_VSID_L,
		.penc	= 0,
		.avpnm	= 0x1UL,
		.tlbiel = 0,
	},
};


int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
		      unsigned long pstart, unsigned long mode, int psize)
L
Linus Torvalds 已提交
140
{
141 142
	unsigned long vaddr, paddr;
	unsigned int step, shift;
L
Linus Torvalds 已提交
143
	unsigned long tmp_mode;
144
	int ret = 0;
L
Linus Torvalds 已提交
145

146 147
	shift = mmu_psize_defs[psize].shift;
	step = 1 << shift;
L
Linus Torvalds 已提交
148

149 150
	for (vaddr = vstart, paddr = pstart; vaddr < vend;
	     vaddr += step, paddr += step) {
L
Linus Torvalds 已提交
151
		unsigned long vpn, hash, hpteg;
152 153
		unsigned long vsid = get_kernel_vsid(vaddr);
		unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff);
L
Linus Torvalds 已提交
154

155
		vpn = va >> shift;
L
Linus Torvalds 已提交
156 157 158
		tmp_mode = mode;
		
		/* Make non-kernel text non-executable */
159 160
		if (!in_kernel_text(vaddr))
			tmp_mode = mode | HPTE_R_N;
L
Linus Torvalds 已提交
161

162
		hash = hpt_hash(va, shift);
L
Linus Torvalds 已提交
163 164
		hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);

165 166 167 168
		/* The crap below can be cleaned once ppd_md.probe() can
		 * set up the hash callbacks, thus we can just used the
		 * normal insert callback here.
		 */
169
#ifdef CONFIG_PPC_ISERIES
170
		if (_machine == PLATFORM_ISERIES_LPAR)
171 172 173 174 175
			ret = iSeries_hpte_insert(hpteg, va,
						  virt_to_abs(paddr),
						  tmp_mode,
						  HPTE_V_BOLTED,
						  psize);
176 177
		else
#endif
L
Linus Torvalds 已提交
178
#ifdef CONFIG_PPC_PSERIES
179
		if (_machine & PLATFORM_LPAR)
L
Linus Torvalds 已提交
180
			ret = pSeries_lpar_hpte_insert(hpteg, va,
181 182 183 184
						       virt_to_abs(paddr),
						       tmp_mode,
						       HPTE_V_BOLTED,
						       psize);
L
Linus Torvalds 已提交
185
		else
186 187
#endif
#ifdef CONFIG_PPC_MULTIPLATFORM
L
Linus Torvalds 已提交
188
			ret = native_hpte_insert(hpteg, va,
189 190 191
						 virt_to_abs(paddr),
						 tmp_mode, HPTE_V_BOLTED,
						 psize);
192
#endif
193 194 195 196 197
		if (ret < 0)
			break;
	}
	return ret < 0 ? ret : 0;
}
L
Linus Torvalds 已提交
198

199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
static int __init htab_dt_scan_page_sizes(unsigned long node,
					  const char *uname, int depth,
					  void *data)
{
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	u32 *prop;
	unsigned long size = 0;

	/* We are scanning "cpu" nodes only */
	if (type == NULL || strcmp(type, "cpu") != 0)
		return 0;

	prop = (u32 *)of_get_flat_dt_prop(node,
					  "ibm,segment-page-sizes", &size);
	if (prop != NULL) {
		DBG("Page sizes from device-tree:\n");
		size /= 4;
		cur_cpu_spec->cpu_features &= ~(CPU_FTR_16M_PAGE);
		while(size > 0) {
			unsigned int shift = prop[0];
			unsigned int slbenc = prop[1];
			unsigned int lpnum = prop[2];
			unsigned int lpenc = 0;
			struct mmu_psize_def *def;
			int idx = -1;

			size -= 3; prop += 3;
			while(size > 0 && lpnum) {
				if (prop[0] == shift)
					lpenc = prop[1];
				prop += 2; size -= 2;
				lpnum--;
			}
			switch(shift) {
			case 0xc:
				idx = MMU_PAGE_4K;
				break;
			case 0x10:
				idx = MMU_PAGE_64K;
				break;
			case 0x14:
				idx = MMU_PAGE_1M;
				break;
			case 0x18:
				idx = MMU_PAGE_16M;
				cur_cpu_spec->cpu_features |= CPU_FTR_16M_PAGE;
				break;
			case 0x22:
				idx = MMU_PAGE_16G;
				break;
			}
			if (idx < 0)
				continue;
			def = &mmu_psize_defs[idx];
			def->shift = shift;
			if (shift <= 23)
				def->avpnm = 0;
			else
				def->avpnm = (1 << (shift - 23)) - 1;
			def->sllp = slbenc;
			def->penc = lpenc;
			/* We don't know for sure what's up with tlbiel, so
			 * for now we only set it for 4K and 64K pages
			 */
			if (idx == MMU_PAGE_4K || idx == MMU_PAGE_64K)
				def->tlbiel = 1;
			else
				def->tlbiel = 0;

			DBG(" %d: shift=%02x, sllp=%04x, avpnm=%08x, "
			    "tlbiel=%d, penc=%d\n",
			    idx, shift, def->sllp, def->avpnm, def->tlbiel,
			    def->penc);
L
Linus Torvalds 已提交
272
		}
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
		return 1;
	}
	return 0;
}


static void __init htab_init_page_sizes(void)
{
	int rc;

	/* Default to 4K pages only */
	memcpy(mmu_psize_defs, mmu_psize_defaults_old,
	       sizeof(mmu_psize_defaults_old));

	/*
	 * Try to find the available page sizes in the device-tree
	 */
	rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL);
	if (rc != 0)  /* Found */
		goto found;

	/*
	 * Not in the device-tree, let's fallback on known size
	 * list for 16M capable GP & GR
	 */
298
	if ((_machine != PLATFORM_ISERIES_LPAR) &&
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
	    cpu_has_feature(CPU_FTR_16M_PAGE))
		memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
		       sizeof(mmu_psize_defaults_gp));
 found:
	/*
	 * Pick a size for the linear mapping. Currently, we only support
	 * 16M, 1M and 4K which is the default
	 */
	if (mmu_psize_defs[MMU_PAGE_16M].shift)
		mmu_linear_psize = MMU_PAGE_16M;
	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
		mmu_linear_psize = MMU_PAGE_1M;

	/*
	 * Pick a size for the ordinary pages. Default is 4K, we support
	 * 64K if cache inhibited large pages are supported by the
	 * processor
	 */
#ifdef CONFIG_PPC_64K_PAGES
	if (mmu_psize_defs[MMU_PAGE_64K].shift &&
	    cpu_has_feature(CPU_FTR_CI_LARGE_PAGE))
		mmu_virtual_psize = MMU_PAGE_64K;
#endif

	printk(KERN_INFO "Page orders: linear mapping = %d, others = %d\n",
	       mmu_psize_defs[mmu_linear_psize].shift,
	       mmu_psize_defs[mmu_virtual_psize].shift);

#ifdef CONFIG_HUGETLB_PAGE
	/* Init large page size. Currently, we pick 16M or 1M depending
	 * on what is available
	 */
	if (mmu_psize_defs[MMU_PAGE_16M].shift)
		mmu_huge_psize = MMU_PAGE_16M;
333 334
	/* With 4k/4level pagetables, we can't (for now) cope with a
	 * huge page size < PMD_SIZE */
335 336 337 338
	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
		mmu_huge_psize = MMU_PAGE_1M;

	/* Calculate HPAGE_SHIFT and sanity check it */
339 340
	if (mmu_psize_defs[mmu_huge_psize].shift > MIN_HUGEPTE_SHIFT &&
	    mmu_psize_defs[mmu_huge_psize].shift < SID_SHIFT)
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
		HPAGE_SHIFT = mmu_psize_defs[mmu_huge_psize].shift;
	else
		HPAGE_SHIFT = 0; /* No huge pages dude ! */
#endif /* CONFIG_HUGETLB_PAGE */
}

static int __init htab_dt_scan_pftsize(unsigned long node,
				       const char *uname, int depth,
				       void *data)
{
	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
	u32 *prop;

	/* We are scanning "cpu" nodes only */
	if (type == NULL || strcmp(type, "cpu") != 0)
		return 0;

	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,pft-size", NULL);
	if (prop != NULL) {
		/* pft_size[0] is the NUMA CEC cookie */
		ppc64_pft_size = prop[1];
		return 1;
L
Linus Torvalds 已提交
363
	}
364
	return 0;
L
Linus Torvalds 已提交
365 366
}

367
static unsigned long __init htab_get_table_size(void)
368
{
369
	unsigned long mem_size, rnd_mem_size, pteg_count;
370

371
	/* If hash size isn't already provided by the platform, we try to
A
Adrian Bunk 已提交
372
	 * retrieve it from the device-tree. If it's not there neither, we
373
	 * calculate it now based on the total RAM size
374
	 */
375 376
	if (ppc64_pft_size == 0)
		of_scan_flat_dt(htab_dt_scan_pftsize, NULL);
377 378 379 380
	if (ppc64_pft_size)
		return 1UL << ppc64_pft_size;

	/* round mem_size up to next power of 2 */
381 382 383
	mem_size = lmb_phys_mem_size();
	rnd_mem_size = 1UL << __ilog2(mem_size);
	if (rnd_mem_size < mem_size)
384 385 386 387 388 389 390 391
		rnd_mem_size <<= 1;

	/* # pages / 2 */
	pteg_count = max(rnd_mem_size >> (12 + 1), 1UL << 11);

	return pteg_count << 7;
}

392 393 394 395 396 397 398 399 400
#ifdef CONFIG_MEMORY_HOTPLUG
void create_section_mapping(unsigned long start, unsigned long end)
{
		BUG_ON(htab_bolt_mapping(start, end, start,
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX,
			mmu_linear_psize));
}
#endif /* CONFIG_MEMORY_HOTPLUG */

L
Linus Torvalds 已提交
401 402
void __init htab_initialize(void)
{
403
	unsigned long table;
L
Linus Torvalds 已提交
404 405 406
	unsigned long pteg_count;
	unsigned long mode_rw;
	unsigned long base = 0, size = 0;
407 408
	int i;

L
Linus Torvalds 已提交
409 410 411 412
	extern unsigned long tce_alloc_start, tce_alloc_end;

	DBG(" -> htab_initialize()\n");

413 414 415
	/* Initialize page sizes */
	htab_init_page_sizes();

L
Linus Torvalds 已提交
416 417 418 419
	/*
	 * Calculate the required size of the htab.  We want the number of
	 * PTEGs to equal one half the number of real pages.
	 */ 
420
	htab_size_bytes = htab_get_table_size();
L
Linus Torvalds 已提交
421 422 423 424
	pteg_count = htab_size_bytes >> 7;

	htab_hash_mask = pteg_count - 1;

425
	if (platform_is_lpar()) {
L
Linus Torvalds 已提交
426 427 428 429 430 431 432 433
		/* Using a hypervisor which owns the htab */
		htab_address = NULL;
		_SDR1 = 0; 
	} else {
		/* Find storage for the HPT.  Must be contiguous in
		 * the absolute address space.
		 */
		table = lmb_alloc(htab_size_bytes, htab_size_bytes);
434
		BUG_ON(table == 0);
L
Linus Torvalds 已提交
435 436 437 438 439 440 441 442 443 444 445

		DBG("Hash table allocated at %lx, size: %lx\n", table,
		    htab_size_bytes);

		htab_address = abs_to_virt(table);

		/* htab absolute addr + encoded htabsize */
		_SDR1 = table + __ilog2(pteg_count) - 11;

		/* Initialize the HPT with no entries */
		memset((void *)table, 0, htab_size_bytes);
446 447 448

		/* Set SDR1 */
		mtspr(SPRN_SDR1, _SDR1);
L
Linus Torvalds 已提交
449 450
	}

451
	mode_rw = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX;
L
Linus Torvalds 已提交
452 453 454 455 456 457 458 459

	/* On U3 based machines, we need to reserve the DART area and
	 * _NOT_ map it to avoid cache paradoxes as it's remapped non
	 * cacheable later on
	 */

	/* create bolted the linear mapping in the hash table */
	for (i=0; i < lmb.memory.cnt; i++) {
460
		base = (unsigned long)__va(lmb.memory.region[i].base);
L
Linus Torvalds 已提交
461 462 463 464 465 466
		size = lmb.memory.region[i].size;

		DBG("creating mapping for region: %lx : %lx\n", base, size);

#ifdef CONFIG_U3_DART
		/* Do not map the DART space. Fortunately, it will be aligned
467 468 469 470 471
		 * in such a way that it will not cross two lmb regions and
		 * will fit within a single 16Mb page.
		 * The DART space is assumed to be a full 16Mb region even if
		 * we only use 2Mb of that space. We will use more of it later
		 * for AGP GART. We have to use a full 16Mb large page.
L
Linus Torvalds 已提交
472 473 474 475 476 477
		 */
		DBG("DART base: %lx\n", dart_tablebase);

		if (dart_tablebase != 0 && dart_tablebase >= base
		    && dart_tablebase < (base + size)) {
			if (base != dart_tablebase)
478 479 480
				BUG_ON(htab_bolt_mapping(base, dart_tablebase,
							 base, mode_rw,
							 mmu_linear_psize));
L
Linus Torvalds 已提交
481
			if ((base + size) > (dart_tablebase + 16*MB))
482 483 484 485 486
				BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
							 base + size,
							 dart_tablebase+16*MB,
							 mode_rw,
							 mmu_linear_psize));
L
Linus Torvalds 已提交
487 488 489
			continue;
		}
#endif /* CONFIG_U3_DART */
490 491 492
		BUG_ON(htab_bolt_mapping(base, base + size, base,
					 mode_rw, mmu_linear_psize));
       }
L
Linus Torvalds 已提交
493 494 495 496 497 498 499 500 501

	/*
	 * If we have a memory_limit and we've allocated TCEs then we need to
	 * explicitly map the TCE area at the top of RAM. We also cope with the
	 * case that the TCEs start below memory_limit.
	 * tce_alloc_start/end are 16MB aligned so the mapping should work
	 * for either 4K or 16MB pages.
	 */
	if (tce_alloc_start) {
502 503
		tce_alloc_start = (unsigned long)__va(tce_alloc_start);
		tce_alloc_end = (unsigned long)__va(tce_alloc_end);
L
Linus Torvalds 已提交
504 505 506 507

		if (base + size >= tce_alloc_start)
			tce_alloc_start = base + size + 1;

508 509 510
 		BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
 					 tce_alloc_start, mode_rw,
					 mmu_linear_psize));
L
Linus Torvalds 已提交
511 512 513 514 515 516 517
	}

	DBG(" <- htab_initialize()\n");
}
#undef KB
#undef MB

518
void htab_initialize_secondary(void)
519 520 521 522 523
{
	if (!platform_is_lpar())
		mtspr(SPRN_SDR1, _SDR1);
}

L
Linus Torvalds 已提交
524 525 526 527 528 529 530
/*
 * Called by asm hashtable.S for doing lazy icache flush
 */
unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
{
	struct page *page;

531 532 533
	if (!pfn_valid(pte_pfn(pte)))
		return pp;

L
Linus Torvalds 已提交
534 535 536 537 538 539 540 541
	page = pte_page(pte);

	/* page is dirty */
	if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
		if (trap == 0x400) {
			__flush_dcache_icache(page_address(page));
			set_bit(PG_arch_1, &page->flags);
		} else
542
			pp |= HPTE_R_N;
L
Linus Torvalds 已提交
543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
	}
	return pp;
}

/* Result code is:
 *  0 - handled
 *  1 - normal page fault
 * -1 - critical hash insertion error
 */
int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
{
	void *pgdir;
	unsigned long vsid;
	struct mm_struct *mm;
	pte_t *ptep;
	cpumask_t tmp;
559
	int rc, user_region = 0, local = 0;
L
Linus Torvalds 已提交
560

561 562
	DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
		ea, access, trap);
563

564 565 566 567 568 569
	if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) {
		DBG_LOW(" out of pgtable range !\n");
 		return 1;
	}

	/* Get region & vsid */
L
Linus Torvalds 已提交
570 571 572 573
 	switch (REGION_ID(ea)) {
	case USER_REGION_ID:
		user_region = 1;
		mm = current->mm;
574 575
		if (! mm) {
			DBG_LOW(" user region with no mm !\n");
L
Linus Torvalds 已提交
576
			return 1;
577
		}
L
Linus Torvalds 已提交
578 579 580 581 582 583 584 585 586 587 588 589
		vsid = get_vsid(mm->context.id, ea);
		break;
	case VMALLOC_REGION_ID:
		mm = &init_mm;
		vsid = get_kernel_vsid(ea);
		break;
	default:
		/* Not a valid range
		 * Send the problem up to do_page_fault 
		 */
		return 1;
	}
590
	DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
L
Linus Torvalds 已提交
591

592
	/* Get pgdir */
L
Linus Torvalds 已提交
593 594 595 596
	pgdir = mm->pgd;
	if (pgdir == NULL)
		return 1;

597
	/* Check CPU locality */
L
Linus Torvalds 已提交
598 599 600 601
	tmp = cpumask_of_cpu(smp_processor_id());
	if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
		local = 1;

602 603 604
	/* Handle hugepage regions */
	if (unlikely(in_hugepage_area(mm->context, ea))) {
		DBG_LOW(" -> huge page !\n");
605
		return hash_huge_page(mm, access, ea, vsid, local, trap);
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
	}

	/* Get PTE and page size from page tables */
	ptep = find_linux_pte(pgdir, ea);
	if (ptep == NULL || !pte_present(*ptep)) {
		DBG_LOW(" no PTE !\n");
		return 1;
	}

#ifndef CONFIG_PPC_64K_PAGES
	DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
#else
	DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
		pte_val(*(ptep + PTRS_PER_PTE)));
#endif
	/* Pre-check access permissions (will be re-checked atomically
	 * in __hash_page_XX but this pre-check is a fast path
	 */
	if (access & ~pte_val(*ptep)) {
		DBG_LOW(" no access !\n");
		return 1;
L
Linus Torvalds 已提交
627 628
	}

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
	/* Do actual hashing */
#ifndef CONFIG_PPC_64K_PAGES
	rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
#else
	if (mmu_virtual_psize == MMU_PAGE_64K)
		rc = __hash_page_64K(ea, access, vsid, ptep, trap, local);
	else
		rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
#endif /* CONFIG_PPC_64K_PAGES */

#ifndef CONFIG_PPC_64K_PAGES
	DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
#else
	DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
		pte_val(*(ptep + PTRS_PER_PTE)));
#endif
	DBG_LOW(" -> rc=%d\n", rc);
	return rc;
L
Linus Torvalds 已提交
647
}
648
EXPORT_SYMBOL_GPL(hash_page);
L
Linus Torvalds 已提交
649

650 651
void hash_preload(struct mm_struct *mm, unsigned long ea,
		  unsigned long access, unsigned long trap)
L
Linus Torvalds 已提交
652
{
653 654 655 656 657 658 659 660 661 662 663 664 665 666
	unsigned long vsid;
	void *pgdir;
	pte_t *ptep;
	cpumask_t mask;
	unsigned long flags;
	int local = 0;

	/* We don't want huge pages prefaulted for now
	 */
	if (unlikely(in_hugepage_area(mm->context, ea)))
		return;

	DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
		" trap=%lx\n", mm, mm->pgd, ea, access, trap);
L
Linus Torvalds 已提交
667

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
	/* Get PTE, VSID, access mask */
	pgdir = mm->pgd;
	if (pgdir == NULL)
		return;
	ptep = find_linux_pte(pgdir, ea);
	if (!ptep)
		return;
	vsid = get_vsid(mm->context.id, ea);

	/* Hash it in */
	local_irq_save(flags);
	mask = cpumask_of_cpu(smp_processor_id());
	if (cpus_equal(mm->cpu_vm_mask, mask))
		local = 1;
#ifndef CONFIG_PPC_64K_PAGES
	__hash_page_4K(ea, access, vsid, ptep, trap, local);
#else
	if (mmu_virtual_psize == MMU_PAGE_64K)
		__hash_page_64K(ea, access, vsid, ptep, trap, local);
L
Linus Torvalds 已提交
687
	else
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707
		__hash_page_4K(ea, access, vsid, ptep, trap, local);
#endif /* CONFIG_PPC_64K_PAGES */
	local_irq_restore(flags);
}

void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int local)
{
	unsigned long hash, index, shift, hidx, slot;

	DBG_LOW("flush_hash_page(va=%016x)\n", va);
	pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
		hash = hpt_hash(va, shift);
		hidx = __rpte_to_hidx(pte, index);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;
		DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx);
		ppc_md.hpte_invalidate(slot, va, psize, local);
	} pte_iterate_hashed_end();
L
Linus Torvalds 已提交
708 709
}

710
void flush_hash_range(unsigned long number, int local)
L
Linus Torvalds 已提交
711
{
712
	if (ppc_md.flush_hash_range)
713
		ppc_md.flush_hash_range(number, local);
714
	else {
L
Linus Torvalds 已提交
715
		int i;
716 717
		struct ppc64_tlb_batch *batch =
			&__get_cpu_var(ppc64_tlb_batch);
L
Linus Torvalds 已提交
718 719

		for (i = 0; i < number; i++)
720 721
			flush_hash_page(batch->vaddr[i], batch->pte[i],
					batch->psize, local);
L
Linus Torvalds 已提交
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
	}
}

static inline void make_bl(unsigned int *insn_addr, void *func)
{
	unsigned long funcp = *((unsigned long *)func);
	int offset = funcp - (unsigned long)insn_addr;

	*insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc));
	flush_icache_range((unsigned long)insn_addr, 4+
			   (unsigned long)insn_addr);
}

/*
 * low_hash_fault is called when we the low level hash code failed
 * to instert a PTE due to an hypervisor error
 */
void low_hash_fault(struct pt_regs *regs, unsigned long address)
{
	if (user_mode(regs)) {
		siginfo_t info;

		info.si_signo = SIGBUS;
		info.si_errno = 0;
		info.si_code = BUS_ADRERR;
		info.si_addr = (void __user *)address;
		force_sig_info(SIGBUS, &info, current);
		return;
	}
	bad_page_fault(regs, address, SIGBUS);
}

void __init htab_finish_init(void)
{
	extern unsigned int *htab_call_hpte_insert1;
	extern unsigned int *htab_call_hpte_insert2;
	extern unsigned int *htab_call_hpte_remove;
	extern unsigned int *htab_call_hpte_updatepp;

761 762 763 764 765 766 767 768 769 770 771 772
#ifdef CONFIG_PPC_64K_PAGES
	extern unsigned int *ht64_call_hpte_insert1;
	extern unsigned int *ht64_call_hpte_insert2;
	extern unsigned int *ht64_call_hpte_remove;
	extern unsigned int *ht64_call_hpte_updatepp;

	make_bl(ht64_call_hpte_insert1, ppc_md.hpte_insert);
	make_bl(ht64_call_hpte_insert2, ppc_md.hpte_insert);
	make_bl(ht64_call_hpte_remove, ppc_md.hpte_remove);
	make_bl(ht64_call_hpte_updatepp, ppc_md.hpte_updatepp);
#endif /* CONFIG_PPC_64K_PAGES */

L
Linus Torvalds 已提交
773 774 775 776 777
	make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert);
	make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert);
	make_bl(htab_call_hpte_remove, ppc_md.hpte_remove);
	make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
}