lpar.c 19.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * pSeries_lpar.c
 * Copyright (C) 2001 Todd Inglett, IBM Corporation
 *
 * pSeries LPAR support.
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

22 23
/* Enables debugging of low-level hash table routines - careful! */
#undef DEBUG
L
Linus Torvalds 已提交
24 25 26

#include <linux/kernel.h>
#include <linux/dma-mapping.h>
27
#include <linux/console.h>
28
#include <linux/export.h>
29
#include <linux/static_key.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40
#include <asm/processor.h>
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/machdep.h>
#include <asm/mmu_context.h>
#include <asm/iommu.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <asm/prom.h>
#include <asm/cputable.h>
D
David Gibson 已提交
41
#include <asm/udbg.h>
P
Paul Mackerras 已提交
42
#include <asm/smp.h>
43
#include <asm/trace.h>
44
#include <asm/firmware.h>
45
#include <asm/plpar_wrappers.h>
46

47
#include "pseries.h"
L
Linus Torvalds 已提交
48

49 50 51 52 53 54 55
/* Flag bits for H_BULK_REMOVE */
#define HBR_REQUEST	0x4000000000000000UL
#define HBR_RESPONSE	0x8000000000000000UL
#define HBR_END		0xc000000000000000UL
#define HBR_AVPN	0x0200000000000000UL
#define HBR_ANDCOND	0x0100000000000000UL

L
Linus Torvalds 已提交
56

57
/* in hvCall.S */
L
Linus Torvalds 已提交
58
EXPORT_SYMBOL(plpar_hcall);
59
EXPORT_SYMBOL(plpar_hcall9);
L
Linus Torvalds 已提交
60
EXPORT_SYMBOL(plpar_hcall_norets);
61

L
Linus Torvalds 已提交
62 63 64
void vpa_init(int cpu)
{
	int hwcpu = get_hard_smp_processor_id(cpu);
65
	unsigned long addr;
L
Linus Torvalds 已提交
66
	long ret;
67 68
	struct paca_struct *pp;
	struct dtl_entry *dtl;
69

70 71 72 73 74 75
	/*
	 * The spec says it "may be problematic" if CPU x registers the VPA of
	 * CPU y. We should never do that, but wail if we ever do.
	 */
	WARN_ON(cpu != smp_processor_id());

76
	if (cpu_has_feature(CPU_FTR_ALTIVEC))
77
		lppaca_of(cpu).vmxregs_in_use = 1;
78

79 80 81
	if (cpu_has_feature(CPU_FTR_ARCH_207S))
		lppaca_of(cpu).ebb_regs_in_use = 1;

82
	addr = __pa(&lppaca_of(cpu));
83
	ret = register_vpa(hwcpu, addr);
L
Linus Torvalds 已提交
84

85
	if (ret) {
86 87
		pr_err("WARNING: VPA registration for cpu %d (hw %d) of area "
		       "%lx failed with %ld\n", cpu, hwcpu, addr, ret);
88 89 90 91 92 93
		return;
	}
	/*
	 * PAPR says this feature is SLB-Buffer but firmware never
	 * reports that.  All SPLPAR support SLB shadow buffer.
	 */
J
Jeremy Kerr 已提交
94
	addr = __pa(paca[cpu].slb_shadow_ptr);
95 96 97
	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
		ret = register_slb_shadow(hwcpu, addr);
		if (ret)
98 99 100
			pr_err("WARNING: SLB shadow buffer registration for "
			       "cpu %d (hw %d) of area %lx failed with %ld\n",
			       cpu, hwcpu, addr, ret);
101
	}
102 103 104 105 106 107 108 109 110 111 112 113

	/*
	 * Register dispatch trace log, if one has been allocated.
	 */
	pp = &paca[cpu];
	dtl = pp->dispatch_log;
	if (dtl) {
		pp->dtl_ridx = 0;
		pp->dtl_curr = dtl;
		lppaca_of(cpu).dtl_idx = 0;

		/* hypervisor reads buffer length from this field */
114
		dtl->enqueue_to_dispatch_time = cpu_to_be32(DISPATCH_LOG_BYTES);
115 116
		ret = register_dtl(hwcpu, __pa(dtl));
		if (ret)
117 118 119
			pr_err("WARNING: DTL registration of cpu %d (hw %d) "
			       "failed with %ld\n", smp_processor_id(),
			       hwcpu, ret);
120 121
		lppaca_of(cpu).dtl_enable_mask = 2;
	}
L
Linus Torvalds 已提交
122 123
}

124
static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
125 126
				     unsigned long vpn, unsigned long pa,
				     unsigned long rflags, unsigned long vflags,
127
				     int psize, int apsize, int ssize)
L
Linus Torvalds 已提交
128 129 130 131
{
	unsigned long lpar_rc;
	unsigned long flags;
	unsigned long slot;
132
	unsigned long hpte_v, hpte_r;
L
Linus Torvalds 已提交
133

134
	if (!(vflags & HPTE_V_BOLTED))
135 136 137
		pr_devel("hpte_insert(group=%lx, vpn=%016lx, "
			 "pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n",
			 hpte_group, vpn,  pa, rflags, vflags, psize);
138

139 140
	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
141 142

	if (!(vflags & HPTE_V_BOLTED))
143
		pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
144

L
Linus Torvalds 已提交
145 146 147 148 149 150 151 152
	/* Now fill in the actual HPTE */
	/* Set CEC cookie to 0         */
	/* Zero page = 0               */
	/* I-cache Invalidate = 0      */
	/* I-cache synchronize = 0     */
	/* Exact = 0                   */
	flags = 0;

153
	/* Make pHyp happy */
154
	if ((rflags & _PAGE_NO_CACHE) && !(rflags & _PAGE_WRITETHRU))
155 156
		hpte_r &= ~HPTE_R_M;

157 158
	if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N))
		flags |= H_COALESCE_CAND;
L
Linus Torvalds 已提交
159

160
	lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot);
161
	if (unlikely(lpar_rc == H_PTEG_FULL)) {
162
		if (!(vflags & HPTE_V_BOLTED))
163
			pr_devel(" full\n");
L
Linus Torvalds 已提交
164
		return -1;
165
	}
L
Linus Torvalds 已提交
166 167 168 169 170 171

	/*
	 * Since we try and ioremap PHBs we don't own, the pte insert
	 * will fail. However we must catch the failure in hash_page
	 * or we will loop forever, so return -2 in this case.
	 */
172
	if (unlikely(lpar_rc != H_SUCCESS)) {
173
		if (!(vflags & HPTE_V_BOLTED))
174
			pr_devel(" lpar err %ld\n", lpar_rc);
L
Linus Torvalds 已提交
175
		return -2;
176 177
	}
	if (!(vflags & HPTE_V_BOLTED))
178
		pr_devel(" -> slot: %lu\n", slot & 7);
L
Linus Torvalds 已提交
179 180 181 182

	/* Because of iSeries, we have to pass down the secondary
	 * bucket bit here as well
	 */
183
	return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3);
L
Linus Torvalds 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
}

static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock);

static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
{
	unsigned long slot_offset;
	unsigned long lpar_rc;
	int i;
	unsigned long dummy1, dummy2;

	/* pick a random slot to start at */
	slot_offset = mftb() & 0x7;

	for (i = 0; i < HPTES_PER_GROUP; i++) {

		/* don't remove a bolted entry */
		lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset,
					   (0x1UL << 4), &dummy1, &dummy2);
203
		if (lpar_rc == H_SUCCESS)
L
Linus Torvalds 已提交
204
			return i;
205 206 207 208 209 210 211

		/*
		 * The test for adjunct partition is performed before the
		 * ANDCOND test.  H_RESOURCE may be returned, so we need to
		 * check for that as well.
		 */
		BUG_ON(lpar_rc != H_NOT_FOUND && lpar_rc != H_RESOURCE);
L
Linus Torvalds 已提交
212 213 214 215 216 217 218 219 220 221 222 223

		slot_offset++;
		slot_offset &= 0x7;
	}

	return -1;
}

static void pSeries_lpar_hptab_clear(void)
{
	unsigned long size_bytes = 1UL << ppc64_pft_size;
	unsigned long hpte_count = size_bytes >> 4;
224 225 226 227
	struct {
		unsigned long pteh;
		unsigned long ptel;
	} ptes[4];
228
	long lpar_rc;
229
	unsigned long i, j;
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245

	/* Read in batches of 4,
	 * invalidate only valid entries not in the VRMA
	 * hpte_count will be a multiple of 4
         */
	for (i = 0; i < hpte_count; i += 4) {
		lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes);
		if (lpar_rc != H_SUCCESS)
			continue;
		for (j = 0; j < 4; j++){
			if ((ptes[j].pteh & HPTE_V_VRMA_MASK) ==
				HPTE_V_VRMA_MASK)
				continue;
			if (ptes[j].pteh & HPTE_V_VALID)
				plpar_pte_remove_raw(0, i + j, 0,
					&(ptes[j].pteh), &(ptes[j].ptel));
246 247
		}
	}
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264

#ifdef __LITTLE_ENDIAN__
	/* Reset exceptions to big endian */
	if (firmware_has_feature(FW_FEATURE_SET_MODE)) {
		long rc;

		rc = pseries_big_endian_exceptions();
		/*
		 * At this point it is unlikely panic() will get anything
		 * out to the user, but at least this will stop us from
		 * continuing on further and creating an even more
		 * difficult to debug situation.
		 */
		if (rc)
			panic("Could not enable big endian exceptions");
	}
#endif
L
Linus Torvalds 已提交
265 266 267 268 269 270 271 272
}

/*
 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 * the low 3 bits of flags happen to line up.  So no transform is needed.
 * We can probably optimize here and assume the high bits of newpp are
 * already zero.  For now I am paranoid.
 */
273 274
static long pSeries_lpar_hpte_updatepp(unsigned long slot,
				       unsigned long newpp,
275
				       unsigned long vpn,
276 277
				       int psize, int apsize,
				       int ssize, int local)
L
Linus Torvalds 已提交
278 279 280
{
	unsigned long lpar_rc;
	unsigned long flags = (newpp & 7) | H_AVPN;
281
	unsigned long want_v;
L
Linus Torvalds 已提交
282

283
	want_v = hpte_encode_avpn(vpn, psize, ssize);
L
Linus Torvalds 已提交
284

285
	pr_devel("    update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...",
286
		 want_v, slot, flags, psize);
L
Linus Torvalds 已提交
287

P
Paul Mackerras 已提交
288
	lpar_rc = plpar_pte_protect(flags, slot, want_v);
289

290
	if (lpar_rc == H_NOT_FOUND) {
291
		pr_devel("not found !\n");
L
Linus Torvalds 已提交
292
		return -1;
293 294
	}

295
	pr_devel("ok\n");
L
Linus Torvalds 已提交
296

297
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315

	return 0;
}

static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot)
{
	unsigned long dword0;
	unsigned long lpar_rc;
	unsigned long dummy_word1;
	unsigned long flags;

	/* Read 1 pte at a time                        */
	/* Do not need RPN to logical page translation */
	/* No cross CEC PFT access                     */
	flags = 0;

	lpar_rc = plpar_pte_read(flags, slot, &dword0, &dummy_word1);

316
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
317 318 319 320

	return dword0;
}

321
static long pSeries_lpar_hpte_find(unsigned long vpn, int psize, int ssize)
L
Linus Torvalds 已提交
322 323
{
	unsigned long hash;
P
Paul Mackerras 已提交
324
	unsigned long i;
L
Linus Torvalds 已提交
325
	long slot;
326
	unsigned long want_v, hpte_v;
L
Linus Torvalds 已提交
327

328 329
	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
	want_v = hpte_encode_avpn(vpn, psize, ssize);
P
Paul Mackerras 已提交
330 331 332 333 334 335 336 337 338 339

	/* Bolted entries are always in the primary group */
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	for (i = 0; i < HPTES_PER_GROUP; i++) {
		hpte_v = pSeries_lpar_hpte_getword0(slot);

		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
			/* HPTE matches */
			return slot;
		++slot;
L
Linus Torvalds 已提交
340 341 342 343 344 345
	}

	return -1;
} 

static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp,
346
					     unsigned long ea,
P
Paul Mackerras 已提交
347
					     int psize, int ssize)
L
Linus Torvalds 已提交
348
{
349 350
	unsigned long vpn;
	unsigned long lpar_rc, slot, vsid, flags;
L
Linus Torvalds 已提交
351

P
Paul Mackerras 已提交
352
	vsid = get_kernel_vsid(ea, ssize);
353
	vpn = hpt_vpn(ea, vsid, ssize);
L
Linus Torvalds 已提交
354

355
	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
L
Linus Torvalds 已提交
356 357 358 359 360
	BUG_ON(slot == -1);

	flags = newpp & 7;
	lpar_rc = plpar_pte_protect(flags, slot, 0);

361
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
362 363
}

364
static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
365 366
					 int psize, int apsize,
					 int ssize, int local)
L
Linus Torvalds 已提交
367
{
368
	unsigned long want_v;
L
Linus Torvalds 已提交
369 370 371
	unsigned long lpar_rc;
	unsigned long dummy1, dummy2;

372 373
	pr_devel("    inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
		 slot, vpn, psize, local);
L
Linus Torvalds 已提交
374

375
	want_v = hpte_encode_avpn(vpn, psize, ssize);
P
Paul Mackerras 已提交
376
	lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2);
377
	if (lpar_rc == H_NOT_FOUND)
L
Linus Torvalds 已提交
378 379
		return;

380
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
381 382
}

383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
/*
 * Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
 * to make sure that we avoid bouncing the hypervisor tlbie lock.
 */
#define PPC64_HUGE_HPTE_BATCH 12

static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
					     unsigned long *vpn, int count,
					     int psize, int ssize)
{
	unsigned long param[8];
	int i = 0, pix = 0, rc;
	unsigned long flags = 0;
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

	if (lock_tlbie)
		spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

	for (i = 0; i < count; i++) {

		if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
			pSeries_lpar_hpte_invalidate(slot[i], vpn[i], psize, 0,
						     ssize, 0);
		} else {
			param[pix] = HBR_REQUEST | HBR_AVPN | slot[i];
			param[pix+1] = hpte_encode_avpn(vpn[i], psize, ssize);
			pix += 2;
			if (pix == 8) {
				rc = plpar_hcall9(H_BULK_REMOVE, param,
						  param[0], param[1], param[2],
						  param[3], param[4], param[5],
						  param[6], param[7]);
				BUG_ON(rc != H_SUCCESS);
				pix = 0;
			}
		}
	}
	if (pix) {
		param[pix] = HBR_END;
		rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
				  param[2], param[3], param[4], param[5],
				  param[6], param[7]);
		BUG_ON(rc != H_SUCCESS);
	}

	if (lock_tlbie)
		spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
}

432 433 434 435
static void pSeries_lpar_hugepage_invalidate(unsigned long vsid,
					     unsigned long addr,
					     unsigned char *hpte_slot_array,
					     int psize, int ssize)
436
{
437
	int i, index = 0;
438 439 440 441
	unsigned long s_addr = addr;
	unsigned int max_hpte_count, valid;
	unsigned long vpn_array[PPC64_HUGE_HPTE_BATCH];
	unsigned long slot_array[PPC64_HUGE_HPTE_BATCH];
442
	unsigned long shift, hidx, vpn = 0, hash, slot;
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481

	shift = mmu_psize_defs[psize].shift;
	max_hpte_count = 1U << (PMD_SHIFT - shift);

	for (i = 0; i < max_hpte_count; i++) {
		valid = hpte_valid(hpte_slot_array, i);
		if (!valid)
			continue;
		hidx =  hpte_hash_index(hpte_slot_array, i);

		/* get the vpn */
		addr = s_addr + (i * (1ul << shift));
		vpn = hpt_vpn(addr, vsid, ssize);
		hash = hpt_hash(vpn, shift, ssize);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;

		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;

		slot_array[index] = slot;
		vpn_array[index] = vpn;
		if (index == PPC64_HUGE_HPTE_BATCH - 1) {
			/*
			 * Now do a bluk invalidate
			 */
			__pSeries_lpar_hugepage_invalidate(slot_array,
							   vpn_array,
							   PPC64_HUGE_HPTE_BATCH,
							   psize, ssize);
			index = 0;
		} else
			index++;
	}
	if (index)
		__pSeries_lpar_hugepage_invalidate(slot_array, vpn_array,
						   index, psize, ssize);
}

482 483 484
static void pSeries_lpar_hpte_removebolted(unsigned long ea,
					   int psize, int ssize)
{
485 486
	unsigned long vpn;
	unsigned long slot, vsid;
487 488

	vsid = get_kernel_vsid(ea, ssize);
489
	vpn = hpt_vpn(ea, vsid, ssize);
490

491
	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
492
	BUG_ON(slot == -1);
493 494 495 496
	/*
	 * lpar doesn't use the passed actual page size
	 */
	pSeries_lpar_hpte_invalidate(slot, vpn, psize, 0, ssize, 0);
497 498
}

L
Linus Torvalds 已提交
499 500 501 502
/*
 * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
 * lock.
 */
503
static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
L
Linus Torvalds 已提交
504
{
505
	unsigned long vpn;
506
	unsigned long i, pix, rc;
507
	unsigned long flags = 0;
508
	struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
509
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
510 511 512
	unsigned long param[9];
	unsigned long hash, index, shift, hidx, slot;
	real_pte_t pte;
P
Paul Mackerras 已提交
513
	int psize, ssize;
L
Linus Torvalds 已提交
514 515 516 517

	if (lock_tlbie)
		spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

518
	psize = batch->psize;
P
Paul Mackerras 已提交
519
	ssize = batch->ssize;
520 521
	pix = 0;
	for (i = 0; i < number; i++) {
522
		vpn = batch->vpn[i];
523
		pte = batch->pte[i];
524 525
		pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
			hash = hpt_hash(vpn, shift, ssize);
526 527 528 529 530
			hidx = __rpte_to_hidx(pte, index);
			if (hidx & _PTEIDX_SECONDARY)
				hash = ~hash;
			slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
			slot += hidx & _PTEIDX_GROUP_IX;
531
			if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
532 533 534
				/*
				 * lpar doesn't use the passed actual page size
				 */
535
				pSeries_lpar_hpte_invalidate(slot, vpn, psize,
536
							     0, ssize, local);
537 538
			} else {
				param[pix] = HBR_REQUEST | HBR_AVPN | slot;
539
				param[pix+1] = hpte_encode_avpn(vpn, psize,
P
Paul Mackerras 已提交
540
								ssize);
541 542 543
				pix += 2;
				if (pix == 8) {
					rc = plpar_hcall9(H_BULK_REMOVE, param,
544 545 546
						param[0], param[1], param[2],
						param[3], param[4], param[5],
						param[6], param[7]);
547 548 549
					BUG_ON(rc != H_SUCCESS);
					pix = 0;
				}
550 551 552 553 554 555 556 557 558 559
			}
		} pte_iterate_hashed_end();
	}
	if (pix) {
		param[pix] = HBR_END;
		rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
				  param[2], param[3], param[4], param[5],
				  param[6], param[7]);
		BUG_ON(rc != H_SUCCESS);
	}
L
Linus Torvalds 已提交
560 561 562 563 564

	if (lock_tlbie)
		spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
}

565 566 567 568 569 570 571 572 573 574 575 576
static int __init disable_bulk_remove(char *str)
{
	if (strcmp(str, "off") == 0 &&
	    firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
			printk(KERN_INFO "Disabling BULK_REMOVE firmware feature");
			powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE;
	}
	return 1;
}

__setup("bulk_remove=", disable_bulk_remove);

577
void __init hpte_init_lpar(void)
L
Linus Torvalds 已提交
578 579 580 581 582 583
{
	ppc_md.hpte_invalidate	= pSeries_lpar_hpte_invalidate;
	ppc_md.hpte_updatepp	= pSeries_lpar_hpte_updatepp;
	ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
	ppc_md.hpte_insert	= pSeries_lpar_hpte_insert;
	ppc_md.hpte_remove	= pSeries_lpar_hpte_remove;
584
	ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted;
L
Linus Torvalds 已提交
585 586
	ppc_md.flush_hash_range	= pSeries_lpar_flush_hash_range;
	ppc_md.hpte_clear_all   = pSeries_lpar_hptab_clear;
587
	ppc_md.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
L
Linus Torvalds 已提交
588
}
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640

#ifdef CONFIG_PPC_SMLPAR
#define CMO_FREE_HINT_DEFAULT 1
static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT;

static int __init cmo_free_hint(char *str)
{
	char *parm;
	parm = strstrip(str);

	if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) {
		printk(KERN_INFO "cmo_free_hint: CMO free page hinting is not active.\n");
		cmo_free_hint_flag = 0;
		return 1;
	}

	cmo_free_hint_flag = 1;
	printk(KERN_INFO "cmo_free_hint: CMO free page hinting is active.\n");

	if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0)
		return 1;

	return 0;
}

__setup("cmo_free_hint=", cmo_free_hint);

static void pSeries_set_page_state(struct page *page, int order,
				   unsigned long state)
{
	int i, j;
	unsigned long cmo_page_sz, addr;

	cmo_page_sz = cmo_get_page_size();
	addr = __pa((unsigned long)page_address(page));

	for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) {
		for (j = 0; j < PAGE_SIZE; j += cmo_page_sz)
			plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0);
	}
}

void arch_free_page(struct page *page, int order)
{
	if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO))
		return;

	pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED);
}
EXPORT_SYMBOL(arch_free_page);

#endif
641 642

#ifdef CONFIG_TRACEPOINTS
643
#ifdef HAVE_JUMP_LABEL
644 645 646 647 648 649 650 651 652 653 654 655
struct static_key hcall_tracepoint_key = STATIC_KEY_INIT;

void hcall_tracepoint_regfunc(void)
{
	static_key_slow_inc(&hcall_tracepoint_key);
}

void hcall_tracepoint_unregfunc(void)
{
	static_key_slow_dec(&hcall_tracepoint_key);
}
#else
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
/*
 * We optimise our hcall path by placing hcall_tracepoint_refcount
 * directly in the TOC so we can check if the hcall tracepoints are
 * enabled via a single load.
 */

/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
extern long hcall_tracepoint_refcount;

void hcall_tracepoint_regfunc(void)
{
	hcall_tracepoint_refcount++;
}

void hcall_tracepoint_unregfunc(void)
{
	hcall_tracepoint_refcount--;
}
674 675 676 677 678 679 680 681 682
#endif

/*
 * Since the tracing code might execute hcalls we need to guard against
 * recursion. One example of this are spinlocks calling H_YIELD on
 * shared processor partitions.
 */
static DEFINE_PER_CPU(unsigned int, hcall_trace_depth);

683

684
void __trace_hcall_entry(unsigned long opcode, unsigned long *args)
685
{
686 687 688
	unsigned long flags;
	unsigned int *depth;

689 690 691 692 693 694 695
	/*
	 * We cannot call tracepoints inside RCU idle regions which
	 * means we must not trace H_CEDE.
	 */
	if (opcode == H_CEDE)
		return;

696 697
	local_irq_save(flags);

698
	depth = this_cpu_ptr(&hcall_trace_depth);
699 700 701 702 703

	if (*depth)
		goto out;

	(*depth)++;
704
	preempt_disable();
705
	trace_hcall_entry(opcode, args);
706 707 708 709
	(*depth)--;

out:
	local_irq_restore(flags);
710 711
}

712 713
void __trace_hcall_exit(long opcode, unsigned long retval,
			unsigned long *retbuf)
714
{
715 716 717
	unsigned long flags;
	unsigned int *depth;

718 719 720
	if (opcode == H_CEDE)
		return;

721 722
	local_irq_save(flags);

723
	depth = this_cpu_ptr(&hcall_trace_depth);
724 725 726 727 728

	if (*depth)
		goto out;

	(*depth)++;
729
	trace_hcall_exit(opcode, retval, retbuf);
730
	preempt_enable();
731 732 733 734
	(*depth)--;

out:
	local_irq_restore(flags);
735 736
}
#endif
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756

/**
 * h_get_mpp
 * H_GET_MPP hcall returns info in 7 parms
 */
int h_get_mpp(struct hvcall_mpp_data *mpp_data)
{
	int rc;
	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];

	rc = plpar_hcall9(H_GET_MPP, retbuf);

	mpp_data->entitled_mem = retbuf[0];
	mpp_data->mapped_mem = retbuf[1];

	mpp_data->group_num = (retbuf[2] >> 2 * 8) & 0xffff;
	mpp_data->pool_num = retbuf[2] & 0xffff;

	mpp_data->mem_weight = (retbuf[3] >> 7 * 8) & 0xff;
	mpp_data->unallocated_mem_weight = (retbuf[3] >> 6 * 8) & 0xff;
757
	mpp_data->unallocated_entitlement = retbuf[3] & 0xffffffffffffUL;
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780

	mpp_data->pool_size = retbuf[4];
	mpp_data->loan_request = retbuf[5];
	mpp_data->backing_mem = retbuf[6];

	return rc;
}
EXPORT_SYMBOL(h_get_mpp);

int h_get_mpp_x(struct hvcall_mpp_x_data *mpp_x_data)
{
	int rc;
	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 };

	rc = plpar_hcall9(H_GET_MPP_X, retbuf);

	mpp_x_data->coalesced_bytes = retbuf[0];
	mpp_x_data->pool_coalesced_bytes = retbuf[1];
	mpp_x_data->pool_purr_cycles = retbuf[2];
	mpp_x_data->pool_spurr_cycles = retbuf[3];

	return rc;
}