lpar.c 19.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * pSeries_lpar.c
 * Copyright (C) 2001 Todd Inglett, IBM Corporation
 *
 * pSeries LPAR support.
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

22 23
/* Enables debugging of low-level hash table routines - careful! */
#undef DEBUG
L
Linus Torvalds 已提交
24 25 26

#include <linux/kernel.h>
#include <linux/dma-mapping.h>
27
#include <linux/console.h>
28
#include <linux/export.h>
L
Linus Torvalds 已提交
29 30 31 32 33 34 35 36 37 38 39
#include <asm/processor.h>
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/machdep.h>
#include <asm/mmu_context.h>
#include <asm/iommu.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <asm/prom.h>
#include <asm/cputable.h>
D
David Gibson 已提交
40
#include <asm/udbg.h>
P
Paul Mackerras 已提交
41
#include <asm/smp.h>
42
#include <asm/trace.h>
43
#include <asm/firmware.h>
44
#include <asm/plpar_wrappers.h>
45

46
#include "pseries.h"
L
Linus Torvalds 已提交
47

48 49 50 51 52 53 54
/* Flag bits for H_BULK_REMOVE */
#define HBR_REQUEST	0x4000000000000000UL
#define HBR_RESPONSE	0x8000000000000000UL
#define HBR_END		0xc000000000000000UL
#define HBR_AVPN	0x0200000000000000UL
#define HBR_ANDCOND	0x0100000000000000UL

L
Linus Torvalds 已提交
55

56
/* in hvCall.S */
L
Linus Torvalds 已提交
57
EXPORT_SYMBOL(plpar_hcall);
58
EXPORT_SYMBOL(plpar_hcall9);
L
Linus Torvalds 已提交
59
EXPORT_SYMBOL(plpar_hcall_norets);
60

L
Linus Torvalds 已提交
61 62 63 64 65
extern void pSeries_find_serial_port(void);

void vpa_init(int cpu)
{
	int hwcpu = get_hard_smp_processor_id(cpu);
66
	unsigned long addr;
L
Linus Torvalds 已提交
67
	long ret;
68 69
	struct paca_struct *pp;
	struct dtl_entry *dtl;
70 71

	if (cpu_has_feature(CPU_FTR_ALTIVEC))
72
		lppaca_of(cpu).vmxregs_in_use = 1;
73

74 75 76
	if (cpu_has_feature(CPU_FTR_ARCH_207S))
		lppaca_of(cpu).ebb_regs_in_use = 1;

77
	addr = __pa(&lppaca_of(cpu));
78
	ret = register_vpa(hwcpu, addr);
L
Linus Torvalds 已提交
79

80
	if (ret) {
81 82
		pr_err("WARNING: VPA registration for cpu %d (hw %d) of area "
		       "%lx failed with %ld\n", cpu, hwcpu, addr, ret);
83 84 85 86 87 88 89 90 91 92
		return;
	}
	/*
	 * PAPR says this feature is SLB-Buffer but firmware never
	 * reports that.  All SPLPAR support SLB shadow buffer.
	 */
	addr = __pa(&slb_shadow[cpu]);
	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
		ret = register_slb_shadow(hwcpu, addr);
		if (ret)
93 94 95
			pr_err("WARNING: SLB shadow buffer registration for "
			       "cpu %d (hw %d) of area %lx failed with %ld\n",
			       cpu, hwcpu, addr, ret);
96
	}
97 98 99 100 101 102 103 104 105 106 107 108

	/*
	 * Register dispatch trace log, if one has been allocated.
	 */
	pp = &paca[cpu];
	dtl = pp->dispatch_log;
	if (dtl) {
		pp->dtl_ridx = 0;
		pp->dtl_curr = dtl;
		lppaca_of(cpu).dtl_idx = 0;

		/* hypervisor reads buffer length from this field */
109
		dtl->enqueue_to_dispatch_time = cpu_to_be32(DISPATCH_LOG_BYTES);
110 111
		ret = register_dtl(hwcpu, __pa(dtl));
		if (ret)
112 113 114
			pr_err("WARNING: DTL registration of cpu %d (hw %d) "
			       "failed with %ld\n", smp_processor_id(),
			       hwcpu, ret);
115 116
		lppaca_of(cpu).dtl_enable_mask = 2;
	}
L
Linus Torvalds 已提交
117 118
}

119
static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
120 121
				     unsigned long vpn, unsigned long pa,
				     unsigned long rflags, unsigned long vflags,
122
				     int psize, int apsize, int ssize)
L
Linus Torvalds 已提交
123 124 125 126
{
	unsigned long lpar_rc;
	unsigned long flags;
	unsigned long slot;
127
	unsigned long hpte_v, hpte_r;
L
Linus Torvalds 已提交
128

129
	if (!(vflags & HPTE_V_BOLTED))
130 131 132
		pr_devel("hpte_insert(group=%lx, vpn=%016lx, "
			 "pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n",
			 hpte_group, vpn,  pa, rflags, vflags, psize);
133

134 135
	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
136 137

	if (!(vflags & HPTE_V_BOLTED))
138
		pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
139

L
Linus Torvalds 已提交
140 141 142 143 144 145 146 147
	/* Now fill in the actual HPTE */
	/* Set CEC cookie to 0         */
	/* Zero page = 0               */
	/* I-cache Invalidate = 0      */
	/* I-cache synchronize = 0     */
	/* Exact = 0                   */
	flags = 0;

148
	/* Make pHyp happy */
149
	if ((rflags & _PAGE_NO_CACHE) && !(rflags & _PAGE_WRITETHRU))
150
		hpte_r &= ~_PAGE_COHERENT;
151 152
	if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N))
		flags |= H_COALESCE_CAND;
L
Linus Torvalds 已提交
153

154
	lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot);
155
	if (unlikely(lpar_rc == H_PTEG_FULL)) {
156
		if (!(vflags & HPTE_V_BOLTED))
157
			pr_devel(" full\n");
L
Linus Torvalds 已提交
158
		return -1;
159
	}
L
Linus Torvalds 已提交
160 161 162 163 164 165

	/*
	 * Since we try and ioremap PHBs we don't own, the pte insert
	 * will fail. However we must catch the failure in hash_page
	 * or we will loop forever, so return -2 in this case.
	 */
166
	if (unlikely(lpar_rc != H_SUCCESS)) {
167
		if (!(vflags & HPTE_V_BOLTED))
168
			pr_devel(" lpar err %ld\n", lpar_rc);
L
Linus Torvalds 已提交
169
		return -2;
170 171
	}
	if (!(vflags & HPTE_V_BOLTED))
172
		pr_devel(" -> slot: %lu\n", slot & 7);
L
Linus Torvalds 已提交
173 174 175 176

	/* Because of iSeries, we have to pass down the secondary
	 * bucket bit here as well
	 */
177
	return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3);
L
Linus Torvalds 已提交
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
}

static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock);

static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
{
	unsigned long slot_offset;
	unsigned long lpar_rc;
	int i;
	unsigned long dummy1, dummy2;

	/* pick a random slot to start at */
	slot_offset = mftb() & 0x7;

	for (i = 0; i < HPTES_PER_GROUP; i++) {

		/* don't remove a bolted entry */
		lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset,
					   (0x1UL << 4), &dummy1, &dummy2);
197
		if (lpar_rc == H_SUCCESS)
L
Linus Torvalds 已提交
198
			return i;
199 200 201 202 203 204 205

		/*
		 * The test for adjunct partition is performed before the
		 * ANDCOND test.  H_RESOURCE may be returned, so we need to
		 * check for that as well.
		 */
		BUG_ON(lpar_rc != H_NOT_FOUND && lpar_rc != H_RESOURCE);
L
Linus Torvalds 已提交
206 207 208 209 210 211 212 213 214 215 216 217

		slot_offset++;
		slot_offset &= 0x7;
	}

	return -1;
}

static void pSeries_lpar_hptab_clear(void)
{
	unsigned long size_bytes = 1UL << ppc64_pft_size;
	unsigned long hpte_count = size_bytes >> 4;
218 219 220 221
	struct {
		unsigned long pteh;
		unsigned long ptel;
	} ptes[4];
222
	long lpar_rc;
223
	unsigned long i, j;
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

	/* Read in batches of 4,
	 * invalidate only valid entries not in the VRMA
	 * hpte_count will be a multiple of 4
         */
	for (i = 0; i < hpte_count; i += 4) {
		lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes);
		if (lpar_rc != H_SUCCESS)
			continue;
		for (j = 0; j < 4; j++){
			if ((ptes[j].pteh & HPTE_V_VRMA_MASK) ==
				HPTE_V_VRMA_MASK)
				continue;
			if (ptes[j].pteh & HPTE_V_VALID)
				plpar_pte_remove_raw(0, i + j, 0,
					&(ptes[j].pteh), &(ptes[j].ptel));
240 241
		}
	}
L
Linus Torvalds 已提交
242 243 244 245 246 247 248 249
}

/*
 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 * the low 3 bits of flags happen to line up.  So no transform is needed.
 * We can probably optimize here and assume the high bits of newpp are
 * already zero.  For now I am paranoid.
 */
250 251
static long pSeries_lpar_hpte_updatepp(unsigned long slot,
				       unsigned long newpp,
252
				       unsigned long vpn,
253 254
				       int psize, int apsize,
				       int ssize, int local)
L
Linus Torvalds 已提交
255 256 257
{
	unsigned long lpar_rc;
	unsigned long flags = (newpp & 7) | H_AVPN;
258
	unsigned long want_v;
L
Linus Torvalds 已提交
259

260
	want_v = hpte_encode_avpn(vpn, psize, ssize);
L
Linus Torvalds 已提交
261

262
	pr_devel("    update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...",
263
		 want_v, slot, flags, psize);
L
Linus Torvalds 已提交
264

P
Paul Mackerras 已提交
265
	lpar_rc = plpar_pte_protect(flags, slot, want_v);
266

267
	if (lpar_rc == H_NOT_FOUND) {
268
		pr_devel("not found !\n");
L
Linus Torvalds 已提交
269
		return -1;
270 271
	}

272
	pr_devel("ok\n");
L
Linus Torvalds 已提交
273

274
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292

	return 0;
}

static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot)
{
	unsigned long dword0;
	unsigned long lpar_rc;
	unsigned long dummy_word1;
	unsigned long flags;

	/* Read 1 pte at a time                        */
	/* Do not need RPN to logical page translation */
	/* No cross CEC PFT access                     */
	flags = 0;

	lpar_rc = plpar_pte_read(flags, slot, &dword0, &dummy_word1);

293
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
294 295 296 297

	return dword0;
}

298
static long pSeries_lpar_hpte_find(unsigned long vpn, int psize, int ssize)
L
Linus Torvalds 已提交
299 300
{
	unsigned long hash;
P
Paul Mackerras 已提交
301
	unsigned long i;
L
Linus Torvalds 已提交
302
	long slot;
303
	unsigned long want_v, hpte_v;
L
Linus Torvalds 已提交
304

305 306
	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
	want_v = hpte_encode_avpn(vpn, psize, ssize);
P
Paul Mackerras 已提交
307 308 309 310 311 312 313 314 315 316

	/* Bolted entries are always in the primary group */
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	for (i = 0; i < HPTES_PER_GROUP; i++) {
		hpte_v = pSeries_lpar_hpte_getword0(slot);

		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
			/* HPTE matches */
			return slot;
		++slot;
L
Linus Torvalds 已提交
317 318 319 320 321 322
	}

	return -1;
} 

static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp,
323
					     unsigned long ea,
P
Paul Mackerras 已提交
324
					     int psize, int ssize)
L
Linus Torvalds 已提交
325
{
326 327
	unsigned long vpn;
	unsigned long lpar_rc, slot, vsid, flags;
L
Linus Torvalds 已提交
328

P
Paul Mackerras 已提交
329
	vsid = get_kernel_vsid(ea, ssize);
330
	vpn = hpt_vpn(ea, vsid, ssize);
L
Linus Torvalds 已提交
331

332
	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
L
Linus Torvalds 已提交
333 334 335 336 337
	BUG_ON(slot == -1);

	flags = newpp & 7;
	lpar_rc = plpar_pte_protect(flags, slot, 0);

338
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
339 340
}

341
static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
342 343
					 int psize, int apsize,
					 int ssize, int local)
L
Linus Torvalds 已提交
344
{
345
	unsigned long want_v;
L
Linus Torvalds 已提交
346 347 348
	unsigned long lpar_rc;
	unsigned long dummy1, dummy2;

349 350
	pr_devel("    inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
		 slot, vpn, psize, local);
L
Linus Torvalds 已提交
351

352
	want_v = hpte_encode_avpn(vpn, psize, ssize);
P
Paul Mackerras 已提交
353
	lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2);
354
	if (lpar_rc == H_NOT_FOUND)
L
Linus Torvalds 已提交
355 356
		return;

357
	BUG_ON(lpar_rc != H_SUCCESS);
L
Linus Torvalds 已提交
358 359
}

360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
/*
 * Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
 * to make sure that we avoid bouncing the hypervisor tlbie lock.
 */
#define PPC64_HUGE_HPTE_BATCH 12

static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
					     unsigned long *vpn, int count,
					     int psize, int ssize)
{
	unsigned long param[8];
	int i = 0, pix = 0, rc;
	unsigned long flags = 0;
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

	if (lock_tlbie)
		spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

	for (i = 0; i < count; i++) {

		if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
			pSeries_lpar_hpte_invalidate(slot[i], vpn[i], psize, 0,
						     ssize, 0);
		} else {
			param[pix] = HBR_REQUEST | HBR_AVPN | slot[i];
			param[pix+1] = hpte_encode_avpn(vpn[i], psize, ssize);
			pix += 2;
			if (pix == 8) {
				rc = plpar_hcall9(H_BULK_REMOVE, param,
						  param[0], param[1], param[2],
						  param[3], param[4], param[5],
						  param[6], param[7]);
				BUG_ON(rc != H_SUCCESS);
				pix = 0;
			}
		}
	}
	if (pix) {
		param[pix] = HBR_END;
		rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
				  param[2], param[3], param[4], param[5],
				  param[6], param[7]);
		BUG_ON(rc != H_SUCCESS);
	}

	if (lock_tlbie)
		spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
}

static void pSeries_lpar_hugepage_invalidate(struct mm_struct *mm,
				       unsigned char *hpte_slot_array,
				       unsigned long addr, int psize)
{
	int ssize = 0, i, index = 0;
	unsigned long s_addr = addr;
	unsigned int max_hpte_count, valid;
	unsigned long vpn_array[PPC64_HUGE_HPTE_BATCH];
	unsigned long slot_array[PPC64_HUGE_HPTE_BATCH];
	unsigned long shift, hidx, vpn = 0, vsid, hash, slot;

	shift = mmu_psize_defs[psize].shift;
	max_hpte_count = 1U << (PMD_SHIFT - shift);

	for (i = 0; i < max_hpte_count; i++) {
		valid = hpte_valid(hpte_slot_array, i);
		if (!valid)
			continue;
		hidx =  hpte_hash_index(hpte_slot_array, i);

		/* get the vpn */
		addr = s_addr + (i * (1ul << shift));
		if (!is_kernel_addr(addr)) {
			ssize = user_segment_size(addr);
			vsid = get_vsid(mm->context.id, addr, ssize);
			WARN_ON(vsid == 0);
		} else {
			vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
			ssize = mmu_kernel_ssize;
		}

		vpn = hpt_vpn(addr, vsid, ssize);
		hash = hpt_hash(vpn, shift, ssize);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;

		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;

		slot_array[index] = slot;
		vpn_array[index] = vpn;
		if (index == PPC64_HUGE_HPTE_BATCH - 1) {
			/*
			 * Now do a bluk invalidate
			 */
			__pSeries_lpar_hugepage_invalidate(slot_array,
							   vpn_array,
							   PPC64_HUGE_HPTE_BATCH,
							   psize, ssize);
			index = 0;
		} else
			index++;
	}
	if (index)
		__pSeries_lpar_hugepage_invalidate(slot_array, vpn_array,
						   index, psize, ssize);
}

467 468 469
static void pSeries_lpar_hpte_removebolted(unsigned long ea,
					   int psize, int ssize)
{
470 471
	unsigned long vpn;
	unsigned long slot, vsid;
472 473

	vsid = get_kernel_vsid(ea, ssize);
474
	vpn = hpt_vpn(ea, vsid, ssize);
475

476
	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
477
	BUG_ON(slot == -1);
478 479 480 481
	/*
	 * lpar doesn't use the passed actual page size
	 */
	pSeries_lpar_hpte_invalidate(slot, vpn, psize, 0, ssize, 0);
482 483
}

L
Linus Torvalds 已提交
484 485 486 487
/*
 * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
 * lock.
 */
488
static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
L
Linus Torvalds 已提交
489
{
490
	unsigned long vpn;
491
	unsigned long i, pix, rc;
492
	unsigned long flags = 0;
L
Linus Torvalds 已提交
493
	struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
494
	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
495 496 497
	unsigned long param[9];
	unsigned long hash, index, shift, hidx, slot;
	real_pte_t pte;
P
Paul Mackerras 已提交
498
	int psize, ssize;
L
Linus Torvalds 已提交
499 500 501 502

	if (lock_tlbie)
		spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

503
	psize = batch->psize;
P
Paul Mackerras 已提交
504
	ssize = batch->ssize;
505 506
	pix = 0;
	for (i = 0; i < number; i++) {
507
		vpn = batch->vpn[i];
508
		pte = batch->pte[i];
509 510
		pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
			hash = hpt_hash(vpn, shift, ssize);
511 512 513 514 515
			hidx = __rpte_to_hidx(pte, index);
			if (hidx & _PTEIDX_SECONDARY)
				hash = ~hash;
			slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
			slot += hidx & _PTEIDX_GROUP_IX;
516
			if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
517 518 519
				/*
				 * lpar doesn't use the passed actual page size
				 */
520
				pSeries_lpar_hpte_invalidate(slot, vpn, psize,
521
							     0, ssize, local);
522 523
			} else {
				param[pix] = HBR_REQUEST | HBR_AVPN | slot;
524
				param[pix+1] = hpte_encode_avpn(vpn, psize,
P
Paul Mackerras 已提交
525
								ssize);
526 527 528
				pix += 2;
				if (pix == 8) {
					rc = plpar_hcall9(H_BULK_REMOVE, param,
529 530 531
						param[0], param[1], param[2],
						param[3], param[4], param[5],
						param[6], param[7]);
532 533 534
					BUG_ON(rc != H_SUCCESS);
					pix = 0;
				}
535 536 537 538 539 540 541 542 543 544
			}
		} pte_iterate_hashed_end();
	}
	if (pix) {
		param[pix] = HBR_END;
		rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
				  param[2], param[3], param[4], param[5],
				  param[6], param[7]);
		BUG_ON(rc != H_SUCCESS);
	}
L
Linus Torvalds 已提交
545 546 547 548 549

	if (lock_tlbie)
		spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
}

550 551 552 553 554 555 556 557 558 559 560 561
static int __init disable_bulk_remove(char *str)
{
	if (strcmp(str, "off") == 0 &&
	    firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
			printk(KERN_INFO "Disabling BULK_REMOVE firmware feature");
			powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE;
	}
	return 1;
}

__setup("bulk_remove=", disable_bulk_remove);

562
void __init hpte_init_lpar(void)
L
Linus Torvalds 已提交
563 564 565 566 567 568
{
	ppc_md.hpte_invalidate	= pSeries_lpar_hpte_invalidate;
	ppc_md.hpte_updatepp	= pSeries_lpar_hpte_updatepp;
	ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
	ppc_md.hpte_insert	= pSeries_lpar_hpte_insert;
	ppc_md.hpte_remove	= pSeries_lpar_hpte_remove;
569
	ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted;
L
Linus Torvalds 已提交
570 571
	ppc_md.flush_hash_range	= pSeries_lpar_flush_hash_range;
	ppc_md.hpte_clear_all   = pSeries_lpar_hptab_clear;
572
	ppc_md.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
L
Linus Torvalds 已提交
573
}
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625

#ifdef CONFIG_PPC_SMLPAR
#define CMO_FREE_HINT_DEFAULT 1
static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT;

static int __init cmo_free_hint(char *str)
{
	char *parm;
	parm = strstrip(str);

	if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) {
		printk(KERN_INFO "cmo_free_hint: CMO free page hinting is not active.\n");
		cmo_free_hint_flag = 0;
		return 1;
	}

	cmo_free_hint_flag = 1;
	printk(KERN_INFO "cmo_free_hint: CMO free page hinting is active.\n");

	if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0)
		return 1;

	return 0;
}

__setup("cmo_free_hint=", cmo_free_hint);

static void pSeries_set_page_state(struct page *page, int order,
				   unsigned long state)
{
	int i, j;
	unsigned long cmo_page_sz, addr;

	cmo_page_sz = cmo_get_page_size();
	addr = __pa((unsigned long)page_address(page));

	for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) {
		for (j = 0; j < PAGE_SIZE; j += cmo_page_sz)
			plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0);
	}
}

void arch_free_page(struct page *page, int order)
{
	if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO))
		return;

	pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED);
}
EXPORT_SYMBOL(arch_free_page);

#endif
626 627 628 629 630 631 632 633 634 635 636

#ifdef CONFIG_TRACEPOINTS
/*
 * We optimise our hcall path by placing hcall_tracepoint_refcount
 * directly in the TOC so we can check if the hcall tracepoints are
 * enabled via a single load.
 */

/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
extern long hcall_tracepoint_refcount;

637 638 639 640 641 642 643
/* 
 * Since the tracing code might execute hcalls we need to guard against
 * recursion. One example of this are spinlocks calling H_YIELD on
 * shared processor partitions.
 */
static DEFINE_PER_CPU(unsigned int, hcall_trace_depth);

644 645 646 647 648 649 650 651 652 653
void hcall_tracepoint_regfunc(void)
{
	hcall_tracepoint_refcount++;
}

void hcall_tracepoint_unregfunc(void)
{
	hcall_tracepoint_refcount--;
}

654
void __trace_hcall_entry(unsigned long opcode, unsigned long *args)
655
{
656 657 658
	unsigned long flags;
	unsigned int *depth;

659 660 661 662 663 664 665
	/*
	 * We cannot call tracepoints inside RCU idle regions which
	 * means we must not trace H_CEDE.
	 */
	if (opcode == H_CEDE)
		return;

666 667 668 669 670 671 672 673
	local_irq_save(flags);

	depth = &__get_cpu_var(hcall_trace_depth);

	if (*depth)
		goto out;

	(*depth)++;
674
	preempt_disable();
675
	trace_hcall_entry(opcode, args);
676 677 678 679
	(*depth)--;

out:
	local_irq_restore(flags);
680 681
}

682 683
void __trace_hcall_exit(long opcode, unsigned long retval,
			unsigned long *retbuf)
684
{
685 686 687
	unsigned long flags;
	unsigned int *depth;

688 689 690
	if (opcode == H_CEDE)
		return;

691 692 693 694 695 696 697 698
	local_irq_save(flags);

	depth = &__get_cpu_var(hcall_trace_depth);

	if (*depth)
		goto out;

	(*depth)++;
699
	trace_hcall_exit(opcode, retval, retbuf);
700
	preempt_enable();
701 702 703 704
	(*depth)--;

out:
	local_irq_restore(flags);
705 706
}
#endif
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726

/**
 * h_get_mpp
 * H_GET_MPP hcall returns info in 7 parms
 */
int h_get_mpp(struct hvcall_mpp_data *mpp_data)
{
	int rc;
	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];

	rc = plpar_hcall9(H_GET_MPP, retbuf);

	mpp_data->entitled_mem = retbuf[0];
	mpp_data->mapped_mem = retbuf[1];

	mpp_data->group_num = (retbuf[2] >> 2 * 8) & 0xffff;
	mpp_data->pool_num = retbuf[2] & 0xffff;

	mpp_data->mem_weight = (retbuf[3] >> 7 * 8) & 0xff;
	mpp_data->unallocated_mem_weight = (retbuf[3] >> 6 * 8) & 0xff;
727
	mpp_data->unallocated_entitlement = retbuf[3] & 0xffffffffffffUL;
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750

	mpp_data->pool_size = retbuf[4];
	mpp_data->loan_request = retbuf[5];
	mpp_data->backing_mem = retbuf[6];

	return rc;
}
EXPORT_SYMBOL(h_get_mpp);

int h_get_mpp_x(struct hvcall_mpp_x_data *mpp_x_data)
{
	int rc;
	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 };

	rc = plpar_hcall9(H_GET_MPP_X, retbuf);

	mpp_x_data->coalesced_bytes = retbuf[0];
	mpp_x_data->pool_coalesced_bytes = retbuf[1];
	mpp_x_data->pool_purr_cycles = retbuf[2];
	mpp_x_data->pool_spurr_cycles = retbuf[3];

	return rc;
}