book3s_hv.c 64.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*
 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
 *
 * Authors:
 *    Paul Mackerras <paulus@au1.ibm.com>
 *    Alexander Graf <agraf@suse.de>
 *    Kevin Wolf <mail@kevin-wolf.de>
 *
 * Description: KVM functions specific to running on Book 3S
 * processors in hypervisor mode (specifically POWER7 and later).
 *
 * This file is derived from arch/powerpc/kvm/book3s.c,
 * by Alexander Graf <agraf@suse.de>.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 */

#include <linux/kvm_host.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/preempt.h>
#include <linux/sched.h>
#include <linux/delay.h>
27
#include <linux/export.h>
28 29 30
#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/cpumask.h>
31 32
#include <linux/spinlock.h>
#include <linux/page-flags.h>
33
#include <linux/srcu.h>
34
#include <linux/miscdevice.h>
35 36 37

#include <asm/reg.h>
#include <asm/cputable.h>
38
#include <asm/cache.h>
39 40 41 42 43 44 45 46 47
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/uaccess.h>
#include <asm/io.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/mmu_context.h>
#include <asm/lppaca.h>
#include <asm/processor.h>
48
#include <asm/cputhreads.h>
49
#include <asm/page.h>
50
#include <asm/hvcall.h>
51
#include <asm/switch_to.h>
52
#include <asm/smp.h>
53 54 55
#include <linux/gfp.h>
#include <linux/vmalloc.h>
#include <linux/highmem.h>
56
#include <linux/hugetlb.h>
57
#include <linux/module.h>
58

59 60
#include "book3s.h"

61 62 63
#define CREATE_TRACE_POINTS
#include "trace_hv.h"

64 65 66 67
/* #define EXIT_DEBUG */
/* #define EXIT_DEBUG_SIMPLE */
/* #define EXIT_DEBUG_INT */

68 69 70
/* Used to indicate that a guest page fault needs to be handled */
#define RESUME_PAGE_FAULT	(RESUME_GUEST | RESUME_FLAG_ARCH1)

71 72 73
/* Used as a "null" value for timebase values */
#define TB_NIL	(~(u64)0)

74 75
static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);

76 77 78 79 80 81 82
#if defined(CONFIG_PPC_64K_PAGES)
#define MPP_BUFFER_ORDER	0
#elif defined(CONFIG_PPC_4K_PAGES)
#define MPP_BUFFER_ORDER	3
#endif


83
static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
84
static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
85

86
static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
{
	int me;
	int cpu = vcpu->cpu;
	wait_queue_head_t *wqp;

	wqp = kvm_arch_vcpu_wq(vcpu);
	if (waitqueue_active(wqp)) {
		wake_up_interruptible(wqp);
		++vcpu->stat.halt_wakeup;
	}

	me = get_cpu();

	/* CPU points to the first thread of the core */
	if (cpu != me && cpu >= 0 && cpu < nr_cpu_ids) {
102
#ifdef CONFIG_PPC_ICP_NATIVE
103 104 105
		int real_cpu = cpu + vcpu->arch.ptid;
		if (paca[real_cpu].kvm_hstate.xics_phys)
			xics_wake_cpu(real_cpu);
106 107 108
		else
#endif
		if (cpu_online(cpu))
109 110 111 112 113
			smp_send_reschedule(cpu);
	}
	put_cpu();
}

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
/*
 * We use the vcpu_load/put functions to measure stolen time.
 * Stolen time is counted as time when either the vcpu is able to
 * run as part of a virtual core, but the task running the vcore
 * is preempted or sleeping, or when the vcpu needs something done
 * in the kernel by the task running the vcpu, but that task is
 * preempted or sleeping.  Those two things have to be counted
 * separately, since one of the vcpu tasks will take on the job
 * of running the core, and the other vcpu tasks in the vcore will
 * sleep waiting for it to do that, but that sleep shouldn't count
 * as stolen time.
 *
 * Hence we accumulate stolen time when the vcpu can run as part of
 * a vcore using vc->stolen_tb, and the stolen time when the vcpu
 * needs its task to do other things in the kernel (for example,
 * service a page fault) in busy_stolen.  We don't accumulate
 * stolen time for a vcore when it is inactive, or for a vcpu
 * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
 * a misnomer; it means that the vcpu task is not executing in
 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
 * the kernel.  We don't have any way of dividing up that time
 * between time that the vcpu is genuinely stopped, time that
 * the task is actively working on behalf of the vcpu, and time
 * that the task is preempted, so we don't count any of it as
 * stolen.
 *
 * Updates to busy_stolen are protected by arch.tbacct_lock;
141 142 143 144
 * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
 * lock.  The stolen times are measured in units of timebase ticks.
 * (Note that the != TB_NIL checks below are purely defensive;
 * they should never fail.)
145 146
 */

147
static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
148
{
149
	struct kvmppc_vcore *vc = vcpu->arch.vcore;
150
	unsigned long flags;
151

152 153 154 155 156 157 158 159 160 161 162 163 164
	/*
	 * We can test vc->runner without taking the vcore lock,
	 * because only this task ever sets vc->runner to this
	 * vcpu, and once it is set to this vcpu, only this task
	 * ever sets it to NULL.
	 */
	if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE) {
		spin_lock_irqsave(&vc->stoltb_lock, flags);
		if (vc->preempt_tb != TB_NIL) {
			vc->stolen_tb += mftb() - vc->preempt_tb;
			vc->preempt_tb = TB_NIL;
		}
		spin_unlock_irqrestore(&vc->stoltb_lock, flags);
165
	}
166
	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
167 168 169 170 171
	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
	    vcpu->arch.busy_preempt != TB_NIL) {
		vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
		vcpu->arch.busy_preempt = TB_NIL;
	}
172
	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
173 174
}

175
static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
176
{
177
	struct kvmppc_vcore *vc = vcpu->arch.vcore;
178
	unsigned long flags;
179

180 181
	if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE) {
		spin_lock_irqsave(&vc->stoltb_lock, flags);
182
		vc->preempt_tb = mftb();
183 184 185
		spin_unlock_irqrestore(&vc->stoltb_lock, flags);
	}
	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
186 187
	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
		vcpu->arch.busy_preempt = mftb();
188
	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
189 190
}

191
static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
192 193
{
	vcpu->arch.shregs.msr = msr;
194
	kvmppc_end_cede(vcpu);
195 196
}

197
void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
198 199 200 201
{
	vcpu->arch.pvr = pvr;
}

202 203 204 205 206 207 208 209
int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
{
	unsigned long pcr = 0;
	struct kvmppc_vcore *vc = vcpu->arch.vcore;

	if (arch_compat) {
		switch (arch_compat) {
		case PVR_ARCH_205:
210 211 212 213 214
			/*
			 * If an arch bit is set in PCR, all the defined
			 * higher-order arch bits also have to be set.
			 */
			pcr = PCR_ARCH_206 | PCR_ARCH_205;
215 216 217
			break;
		case PVR_ARCH_206:
		case PVR_ARCH_206p:
218 219 220
			pcr = PCR_ARCH_206;
			break;
		case PVR_ARCH_207:
221 222 223 224
			break;
		default:
			return -EINVAL;
		}
225 226 227 228 229 230 231

		if (!cpu_has_feature(CPU_FTR_ARCH_207S)) {
			/* POWER7 can't emulate POWER8 */
			if (!(pcr & PCR_ARCH_206))
				return -EINVAL;
			pcr &= ~PCR_ARCH_206;
		}
232 233 234 235 236 237 238 239 240 241
	}

	spin_lock(&vc->lock);
	vc->arch_compat = arch_compat;
	vc->pcr = pcr;
	spin_unlock(&vc->lock);

	return 0;
}

242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
{
	int r;

	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
	       vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
	for (r = 0; r < 16; ++r)
		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
		       r, kvmppc_get_gpr(vcpu, r),
		       r+16, kvmppc_get_gpr(vcpu, r+16));
	pr_err("ctr = %.16lx  lr  = %.16lx\n",
	       vcpu->arch.ctr, vcpu->arch.lr);
	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
	       vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
	pr_err("cr = %.8x  xer = %.16lx  dsisr = %.8x\n",
	       vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
	pr_err("fault dar = %.16lx dsisr = %.8x\n",
	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
	pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
	for (r = 0; r < vcpu->arch.slb_max; ++r)
		pr_err("  ESID = %.16llx VSID = %.16llx\n",
		       vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
	pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
271
	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
272 273 274
	       vcpu->arch.last_inst);
}

275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
{
	int r;
	struct kvm_vcpu *v, *ret = NULL;

	mutex_lock(&kvm->lock);
	kvm_for_each_vcpu(r, v, kvm) {
		if (v->vcpu_id == id) {
			ret = v;
			break;
		}
	}
	mutex_unlock(&kvm->lock);
	return ret;
}

static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
{
293
	vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
294
	vpa->yield_count = cpu_to_be32(1);
295 296
}

297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
		   unsigned long addr, unsigned long len)
{
	/* check address is cacheline aligned */
	if (addr & (L1_CACHE_BYTES - 1))
		return -EINVAL;
	spin_lock(&vcpu->arch.vpa_update_lock);
	if (v->next_gpa != addr || v->len != len) {
		v->next_gpa = addr;
		v->len = addr ? len : 0;
		v->update_pending = 1;
	}
	spin_unlock(&vcpu->arch.vpa_update_lock);
	return 0;
}

313 314 315 316
/* Length for a per-processor buffer is passed in at offset 4 in the buffer */
struct reg_vpa {
	u32 dummy;
	union {
317 318
		__be16 hword;
		__be32 word;
319 320 321 322 323 324 325 326 327 328
	} length;
};

static int vpa_is_registered(struct kvmppc_vpa *vpap)
{
	if (vpap->update_pending)
		return vpap->next_gpa != 0;
	return vpap->pinned_addr != NULL;
}

329 330 331 332 333
static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
				       unsigned long flags,
				       unsigned long vcpuid, unsigned long vpa)
{
	struct kvm *kvm = vcpu->kvm;
334
	unsigned long len, nb;
335 336
	void *va;
	struct kvm_vcpu *tvcpu;
337 338 339
	int err;
	int subfunc;
	struct kvmppc_vpa *vpap;
340 341 342 343 344

	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
	if (!tvcpu)
		return H_PARAMETER;

345 346 347 348 349
	subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
	if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
	    subfunc == H_VPA_REG_SLB) {
		/* Registering new area - address must be cache-line aligned */
		if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
350
			return H_PARAMETER;
351 352

		/* convert logical addr to kernel addr and read length */
353 354
		va = kvmppc_pin_guest_page(kvm, vpa, &nb);
		if (va == NULL)
355
			return H_PARAMETER;
356
		if (subfunc == H_VPA_REG_VPA)
357
			len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
358
		else
359
			len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
360
		kvmppc_unpin_guest_page(kvm, va, vpa, false);
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376

		/* Check length */
		if (len > nb || len < sizeof(struct reg_vpa))
			return H_PARAMETER;
	} else {
		vpa = 0;
		len = 0;
	}

	err = H_PARAMETER;
	vpap = NULL;
	spin_lock(&tvcpu->arch.vpa_update_lock);

	switch (subfunc) {
	case H_VPA_REG_VPA:		/* register VPA */
		if (len < sizeof(struct lppaca))
377
			break;
378 379 380 381 382 383
		vpap = &tvcpu->arch.vpa;
		err = 0;
		break;

	case H_VPA_REG_DTL:		/* register DTL */
		if (len < sizeof(struct dtl_entry))
384
			break;
385 386 387 388 389
		len -= len % sizeof(struct dtl_entry);

		/* Check that they have previously registered a VPA */
		err = H_RESOURCE;
		if (!vpa_is_registered(&tvcpu->arch.vpa))
390
			break;
391 392 393 394 395 396 397 398 399

		vpap = &tvcpu->arch.dtl;
		err = 0;
		break;

	case H_VPA_REG_SLB:		/* register SLB shadow buffer */
		/* Check that they have previously registered a VPA */
		err = H_RESOURCE;
		if (!vpa_is_registered(&tvcpu->arch.vpa))
400
			break;
401 402 403 404 405 406 407 408 409 410

		vpap = &tvcpu->arch.slb_shadow;
		err = 0;
		break;

	case H_VPA_DEREG_VPA:		/* deregister VPA */
		/* Check they don't still have a DTL or SLB buf registered */
		err = H_RESOURCE;
		if (vpa_is_registered(&tvcpu->arch.dtl) ||
		    vpa_is_registered(&tvcpu->arch.slb_shadow))
411
			break;
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431

		vpap = &tvcpu->arch.vpa;
		err = 0;
		break;

	case H_VPA_DEREG_DTL:		/* deregister DTL */
		vpap = &tvcpu->arch.dtl;
		err = 0;
		break;

	case H_VPA_DEREG_SLB:		/* deregister SLB shadow buffer */
		vpap = &tvcpu->arch.slb_shadow;
		err = 0;
		break;
	}

	if (vpap) {
		vpap->next_gpa = vpa;
		vpap->len = len;
		vpap->update_pending = 1;
432
	}
433

434 435
	spin_unlock(&tvcpu->arch.vpa_update_lock);

436
	return err;
437 438
}

439
static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
440
{
441
	struct kvm *kvm = vcpu->kvm;
442 443
	void *va;
	unsigned long nb;
444
	unsigned long gpa;
445

446 447 448 449 450 451 452 453 454 455 456 457 458 459
	/*
	 * We need to pin the page pointed to by vpap->next_gpa,
	 * but we can't call kvmppc_pin_guest_page under the lock
	 * as it does get_user_pages() and down_read().  So we
	 * have to drop the lock, pin the page, then get the lock
	 * again and check that a new area didn't get registered
	 * in the meantime.
	 */
	for (;;) {
		gpa = vpap->next_gpa;
		spin_unlock(&vcpu->arch.vpa_update_lock);
		va = NULL;
		nb = 0;
		if (gpa)
460
			va = kvmppc_pin_guest_page(kvm, gpa, &nb);
461 462 463 464 465
		spin_lock(&vcpu->arch.vpa_update_lock);
		if (gpa == vpap->next_gpa)
			break;
		/* sigh... unpin that one and try again */
		if (va)
466
			kvmppc_unpin_guest_page(kvm, va, gpa, false);
467 468 469 470 471 472 473 474 475
	}

	vpap->update_pending = 0;
	if (va && nb < vpap->len) {
		/*
		 * If it's now too short, it must be that userspace
		 * has changed the mappings underlying guest memory,
		 * so unregister the region.
		 */
476
		kvmppc_unpin_guest_page(kvm, va, gpa, false);
477
		va = NULL;
478 479
	}
	if (vpap->pinned_addr)
480 481 482
		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
					vpap->dirty);
	vpap->gpa = gpa;
483
	vpap->pinned_addr = va;
484
	vpap->dirty = false;
485 486 487 488 489 490
	if (va)
		vpap->pinned_end = va + vpap->len;
}

static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
{
491 492 493 494 495
	if (!(vcpu->arch.vpa.update_pending ||
	      vcpu->arch.slb_shadow.update_pending ||
	      vcpu->arch.dtl.update_pending))
		return;

496 497
	spin_lock(&vcpu->arch.vpa_update_lock);
	if (vcpu->arch.vpa.update_pending) {
498
		kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
499 500
		if (vcpu->arch.vpa.pinned_addr)
			init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
501 502
	}
	if (vcpu->arch.dtl.update_pending) {
503
		kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
504 505 506 507
		vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
		vcpu->arch.dtl_index = 0;
	}
	if (vcpu->arch.slb_shadow.update_pending)
508
		kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
509 510 511
	spin_unlock(&vcpu->arch.vpa_update_lock);
}

512 513 514 515 516 517 518
/*
 * Return the accumulated stolen time for the vcore up until `now'.
 * The caller should hold the vcore lock.
 */
static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
{
	u64 p;
519
	unsigned long flags;
520

521 522
	spin_lock_irqsave(&vc->stoltb_lock, flags);
	p = vc->stolen_tb;
523
	if (vc->vcore_state != VCORE_INACTIVE &&
524 525 526
	    vc->preempt_tb != TB_NIL)
		p += now - vc->preempt_tb;
	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
527 528 529
	return p;
}

530 531 532 533 534
static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
				    struct kvmppc_vcore *vc)
{
	struct dtl_entry *dt;
	struct lppaca *vpa;
535 536 537
	unsigned long stolen;
	unsigned long core_stolen;
	u64 now;
538 539 540

	dt = vcpu->arch.dtl_ptr;
	vpa = vcpu->arch.vpa.pinned_addr;
541 542 543 544
	now = mftb();
	core_stolen = vcore_stolen_time(vc, now);
	stolen = core_stolen - vcpu->arch.stolen_logged;
	vcpu->arch.stolen_logged = core_stolen;
545
	spin_lock_irq(&vcpu->arch.tbacct_lock);
546 547
	stolen += vcpu->arch.busy_stolen;
	vcpu->arch.busy_stolen = 0;
548
	spin_unlock_irq(&vcpu->arch.tbacct_lock);
549 550 551 552
	if (!dt || !vpa)
		return;
	memset(dt, 0, sizeof(struct dtl_entry));
	dt->dispatch_reason = 7;
553 554 555 556 557
	dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
	dt->timebase = cpu_to_be64(now + vc->tb_offset);
	dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
	dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
	dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
558 559 560 561 562 563
	++dt;
	if (dt == vcpu->arch.dtl.pinned_end)
		dt = vcpu->arch.dtl.pinned_addr;
	vcpu->arch.dtl_ptr = dt;
	/* order writing *dt vs. writing vpa->dtl_idx */
	smp_wmb();
564
	vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
565
	vcpu->arch.dtl.dirty = true;
566 567
}

568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
		return true;
	if ((!vcpu->arch.vcore->arch_compat) &&
	    cpu_has_feature(CPU_FTR_ARCH_207S))
		return true;
	return false;
}

static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
			     unsigned long resource, unsigned long value1,
			     unsigned long value2)
{
	switch (resource) {
	case H_SET_MODE_RESOURCE_SET_CIABR:
		if (!kvmppc_power8_compatible(vcpu))
			return H_P2;
		if (value2)
			return H_P4;
		if (mflags)
			return H_UNSUPPORTED_FLAG_START;
		/* Guests can't breakpoint the hypervisor */
		if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
			return H_P3;
		vcpu->arch.ciabr  = value1;
		return H_SUCCESS;
	case H_SET_MODE_RESOURCE_SET_DAWR:
		if (!kvmppc_power8_compatible(vcpu))
			return H_P2;
		if (mflags)
			return H_UNSUPPORTED_FLAG_START;
		if (value2 & DABRX_HYP)
			return H_P4;
		vcpu->arch.dawr  = value1;
		vcpu->arch.dawrx = value2;
		return H_SUCCESS;
	default:
		return H_TOO_HARD;
	}
}

610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
{
	struct kvmppc_vcore *vcore = target->arch.vcore;

	/*
	 * We expect to have been called by the real mode handler
	 * (kvmppc_rm_h_confer()) which would have directly returned
	 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
	 * have useful work to do and should not confer) so we don't
	 * recheck that here.
	 */

	spin_lock(&vcore->lock);
	if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
	    vcore->vcore_state != VCORE_INACTIVE)
		target = vcore->runner;
	spin_unlock(&vcore->lock);

	return kvm_vcpu_yield_to(target);
}

static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
{
	int yield_count = 0;
	struct lppaca *lppaca;

	spin_lock(&vcpu->arch.vpa_update_lock);
	lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
	if (lppaca)
639
		yield_count = be32_to_cpu(lppaca->yield_count);
640 641 642 643
	spin_unlock(&vcpu->arch.vpa_update_lock);
	return yield_count;
}

644 645 646 647
int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
{
	unsigned long req = kvmppc_get_gpr(vcpu, 3);
	unsigned long target, ret = H_SUCCESS;
648
	int yield_count;
649
	struct kvm_vcpu *tvcpu;
650
	int idx, rc;
651

652 653 654 655
	if (req <= MAX_HCALL_OPCODE &&
	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
		return RESUME_HOST;

656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
	switch (req) {
	case H_CEDE:
		break;
	case H_PROD:
		target = kvmppc_get_gpr(vcpu, 4);
		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
		if (!tvcpu) {
			ret = H_PARAMETER;
			break;
		}
		tvcpu->arch.prodded = 1;
		smp_mb();
		if (vcpu->arch.ceded) {
			if (waitqueue_active(&vcpu->wq)) {
				wake_up_interruptible(&vcpu->wq);
				vcpu->stat.halt_wakeup++;
			}
		}
		break;
	case H_CONFER:
676 677 678 679 680 681 682 683
		target = kvmppc_get_gpr(vcpu, 4);
		if (target == -1)
			break;
		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
		if (!tvcpu) {
			ret = H_PARAMETER;
			break;
		}
684 685 686 687
		yield_count = kvmppc_get_gpr(vcpu, 5);
		if (kvmppc_get_yield_count(tvcpu) != yield_count)
			break;
		kvm_arch_vcpu_yield_to(tvcpu);
688 689 690 691 692 693
		break;
	case H_REGISTER_VPA:
		ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
					kvmppc_get_gpr(vcpu, 5),
					kvmppc_get_gpr(vcpu, 6));
		break;
694 695 696 697
	case H_RTAS:
		if (list_empty(&vcpu->kvm->arch.rtas_tokens))
			return RESUME_HOST;

698
		idx = srcu_read_lock(&vcpu->kvm->srcu);
699
		rc = kvmppc_rtas_hcall(vcpu);
700
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
701 702 703 704 705 706 707 708

		if (rc == -ENOENT)
			return RESUME_HOST;
		else if (rc == 0)
			break;

		/* Send the error out to userspace via KVM_RUN */
		return rc;
709 710 711 712 713 714 715 716 717 718
	case H_LOGICAL_CI_LOAD:
		ret = kvmppc_h_logical_ci_load(vcpu);
		if (ret == H_TOO_HARD)
			return RESUME_HOST;
		break;
	case H_LOGICAL_CI_STORE:
		ret = kvmppc_h_logical_ci_store(vcpu);
		if (ret == H_TOO_HARD)
			return RESUME_HOST;
		break;
719 720 721 722 723 724 725 726
	case H_SET_MODE:
		ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
					kvmppc_get_gpr(vcpu, 5),
					kvmppc_get_gpr(vcpu, 6),
					kvmppc_get_gpr(vcpu, 7));
		if (ret == H_TOO_HARD)
			return RESUME_HOST;
		break;
727 728 729 730
	case H_XIRR:
	case H_CPPR:
	case H_EOI:
	case H_IPI:
731 732
	case H_IPOLL:
	case H_XIRR_X:
733 734 735 736
		if (kvmppc_xics_enabled(vcpu)) {
			ret = kvmppc_xics_hcall(vcpu, req);
			break;
		} /* fallthrough */
737 738 739 740 741 742 743 744
	default:
		return RESUME_HOST;
	}
	kvmppc_set_gpr(vcpu, 3, ret);
	vcpu->arch.hcall_needed = 0;
	return RESUME_GUEST;
}

745 746 747 748 749 750 751
static int kvmppc_hcall_impl_hv(unsigned long cmd)
{
	switch (cmd) {
	case H_CEDE:
	case H_PROD:
	case H_CONFER:
	case H_REGISTER_VPA:
752
	case H_SET_MODE:
753 754
	case H_LOGICAL_CI_LOAD:
	case H_LOGICAL_CI_STORE:
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
#ifdef CONFIG_KVM_XICS
	case H_XIRR:
	case H_CPPR:
	case H_EOI:
	case H_IPI:
	case H_IPOLL:
	case H_XIRR_X:
#endif
		return 1;
	}

	/* See if it's in the real-mode table */
	return kvmppc_hcall_impl_hv_realmode(cmd);
}

770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
static int kvmppc_emulate_debug_inst(struct kvm_run *run,
					struct kvm_vcpu *vcpu)
{
	u32 last_inst;

	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
					EMULATE_DONE) {
		/*
		 * Fetch failed, so return to guest and
		 * try executing it again.
		 */
		return RESUME_GUEST;
	}

	if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
		run->exit_reason = KVM_EXIT_DEBUG;
		run->debug.arch.address = kvmppc_get_pc(vcpu);
		return RESUME_HOST;
	} else {
		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
		return RESUME_GUEST;
	}
}

794 795
static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
				 struct task_struct *tsk)
796 797 798 799 800 801 802 803 804 805 806 807 808 809
{
	int r = RESUME_HOST;

	vcpu->stat.sum_exits++;

	run->exit_reason = KVM_EXIT_UNKNOWN;
	run->ready_for_interrupt_injection = 1;
	switch (vcpu->arch.trap) {
	/* We're good on these - the host merely wanted to get our attention */
	case BOOK3S_INTERRUPT_HV_DECREMENTER:
		vcpu->stat.dec_exits++;
		r = RESUME_GUEST;
		break;
	case BOOK3S_INTERRUPT_EXTERNAL:
810
	case BOOK3S_INTERRUPT_H_DOORBELL:
811 812 813
		vcpu->stat.ext_intr_exits++;
		r = RESUME_GUEST;
		break;
814 815
	/* HMI is hypervisor interrupt and host has handled it. Resume guest.*/
	case BOOK3S_INTERRUPT_HMI:
816 817 818
	case BOOK3S_INTERRUPT_PERFMON:
		r = RESUME_GUEST;
		break;
819 820 821 822 823 824 825 826 827 828 829
	case BOOK3S_INTERRUPT_MACHINE_CHECK:
		/*
		 * Deliver a machine check interrupt to the guest.
		 * We have to do this, even if the host has handled the
		 * machine check, because machine checks use SRR0/1 and
		 * the interrupt might have trashed guest state in them.
		 */
		kvmppc_book3s_queue_irqprio(vcpu,
					    BOOK3S_INTERRUPT_MACHINE_CHECK);
		r = RESUME_GUEST;
		break;
830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
	case BOOK3S_INTERRUPT_PROGRAM:
	{
		ulong flags;
		/*
		 * Normally program interrupts are delivered directly
		 * to the guest by the hardware, but we can get here
		 * as a result of a hypervisor emulation interrupt
		 * (e40) getting turned into a 700 by BML RTAS.
		 */
		flags = vcpu->arch.shregs.msr & 0x1f0000ull;
		kvmppc_core_queue_program(vcpu, flags);
		r = RESUME_GUEST;
		break;
	}
	case BOOK3S_INTERRUPT_SYSCALL:
	{
		/* hcall - punt to userspace */
		int i;

849 850 851 852
		/* hypercall with MSR_PR has already been handled in rmode,
		 * and never reaches here.
		 */

853 854 855 856 857 858 859 860 861
		run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
		for (i = 0; i < 9; ++i)
			run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
		run->exit_reason = KVM_EXIT_PAPR_HCALL;
		vcpu->arch.hcall_needed = 1;
		r = RESUME_HOST;
		break;
	}
	/*
862 863 864 865 866
	 * We get these next two if the guest accesses a page which it thinks
	 * it has mapped but which is not actually present, either because
	 * it is for an emulated I/O device or because the corresonding
	 * host page has been paged out.  Any other HDSI/HISI interrupts
	 * have been handled already.
867 868
	 */
	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
869
		r = RESUME_PAGE_FAULT;
870 871
		break;
	case BOOK3S_INTERRUPT_H_INST_STORAGE:
872 873 874
		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
		vcpu->arch.fault_dsisr = 0;
		r = RESUME_PAGE_FAULT;
875 876 877
		break;
	/*
	 * This occurs if the guest executes an illegal instruction.
878 879 880 881
	 * If the guest debug is disabled, generate a program interrupt
	 * to the guest. If guest debug is enabled, we need to check
	 * whether the instruction is a software breakpoint instruction.
	 * Accordingly return to Guest or Host.
882 883
	 */
	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
884 885 886 887
		if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
			vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
				swab32(vcpu->arch.emul_inst) :
				vcpu->arch.emul_inst;
888 889 890 891 892 893
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
			r = kvmppc_emulate_debug_inst(run, vcpu);
		} else {
			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
			r = RESUME_GUEST;
		}
894 895 896 897 898 899 900 901
		break;
	/*
	 * This occurs if the guest (kernel or userspace), does something that
	 * is prohibited by HFSCR.  We just generate a program interrupt to
	 * the guest.
	 */
	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
902 903 904 905 906 907 908
		r = RESUME_GUEST;
		break;
	default:
		kvmppc_dump_regs(vcpu);
		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
			vcpu->arch.trap, kvmppc_get_pc(vcpu),
			vcpu->arch.shregs.msr);
909
		run->hw.hardware_exit_reason = vcpu->arch.trap;
910 911 912 913 914 915 916
		r = RESUME_HOST;
		break;
	}

	return r;
}

917 918
static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
					    struct kvm_sregs *sregs)
919 920 921 922
{
	int i;

	memset(sregs, 0, sizeof(struct kvm_sregs));
923
	sregs->pvr = vcpu->arch.pvr;
924 925 926 927 928 929 930 931
	for (i = 0; i < vcpu->arch.slb_max; i++) {
		sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
		sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
	}

	return 0;
}

932 933
static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
					    struct kvm_sregs *sregs)
934 935 936
{
	int i, j;

937 938 939
	/* Only accept the same PVR as the host's, since we can't spoof it */
	if (sregs->pvr != vcpu->arch.pvr)
		return -EINVAL;
940 941 942 943 944 945 946 947 948 949 950 951 952 953

	j = 0;
	for (i = 0; i < vcpu->arch.slb_nr; i++) {
		if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
			vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
			vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
			++j;
		}
	}
	vcpu->arch.slb_max = j;

	return 0;
}

954 955
static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
		bool preserve_top32)
956
{
957
	struct kvm *kvm = vcpu->kvm;
958 959 960
	struct kvmppc_vcore *vc = vcpu->arch.vcore;
	u64 mask;

961
	mutex_lock(&kvm->lock);
962
	spin_lock(&vc->lock);
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
	/*
	 * If ILE (interrupt little-endian) has changed, update the
	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
	 */
	if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
		struct kvm_vcpu *vcpu;
		int i;

		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (vcpu->arch.vcore != vc)
				continue;
			if (new_lpcr & LPCR_ILE)
				vcpu->arch.intr_msr |= MSR_LE;
			else
				vcpu->arch.intr_msr &= ~MSR_LE;
		}
	}

981 982 983
	/*
	 * Userspace can only modify DPFD (default prefetch depth),
	 * ILE (interrupt little-endian) and TC (translation control).
984
	 * On POWER8 userspace can also modify AIL (alt. interrupt loc.)
985 986
	 */
	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
987 988
	if (cpu_has_feature(CPU_FTR_ARCH_207S))
		mask |= LPCR_AIL;
989 990 991 992

	/* Broken 32-bit version of LPCR must not clear top bits */
	if (preserve_top32)
		mask &= 0xFFFFFFFF;
993 994
	vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
	spin_unlock(&vc->lock);
995
	mutex_unlock(&kvm->lock);
996 997
}

998 999
static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
				 union kvmppc_one_reg *val)
1000
{
1001 1002
	int r = 0;
	long int i;
1003

1004
	switch (id) {
1005 1006 1007
	case KVM_REG_PPC_DEBUG_INST:
		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
		break;
1008
	case KVM_REG_PPC_HIOR:
1009 1010 1011 1012 1013
		*val = get_reg_val(id, 0);
		break;
	case KVM_REG_PPC_DABR:
		*val = get_reg_val(id, vcpu->arch.dabr);
		break;
1014 1015 1016
	case KVM_REG_PPC_DABRX:
		*val = get_reg_val(id, vcpu->arch.dabrx);
		break;
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	case KVM_REG_PPC_DSCR:
		*val = get_reg_val(id, vcpu->arch.dscr);
		break;
	case KVM_REG_PPC_PURR:
		*val = get_reg_val(id, vcpu->arch.purr);
		break;
	case KVM_REG_PPC_SPURR:
		*val = get_reg_val(id, vcpu->arch.spurr);
		break;
	case KVM_REG_PPC_AMR:
		*val = get_reg_val(id, vcpu->arch.amr);
		break;
	case KVM_REG_PPC_UAMOR:
		*val = get_reg_val(id, vcpu->arch.uamor);
		break;
1032
	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1033 1034 1035 1036 1037 1038
		i = id - KVM_REG_PPC_MMCR0;
		*val = get_reg_val(id, vcpu->arch.mmcr[i]);
		break;
	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
		i = id - KVM_REG_PPC_PMC1;
		*val = get_reg_val(id, vcpu->arch.pmc[i]);
1039
		break;
1040 1041 1042 1043
	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
		i = id - KVM_REG_PPC_SPMC1;
		*val = get_reg_val(id, vcpu->arch.spmc[i]);
		break;
1044 1045 1046 1047 1048 1049
	case KVM_REG_PPC_SIAR:
		*val = get_reg_val(id, vcpu->arch.siar);
		break;
	case KVM_REG_PPC_SDAR:
		*val = get_reg_val(id, vcpu->arch.sdar);
		break;
1050 1051
	case KVM_REG_PPC_SIER:
		*val = get_reg_val(id, vcpu->arch.sier);
1052
		break;
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	case KVM_REG_PPC_IAMR:
		*val = get_reg_val(id, vcpu->arch.iamr);
		break;
	case KVM_REG_PPC_PSPB:
		*val = get_reg_val(id, vcpu->arch.pspb);
		break;
	case KVM_REG_PPC_DPDES:
		*val = get_reg_val(id, vcpu->arch.vcore->dpdes);
		break;
	case KVM_REG_PPC_DAWR:
		*val = get_reg_val(id, vcpu->arch.dawr);
		break;
	case KVM_REG_PPC_DAWRX:
		*val = get_reg_val(id, vcpu->arch.dawrx);
		break;
	case KVM_REG_PPC_CIABR:
		*val = get_reg_val(id, vcpu->arch.ciabr);
		break;
	case KVM_REG_PPC_CSIGR:
		*val = get_reg_val(id, vcpu->arch.csigr);
		break;
	case KVM_REG_PPC_TACR:
		*val = get_reg_val(id, vcpu->arch.tacr);
		break;
	case KVM_REG_PPC_TCSCR:
		*val = get_reg_val(id, vcpu->arch.tcscr);
		break;
	case KVM_REG_PPC_PID:
		*val = get_reg_val(id, vcpu->arch.pid);
		break;
	case KVM_REG_PPC_ACOP:
		*val = get_reg_val(id, vcpu->arch.acop);
		break;
	case KVM_REG_PPC_WORT:
		*val = get_reg_val(id, vcpu->arch.wort);
1088
		break;
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	case KVM_REG_PPC_VPA_ADDR:
		spin_lock(&vcpu->arch.vpa_update_lock);
		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
		spin_unlock(&vcpu->arch.vpa_update_lock);
		break;
	case KVM_REG_PPC_VPA_SLB:
		spin_lock(&vcpu->arch.vpa_update_lock);
		val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
		val->vpaval.length = vcpu->arch.slb_shadow.len;
		spin_unlock(&vcpu->arch.vpa_update_lock);
		break;
	case KVM_REG_PPC_VPA_DTL:
		spin_lock(&vcpu->arch.vpa_update_lock);
		val->vpaval.addr = vcpu->arch.dtl.next_gpa;
		val->vpaval.length = vcpu->arch.dtl.len;
		spin_unlock(&vcpu->arch.vpa_update_lock);
		break;
1106 1107 1108
	case KVM_REG_PPC_TB_OFFSET:
		*val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
		break;
1109
	case KVM_REG_PPC_LPCR:
1110
	case KVM_REG_PPC_LPCR_64:
1111 1112
		*val = get_reg_val(id, vcpu->arch.vcore->lpcr);
		break;
1113 1114 1115
	case KVM_REG_PPC_PPR:
		*val = get_reg_val(id, vcpu->arch.ppr);
		break;
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	case KVM_REG_PPC_TFHAR:
		*val = get_reg_val(id, vcpu->arch.tfhar);
		break;
	case KVM_REG_PPC_TFIAR:
		*val = get_reg_val(id, vcpu->arch.tfiar);
		break;
	case KVM_REG_PPC_TEXASR:
		*val = get_reg_val(id, vcpu->arch.texasr);
		break;
	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
		i = id - KVM_REG_PPC_TM_GPR0;
		*val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
		break;
	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
	{
		int j;
		i = id - KVM_REG_PPC_TM_VSR0;
		if (i < 32)
			for (j = 0; j < TS_FPRWIDTH; j++)
				val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
		else {
			if (cpu_has_feature(CPU_FTR_ALTIVEC))
				val->vval = vcpu->arch.vr_tm.vr[i-32];
			else
				r = -ENXIO;
		}
		break;
	}
	case KVM_REG_PPC_TM_CR:
		*val = get_reg_val(id, vcpu->arch.cr_tm);
		break;
	case KVM_REG_PPC_TM_LR:
		*val = get_reg_val(id, vcpu->arch.lr_tm);
		break;
	case KVM_REG_PPC_TM_CTR:
		*val = get_reg_val(id, vcpu->arch.ctr_tm);
		break;
	case KVM_REG_PPC_TM_FPSCR:
		*val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
		break;
	case KVM_REG_PPC_TM_AMR:
		*val = get_reg_val(id, vcpu->arch.amr_tm);
		break;
	case KVM_REG_PPC_TM_PPR:
		*val = get_reg_val(id, vcpu->arch.ppr_tm);
		break;
	case KVM_REG_PPC_TM_VRSAVE:
		*val = get_reg_val(id, vcpu->arch.vrsave_tm);
		break;
	case KVM_REG_PPC_TM_VSCR:
		if (cpu_has_feature(CPU_FTR_ALTIVEC))
			*val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
		else
			r = -ENXIO;
		break;
	case KVM_REG_PPC_TM_DSCR:
		*val = get_reg_val(id, vcpu->arch.dscr_tm);
		break;
	case KVM_REG_PPC_TM_TAR:
		*val = get_reg_val(id, vcpu->arch.tar_tm);
		break;
#endif
1179 1180 1181
	case KVM_REG_PPC_ARCH_COMPAT:
		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
		break;
1182
	default:
1183
		r = -EINVAL;
1184 1185 1186 1187 1188 1189
		break;
	}

	return r;
}

1190 1191
static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
				 union kvmppc_one_reg *val)
1192
{
1193 1194
	int r = 0;
	long int i;
1195
	unsigned long addr, len;
1196

1197
	switch (id) {
1198 1199
	case KVM_REG_PPC_HIOR:
		/* Only allow this to be set to zero */
1200
		if (set_reg_val(id, *val))
1201 1202
			r = -EINVAL;
		break;
1203 1204 1205
	case KVM_REG_PPC_DABR:
		vcpu->arch.dabr = set_reg_val(id, *val);
		break;
1206 1207 1208
	case KVM_REG_PPC_DABRX:
		vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
		break;
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
	case KVM_REG_PPC_DSCR:
		vcpu->arch.dscr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_PURR:
		vcpu->arch.purr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_SPURR:
		vcpu->arch.spurr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_AMR:
		vcpu->arch.amr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_UAMOR:
		vcpu->arch.uamor = set_reg_val(id, *val);
		break;
1224
	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1225 1226 1227 1228 1229 1230 1231
		i = id - KVM_REG_PPC_MMCR0;
		vcpu->arch.mmcr[i] = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
		i = id - KVM_REG_PPC_PMC1;
		vcpu->arch.pmc[i] = set_reg_val(id, *val);
		break;
1232 1233 1234 1235
	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
		i = id - KVM_REG_PPC_SPMC1;
		vcpu->arch.spmc[i] = set_reg_val(id, *val);
		break;
1236 1237 1238 1239 1240 1241
	case KVM_REG_PPC_SIAR:
		vcpu->arch.siar = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_SDAR:
		vcpu->arch.sdar = set_reg_val(id, *val);
		break;
1242 1243
	case KVM_REG_PPC_SIER:
		vcpu->arch.sier = set_reg_val(id, *val);
1244
		break;
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
	case KVM_REG_PPC_IAMR:
		vcpu->arch.iamr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_PSPB:
		vcpu->arch.pspb = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_DPDES:
		vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_DAWR:
		vcpu->arch.dawr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_DAWRX:
		vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
		break;
	case KVM_REG_PPC_CIABR:
		vcpu->arch.ciabr = set_reg_val(id, *val);
		/* Don't allow setting breakpoints in hypervisor code */
		if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
			vcpu->arch.ciabr &= ~CIABR_PRIV;	/* disable */
		break;
	case KVM_REG_PPC_CSIGR:
		vcpu->arch.csigr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TACR:
		vcpu->arch.tacr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TCSCR:
		vcpu->arch.tcscr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_PID:
		vcpu->arch.pid = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_ACOP:
		vcpu->arch.acop = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_WORT:
		vcpu->arch.wort = set_reg_val(id, *val);
1283
		break;
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
	case KVM_REG_PPC_VPA_ADDR:
		addr = set_reg_val(id, *val);
		r = -EINVAL;
		if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
			      vcpu->arch.dtl.next_gpa))
			break;
		r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
		break;
	case KVM_REG_PPC_VPA_SLB:
		addr = val->vpaval.addr;
		len = val->vpaval.length;
		r = -EINVAL;
		if (addr && !vcpu->arch.vpa.next_gpa)
			break;
		r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
		break;
	case KVM_REG_PPC_VPA_DTL:
		addr = val->vpaval.addr;
		len = val->vpaval.length;
		r = -EINVAL;
1304 1305
		if (addr && (len < sizeof(struct dtl_entry) ||
			     !vcpu->arch.vpa.next_gpa))
1306 1307 1308 1309
			break;
		len -= len % sizeof(struct dtl_entry);
		r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
		break;
1310 1311 1312 1313 1314
	case KVM_REG_PPC_TB_OFFSET:
		/* round up to multiple of 2^24 */
		vcpu->arch.vcore->tb_offset =
			ALIGN(set_reg_val(id, *val), 1UL << 24);
		break;
1315
	case KVM_REG_PPC_LPCR:
1316 1317 1318 1319
		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
		break;
	case KVM_REG_PPC_LPCR_64:
		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1320
		break;
1321 1322 1323
	case KVM_REG_PPC_PPR:
		vcpu->arch.ppr = set_reg_val(id, *val);
		break;
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	case KVM_REG_PPC_TFHAR:
		vcpu->arch.tfhar = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TFIAR:
		vcpu->arch.tfiar = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TEXASR:
		vcpu->arch.texasr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
		i = id - KVM_REG_PPC_TM_GPR0;
		vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
	{
		int j;
		i = id - KVM_REG_PPC_TM_VSR0;
		if (i < 32)
			for (j = 0; j < TS_FPRWIDTH; j++)
				vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
		else
			if (cpu_has_feature(CPU_FTR_ALTIVEC))
				vcpu->arch.vr_tm.vr[i-32] = val->vval;
			else
				r = -ENXIO;
		break;
	}
	case KVM_REG_PPC_TM_CR:
		vcpu->arch.cr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_LR:
		vcpu->arch.lr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_CTR:
		vcpu->arch.ctr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_FPSCR:
		vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_AMR:
		vcpu->arch.amr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_PPR:
		vcpu->arch.ppr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_VRSAVE:
		vcpu->arch.vrsave_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_VSCR:
		if (cpu_has_feature(CPU_FTR_ALTIVEC))
			vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
		else
			r = - ENXIO;
		break;
	case KVM_REG_PPC_TM_DSCR:
		vcpu->arch.dscr_tm = set_reg_val(id, *val);
		break;
	case KVM_REG_PPC_TM_TAR:
		vcpu->arch.tar_tm = set_reg_val(id, *val);
		break;
#endif
1386 1387 1388
	case KVM_REG_PPC_ARCH_COMPAT:
		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
		break;
1389
	default:
1390
		r = -EINVAL;
1391 1392 1393 1394 1395 1396
		break;
	}

	return r;
}

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
{
	struct kvmppc_vcore *vcore;

	vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);

	if (vcore == NULL)
		return NULL;

	INIT_LIST_HEAD(&vcore->runnable_threads);
	spin_lock_init(&vcore->lock);
1408
	spin_lock_init(&vcore->stoltb_lock);
1409 1410 1411 1412 1413 1414
	init_waitqueue_head(&vcore->wq);
	vcore->preempt_tb = TB_NIL;
	vcore->lpcr = kvm->arch.lpcr;
	vcore->first_vcpuid = core * threads_per_subcore;
	vcore->kvm = kvm;

1415 1416 1417 1418 1419 1420 1421
	vcore->mpp_buffer_is_valid = false;

	if (cpu_has_feature(CPU_FTR_ARCH_207S))
		vcore->mpp_buffer = (void *)__get_free_pages(
			GFP_KERNEL|__GFP_ZERO,
			MPP_BUFFER_ORDER);

1422 1423 1424
	return vcore;
}

1425 1426
static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
						   unsigned int id)
1427 1428
{
	struct kvm_vcpu *vcpu;
1429 1430 1431
	int err = -EINVAL;
	int core;
	struct kvmppc_vcore *vcore;
1432

1433
	core = id / threads_per_subcore;
1434 1435 1436 1437
	if (core >= KVM_MAX_VCORES)
		goto out;

	err = -ENOMEM;
1438
	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1439 1440 1441 1442 1443 1444 1445 1446
	if (!vcpu)
		goto out;

	err = kvm_vcpu_init(vcpu, kvm, id);
	if (err)
		goto free_vcpu;

	vcpu->arch.shared = &vcpu->arch.shregs;
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
	/*
	 * The shared struct is never shared on HV,
	 * so we can always use host endianness
	 */
#ifdef __BIG_ENDIAN__
	vcpu->arch.shared_big_endian = true;
#else
	vcpu->arch.shared_big_endian = false;
#endif
#endif
1458 1459 1460
	vcpu->arch.mmcr[0] = MMCR0_FC;
	vcpu->arch.ctrl = CTRL_RUNLATCH;
	/* default to host PVR, since we can't spoof it */
1461
	kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
1462
	spin_lock_init(&vcpu->arch.vpa_update_lock);
1463 1464
	spin_lock_init(&vcpu->arch.tbacct_lock);
	vcpu->arch.busy_preempt = TB_NIL;
1465
	vcpu->arch.intr_msr = MSR_SF | MSR_ME;
1466 1467 1468

	kvmppc_mmu_book3s_hv_init(vcpu);

1469
	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1470 1471 1472 1473 1474 1475

	init_waitqueue_head(&vcpu->arch.cpu_run);

	mutex_lock(&kvm->lock);
	vcore = kvm->arch.vcores[core];
	if (!vcore) {
1476
		vcore = kvmppc_vcore_create(kvm, core);
1477
		kvm->arch.vcores[core] = vcore;
1478
		kvm->arch.online_vcores++;
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
	}
	mutex_unlock(&kvm->lock);

	if (!vcore)
		goto free_vcpu;

	spin_lock(&vcore->lock);
	++vcore->num_threads;
	spin_unlock(&vcore->lock);
	vcpu->arch.vcore = vcore;
1489
	vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
1490

1491 1492 1493
	vcpu->arch.cpu_type = KVM_CPU_3S_64;
	kvmppc_sanity_check(vcpu);

1494 1495 1496
	return vcpu;

free_vcpu:
1497
	kmem_cache_free(kvm_vcpu_cache, vcpu);
1498 1499 1500 1501
out:
	return ERR_PTR(err);
}

1502 1503 1504 1505 1506 1507 1508
static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
{
	if (vpa->pinned_addr)
		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
					vpa->dirty);
}

1509
static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
1510
{
1511
	spin_lock(&vcpu->arch.vpa_update_lock);
1512 1513 1514
	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
1515
	spin_unlock(&vcpu->arch.vpa_update_lock);
1516
	kvm_vcpu_uninit(vcpu);
1517
	kmem_cache_free(kvm_vcpu_cache, vcpu);
1518 1519
}

1520 1521 1522 1523 1524 1525
static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
{
	/* Indicate we want to get back into the guest */
	return 1;
}

1526
static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
1527
{
1528
	unsigned long dec_nsec, now;
1529

1530 1531 1532 1533
	now = get_tb();
	if (now > vcpu->arch.dec_expires) {
		/* decrementer has already gone negative */
		kvmppc_core_queue_dec(vcpu);
1534
		kvmppc_core_prepare_to_enter(vcpu);
1535
		return;
1536
	}
1537 1538 1539 1540 1541
	dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
		   / tb_ticks_per_sec;
	hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
		      HRTIMER_MODE_REL);
	vcpu->arch.timer_running = 1;
1542 1543
}

1544
static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
1545
{
1546 1547 1548 1549 1550
	vcpu->arch.ceded = 0;
	if (vcpu->arch.timer_running) {
		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
		vcpu->arch.timer_running = 0;
	}
1551 1552
}

1553
extern void __kvmppc_vcore_entry(void);
1554

1555 1556
static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
				   struct kvm_vcpu *vcpu)
1557
{
1558 1559
	u64 now;

1560 1561
	if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
		return;
1562
	spin_lock_irq(&vcpu->arch.tbacct_lock);
1563 1564 1565 1566 1567
	now = mftb();
	vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
		vcpu->arch.stolen_logged;
	vcpu->arch.busy_preempt = now;
	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
1568
	spin_unlock_irq(&vcpu->arch.tbacct_lock);
1569 1570 1571 1572
	--vc->n_runnable;
	list_del(&vcpu->arch.run_list);
}

1573 1574 1575
static int kvmppc_grab_hwthread(int cpu)
{
	struct paca_struct *tpaca;
1576
	long timeout = 10000;
1577 1578 1579 1580 1581

	tpaca = &paca[cpu];

	/* Ensure the thread won't go into the kernel if it wakes */
	tpaca->kvm_hstate.hwthread_req = 1;
1582
	tpaca->kvm_hstate.kvm_vcpu = NULL;
1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612

	/*
	 * If the thread is already executing in the kernel (e.g. handling
	 * a stray interrupt), wait for it to get back to nap mode.
	 * The smp_mb() is to ensure that our setting of hwthread_req
	 * is visible before we look at hwthread_state, so if this
	 * races with the code at system_reset_pSeries and the thread
	 * misses our setting of hwthread_req, we are sure to see its
	 * setting of hwthread_state, and vice versa.
	 */
	smp_mb();
	while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
		if (--timeout <= 0) {
			pr_err("KVM: couldn't grab cpu %d\n", cpu);
			return -EBUSY;
		}
		udelay(1);
	}
	return 0;
}

static void kvmppc_release_hwthread(int cpu)
{
	struct paca_struct *tpaca;

	tpaca = &paca[cpu];
	tpaca->kvm_hstate.hwthread_req = 0;
	tpaca->kvm_hstate.kvm_vcpu = NULL;
}

1613 1614 1615 1616 1617 1618
static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
{
	int cpu;
	struct paca_struct *tpaca;
	struct kvmppc_vcore *vc = vcpu->arch.vcore;

1619 1620 1621 1622
	if (vcpu->arch.timer_running) {
		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
		vcpu->arch.timer_running = 0;
	}
1623 1624 1625 1626
	cpu = vc->pcpu + vcpu->arch.ptid;
	tpaca = &paca[cpu];
	tpaca->kvm_hstate.kvm_vcpu = vcpu;
	tpaca->kvm_hstate.kvm_vcore = vc;
1627
	tpaca->kvm_hstate.ptid = vcpu->arch.ptid;
1628
	vcpu->cpu = vc->pcpu;
1629
	smp_wmb();
1630
#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
1631
	if (cpu != smp_processor_id()) {
1632
		xics_wake_cpu(cpu);
1633 1634
		if (vcpu->arch.ptid)
			++vc->n_woken;
1635
	}
1636 1637
#endif
}
1638

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
{
	int i;

	HMT_low();
	i = 0;
	while (vc->nap_count < vc->n_woken) {
		if (++i >= 1000000) {
			pr_err("kvmppc_wait_for_nap timeout %d %d\n",
			       vc->nap_count, vc->n_woken);
			break;
		}
		cpu_relax();
	}
	HMT_medium();
}

/*
 * Check that we are on thread 0 and that any other threads in
1658 1659
 * this core are off-line.  Then grab the threads so they can't
 * enter the kernel.
1660 1661 1662 1663
 */
static int on_primary_thread(void)
{
	int cpu = smp_processor_id();
1664
	int thr;
1665

1666 1667
	/* Are we on a primary subcore? */
	if (cpu_thread_in_subcore(cpu))
1668
		return 0;
1669 1670 1671

	thr = 0;
	while (++thr < threads_per_subcore)
1672 1673
		if (cpu_online(cpu + thr))
			return 0;
1674 1675

	/* Grab all hw threads so they can't go into the kernel */
1676
	for (thr = 1; thr < threads_per_subcore; ++thr) {
1677 1678 1679 1680 1681 1682 1683 1684
		if (kvmppc_grab_hwthread(cpu + thr)) {
			/* Couldn't grab one; let the others go */
			do {
				kvmppc_release_hwthread(cpu + thr);
			} while (--thr > 0);
			return 0;
		}
	}
1685 1686 1687
	return 1;
}

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
static void kvmppc_start_saving_l2_cache(struct kvmppc_vcore *vc)
{
	phys_addr_t phy_addr, mpp_addr;

	phy_addr = (phys_addr_t)virt_to_phys(vc->mpp_buffer);
	mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;

	mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_ABORT);
	logmpp(mpp_addr | PPC_LOGMPP_LOG_L2);

	vc->mpp_buffer_is_valid = true;
}

static void kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore *vc)
{
	phys_addr_t phy_addr, mpp_addr;

	phy_addr = virt_to_phys(vc->mpp_buffer);
	mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;

	/* We must abort any in-progress save operations to ensure
	 * the table is valid so that prefetch engine knows when to
	 * stop prefetching. */
	logmpp(mpp_addr | PPC_LOGMPP_LOG_ABORT);
	mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE);
}

1715 1716 1717 1718
/*
 * Run a set of guest threads on a physical core.
 * Called with vc->lock held.
 */
1719
static void kvmppc_run_core(struct kvmppc_vcore *vc)
1720
{
1721
	struct kvm_vcpu *vcpu, *vnext;
1722 1723
	long ret;
	u64 now;
1724
	int i, need_vpa_update;
1725
	int srcu_idx;
1726
	struct kvm_vcpu *vcpus_to_update[threads_per_core];
1727 1728

	/* don't start if any threads have a signal pending */
1729 1730
	need_vpa_update = 0;
	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1731
		if (signal_pending(vcpu->arch.run_task))
1732 1733 1734 1735 1736
			return;
		if (vcpu->arch.vpa.update_pending ||
		    vcpu->arch.slb_shadow.update_pending ||
		    vcpu->arch.dtl.update_pending)
			vcpus_to_update[need_vpa_update++] = vcpu;
1737 1738 1739 1740 1741 1742 1743 1744 1745
	}

	/*
	 * Initialize *vc, in particular vc->vcore_state, so we can
	 * drop the vcore lock if necessary.
	 */
	vc->n_woken = 0;
	vc->nap_count = 0;
	vc->entry_exit_count = 0;
1746
	vc->preempt_tb = TB_NIL;
1747
	vc->vcore_state = VCORE_STARTING;
1748 1749
	vc->in_guest = 0;
	vc->napping_threads = 0;
1750
	vc->conferring_threads = 0;
1751 1752 1753 1754 1755 1756 1757

	/*
	 * Updating any of the vpas requires calling kvmppc_pin_guest_page,
	 * which can't be called with any spinlocks held.
	 */
	if (need_vpa_update) {
		spin_unlock(&vc->lock);
1758 1759
		for (i = 0; i < need_vpa_update; ++i)
			kvmppc_update_vpas(vcpus_to_update[i]);
1760 1761
		spin_lock(&vc->lock);
	}
1762

1763
	/*
1764 1765 1766
	 * Make sure we are running on primary threads, and that secondary
	 * threads are offline.  Also check if the number of threads in this
	 * guest are greater than the current system threads per guest.
1767
	 */
1768 1769
	if ((threads_per_core > 1) &&
	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
1770 1771 1772 1773 1774
		list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
			vcpu->arch.ret = -EBUSY;
		goto out;
	}

1775

1776
	vc->pcpu = smp_processor_id();
1777
	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1778
		kvmppc_start_thread(vcpu);
1779
		kvmppc_create_dtl_entry(vcpu, vc);
1780
		trace_kvm_guest_enter(vcpu);
1781
	}
1782

1783 1784 1785 1786
	/* Set this explicitly in case thread 0 doesn't have a vcpu */
	get_paca()->kvm_hstate.kvm_vcore = vc;
	get_paca()->kvm_hstate.ptid = 0;

1787
	vc->vcore_state = VCORE_RUNNING;
1788
	preempt_disable();
1789 1790 1791

	trace_kvmppc_run_core(vc, 0);

1792
	spin_unlock(&vc->lock);
1793

1794
	kvm_guest_enter();
1795

1796
	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
1797

1798 1799 1800
	if (vc->mpp_buffer_is_valid)
		kvmppc_start_restoring_l2_cache(vc);

1801
	__kvmppc_vcore_entry();
1802

1803
	spin_lock(&vc->lock);
1804 1805 1806 1807

	if (vc->mpp_buffer)
		kvmppc_start_saving_l2_cache(vc);

1808 1809 1810 1811
	/* disable sending of IPIs on virtual external irqs */
	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
		vcpu->cpu = -1;
	/* wait for secondary threads to finish writing their state to memory */
1812 1813
	if (vc->nap_count < vc->n_woken)
		kvmppc_wait_for_nap(vc);
1814
	for (i = 0; i < threads_per_subcore; ++i)
1815
		kvmppc_release_hwthread(vc->pcpu + i);
1816
	/* prevent other vcpu threads from doing kvmppc_start_thread() now */
1817
	vc->vcore_state = VCORE_EXITING;
1818 1819
	spin_unlock(&vc->lock);

1820
	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
1821

1822 1823
	/* make sure updates to secondary vcpu structs are visible now */
	smp_mb();
1824 1825 1826
	kvm_guest_exit();

	preempt_enable();
1827
	cond_resched();
1828

1829
	spin_lock(&vc->lock);
1830
	now = get_tb();
1831 1832 1833 1834 1835
	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
		/* cancel pending dec exception if dec is positive */
		if (now < vcpu->arch.dec_expires &&
		    kvmppc_core_pending_dec(vcpu))
			kvmppc_core_dequeue_dec(vcpu);
1836

1837 1838
		trace_kvm_guest_exit(vcpu);

1839 1840
		ret = RESUME_GUEST;
		if (vcpu->arch.trap)
1841 1842
			ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
						    vcpu->arch.run_task);
1843

1844 1845
		vcpu->arch.ret = ret;
		vcpu->arch.trap = 0;
1846 1847

		if (vcpu->arch.ceded) {
1848
			if (!is_kvmppc_resume_guest(ret))
1849 1850 1851 1852
				kvmppc_end_cede(vcpu);
			else
				kvmppc_set_timer(vcpu);
		}
1853
	}
1854 1855

 out:
1856
	vc->vcore_state = VCORE_INACTIVE;
1857 1858
	list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
				 arch.run_list) {
1859
		if (!is_kvmppc_resume_guest(vcpu->arch.ret)) {
1860 1861 1862 1863
			kvmppc_remove_runnable(vc, vcpu);
			wake_up(&vcpu->arch.cpu_run);
		}
	}
1864 1865

	trace_kvmppc_run_core(vc, 1);
1866 1867
}

1868 1869 1870 1871 1872
/*
 * Wait for some other vcpu thread to execute us, and
 * wake us up when we need to handle something in the host.
 */
static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
1873 1874 1875
{
	DEFINE_WAIT(wait);

1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
	prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
		schedule();
	finish_wait(&vcpu->arch.cpu_run, &wait);
}

/*
 * All the vcpus in this vcore are idle, so wait for a decrementer
 * or external interrupt to one of the vcpus.  vc->lock is held.
 */
static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
{
1888 1889 1890
	struct kvm_vcpu *vcpu;
	int do_sleep = 1;

1891 1892 1893
	DEFINE_WAIT(wait);

	prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910

	/*
	 * Check one last time for pending exceptions and ceded state after
	 * we put ourselves on the wait queue
	 */
	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
		if (vcpu->arch.pending_exceptions || !vcpu->arch.ceded) {
			do_sleep = 0;
			break;
		}
	}

	if (!do_sleep) {
		finish_wait(&vc->wq, &wait);
		return;
	}

1911
	vc->vcore_state = VCORE_SLEEPING;
1912
	trace_kvmppc_vcore_blocked(vc, 0);
1913
	spin_unlock(&vc->lock);
1914
	schedule();
1915 1916 1917
	finish_wait(&vc->wq, &wait);
	spin_lock(&vc->lock);
	vc->vcore_state = VCORE_INACTIVE;
1918
	trace_kvmppc_vcore_blocked(vc, 1);
1919
}
1920

1921 1922 1923 1924 1925
static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
{
	int n_ceded;
	struct kvmppc_vcore *vc;
	struct kvm_vcpu *v, *vn;
1926

1927 1928
	trace_kvmppc_run_vcpu_enter(vcpu);

1929 1930 1931
	kvm_run->exit_reason = 0;
	vcpu->arch.ret = RESUME_GUEST;
	vcpu->arch.trap = 0;
1932
	kvmppc_update_vpas(vcpu);
1933 1934 1935 1936 1937 1938

	/*
	 * Synchronize with other threads in this virtual core
	 */
	vc = vcpu->arch.vcore;
	spin_lock(&vc->lock);
1939
	vcpu->arch.ceded = 0;
1940 1941
	vcpu->arch.run_task = current;
	vcpu->arch.kvm_run = kvm_run;
1942
	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
1943
	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
1944
	vcpu->arch.busy_preempt = TB_NIL;
1945 1946 1947
	list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
	++vc->n_runnable;

1948 1949 1950 1951 1952
	/*
	 * This happens the first time this is called for a vcpu.
	 * If the vcore is already running, we may be able to start
	 * this thread straight away and have it join in.
	 */
1953
	if (!signal_pending(current)) {
1954 1955
		if (vc->vcore_state == VCORE_RUNNING &&
		    VCORE_EXIT_COUNT(vc) == 0) {
1956
			kvmppc_create_dtl_entry(vcpu, vc);
1957
			kvmppc_start_thread(vcpu);
1958
			trace_kvm_guest_enter(vcpu);
1959 1960
		} else if (vc->vcore_state == VCORE_SLEEPING) {
			wake_up(&vc->wq);
1961 1962
		}

1963
	}
1964

1965 1966
	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
	       !signal_pending(current)) {
1967
		if (vc->vcore_state != VCORE_INACTIVE) {
1968 1969 1970 1971 1972 1973 1974
			spin_unlock(&vc->lock);
			kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE);
			spin_lock(&vc->lock);
			continue;
		}
		list_for_each_entry_safe(v, vn, &vc->runnable_threads,
					 arch.run_list) {
1975
			kvmppc_core_prepare_to_enter(v);
1976 1977 1978 1979 1980 1981 1982 1983
			if (signal_pending(v->arch.run_task)) {
				kvmppc_remove_runnable(vc, v);
				v->stat.signal_exits++;
				v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
				v->arch.ret = -EINTR;
				wake_up(&v->arch.cpu_run);
			}
		}
1984 1985 1986 1987
		if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
			break;
		vc->runner = vcpu;
		n_ceded = 0;
1988
		list_for_each_entry(v, &vc->runnable_threads, arch.run_list) {
1989 1990
			if (!v->arch.pending_exceptions)
				n_ceded += v->arch.ceded;
1991 1992 1993
			else
				v->arch.ceded = 0;
		}
1994 1995 1996 1997
		if (n_ceded == vc->n_runnable)
			kvmppc_vcore_blocked(vc);
		else
			kvmppc_run_core(vc);
1998
		vc->runner = NULL;
1999
	}
2000

2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020
	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
	       (vc->vcore_state == VCORE_RUNNING ||
		vc->vcore_state == VCORE_EXITING)) {
		spin_unlock(&vc->lock);
		kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE);
		spin_lock(&vc->lock);
	}

	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
		kvmppc_remove_runnable(vc, vcpu);
		vcpu->stat.signal_exits++;
		kvm_run->exit_reason = KVM_EXIT_INTR;
		vcpu->arch.ret = -EINTR;
	}

	if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
		/* Wake up some vcpu to run the core */
		v = list_first_entry(&vc->runnable_threads,
				     struct kvm_vcpu, arch.run_list);
		wake_up(&v->arch.cpu_run);
2021 2022
	}

2023
	trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
2024 2025
	spin_unlock(&vc->lock);
	return vcpu->arch.ret;
2026 2027
}

2028
static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
2029 2030
{
	int r;
2031
	int srcu_idx;
2032

2033 2034 2035 2036 2037
	if (!vcpu->arch.sane) {
		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		return -EINVAL;
	}

2038 2039
	kvmppc_core_prepare_to_enter(vcpu);

2040 2041 2042 2043 2044 2045
	/* No need to go into the guest when all we'll do is come back out */
	if (signal_pending(current)) {
		run->exit_reason = KVM_EXIT_INTR;
		return -EINTR;
	}

2046 2047 2048 2049
	atomic_inc(&vcpu->kvm->arch.vcpus_running);
	/* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */
	smp_mb();

2050
	/* On the first time here, set up HTAB and VRMA */
2051
	if (!vcpu->kvm->arch.rma_setup_done) {
2052
		r = kvmppc_hv_setup_htab_rma(vcpu);
2053
		if (r)
2054
			goto out;
2055
	}
2056 2057 2058 2059 2060

	flush_fp_to_thread(current);
	flush_altivec_to_thread(current);
	flush_vsx_to_thread(current);
	vcpu->arch.wqp = &vcpu->arch.vcore->wq;
2061
	vcpu->arch.pgdir = current->mm->pgd;
2062
	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2063

2064 2065 2066 2067 2068
	do {
		r = kvmppc_run_vcpu(run, vcpu);

		if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
		    !(vcpu->arch.shregs.msr & MSR_PR)) {
2069
			trace_kvm_hcall_enter(vcpu);
2070
			r = kvmppc_pseries_do_hcall(vcpu);
2071
			trace_kvm_hcall_exit(vcpu, r);
2072
			kvmppc_core_prepare_to_enter(vcpu);
2073 2074 2075 2076 2077
		} else if (r == RESUME_PAGE_FAULT) {
			srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
			r = kvmppc_book3s_hv_page_fault(run, vcpu,
				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
			srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2078
		}
2079
	} while (is_kvmppc_resume_guest(r));
2080 2081

 out:
2082
	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2083
	atomic_dec(&vcpu->kvm->arch.vcpus_running);
2084 2085 2086
	return r;
}

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
				     int linux_psize)
{
	struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];

	if (!def->shift)
		return;
	(*sps)->page_shift = def->shift;
	(*sps)->slb_enc = def->sllp;
	(*sps)->enc[0].page_shift = def->shift;
2097
	(*sps)->enc[0].pte_enc = def->penc[linux_psize];
2098 2099 2100 2101 2102 2103 2104
	/*
	 * Add 16MB MPSS support if host supports it
	 */
	if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) {
		(*sps)->enc[1].page_shift = 24;
		(*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M];
	}
2105 2106 2107
	(*sps)++;
}

2108 2109
static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
					 struct kvm_ppc_smmu_info *info)
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126
{
	struct kvm_ppc_one_seg_page_size *sps;

	info->flags = KVM_PPC_PAGE_SIZES_REAL;
	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
		info->flags |= KVM_PPC_1T_SEGMENTS;
	info->slb_size = mmu_slb_size;

	/* We only support these sizes for now, and no muti-size segments */
	sps = &info->sps[0];
	kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
	kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
	kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);

	return 0;
}

2127 2128 2129
/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
2130 2131
static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
					 struct kvm_dirty_log *log)
2132 2133 2134 2135 2136 2137 2138 2139
{
	struct kvm_memory_slot *memslot;
	int r;
	unsigned long n;

	mutex_lock(&kvm->slots_lock);

	r = -EINVAL;
2140
	if (log->slot >= KVM_USER_MEM_SLOTS)
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
		goto out;

	memslot = id_to_memslot(kvm->memslots, log->slot);
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

	n = kvm_dirty_bitmap_bytes(memslot);
	memset(memslot->dirty_bitmap, 0, n);

2151
	r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
	if (r)
		goto out;

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

	r = 0;
out:
	mutex_unlock(&kvm->slots_lock);
	return r;
}

2165 2166
static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
					struct kvm_memory_slot *dont)
2167 2168 2169 2170
{
	if (!dont || free->arch.rmap != dont->arch.rmap) {
		vfree(free->arch.rmap);
		free->arch.rmap = NULL;
2171
	}
2172 2173
}

2174 2175
static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
					 unsigned long npages)
2176 2177 2178 2179
{
	slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
	if (!slot->arch.rmap)
		return -ENOMEM;
2180

2181 2182
	return 0;
}
2183

2184 2185 2186
static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
					struct kvm_memory_slot *memslot,
					struct kvm_userspace_memory_region *mem)
2187
{
2188
	return 0;
2189 2190
}

2191 2192 2193
static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
				struct kvm_userspace_memory_region *mem,
				const struct kvm_memory_slot *old)
2194
{
2195 2196 2197
	unsigned long npages = mem->memory_size >> PAGE_SHIFT;
	struct kvm_memory_slot *memslot;

2198
	if (npages && old->npages) {
2199 2200 2201 2202 2203 2204 2205 2206 2207
		/*
		 * If modifying a memslot, reset all the rmap dirty bits.
		 * If this is a new memslot, we don't need to do anything
		 * since the rmap array starts out as all zeroes,
		 * i.e. no pages are dirty.
		 */
		memslot = id_to_memslot(kvm->memslots, mem->slot);
		kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
	}
2208 2209
}

2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235
/*
 * Update LPCR values in kvm->arch and in vcores.
 * Caller must hold kvm->lock.
 */
void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
{
	long int i;
	u32 cores_done = 0;

	if ((kvm->arch.lpcr & mask) == lpcr)
		return;

	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;

	for (i = 0; i < KVM_MAX_VCORES; ++i) {
		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
		if (!vc)
			continue;
		spin_lock(&vc->lock);
		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
		spin_unlock(&vc->lock);
		if (++cores_done >= kvm->arch.online_vcores)
			break;
	}
}

2236 2237 2238 2239 2240
static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
{
	return;
}

2241
static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2242 2243 2244 2245 2246 2247
{
	int err = 0;
	struct kvm *kvm = vcpu->kvm;
	unsigned long hva;
	struct kvm_memory_slot *memslot;
	struct vm_area_struct *vma;
2248
	unsigned long lpcr = 0, senc;
2249
	unsigned long psize, porder;
2250
	int srcu_idx;
2251 2252 2253 2254

	mutex_lock(&kvm->lock);
	if (kvm->arch.rma_setup_done)
		goto out;	/* another vcpu beat us to it */
2255

2256 2257 2258 2259 2260 2261 2262 2263 2264
	/* Allocate hashed page table (if not done already) and reset it */
	if (!kvm->arch.hpt_virt) {
		err = kvmppc_alloc_hpt(kvm, NULL);
		if (err) {
			pr_err("KVM: Couldn't alloc HPT\n");
			goto out;
		}
	}

2265
	/* Look up the memslot for guest physical address 0 */
2266
	srcu_idx = srcu_read_lock(&kvm->srcu);
2267
	memslot = gfn_to_memslot(kvm, 0);
2268

2269 2270 2271
	/* We must have some memory at 0 by now */
	err = -EINVAL;
	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
2272
		goto out_srcu;
2273 2274 2275 2276 2277 2278 2279 2280 2281

	/* Look up the VMA for the start of this memory slot */
	hva = memslot->userspace_addr;
	down_read(&current->mm->mmap_sem);
	vma = find_vma(current->mm, hva);
	if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
		goto up_out;

	psize = vma_kernel_pagesize(vma);
2282
	porder = __ilog2(psize);
2283 2284 2285

	up_read(&current->mm->mmap_sem);

2286 2287 2288 2289 2290
	/* We can handle 4k, 64k or 16M pages in the VRMA */
	err = -EINVAL;
	if (!(psize == 0x1000 || psize == 0x10000 ||
	      psize == 0x1000000))
		goto out_srcu;
2291

2292 2293 2294 2295 2296 2297
	/* Update VRMASD field in the LPCR */
	senc = slb_pgsize_encoding(psize);
	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
		(VRMA_VSID << SLB_VSID_SHIFT_1T);
	/* the -4 is to account for senc values starting at 0x10 */
	lpcr = senc << (LPCR_VRMASD_SH - 4);
2298

2299 2300
	/* Create HPTEs in the hash page table for the VRMA */
	kvmppc_map_vrma(vcpu, memslot, porder);
2301

2302
	kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
2303

2304 2305 2306 2307
	/* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */
	smp_wmb();
	kvm->arch.rma_setup_done = 1;
	err = 0;
2308 2309
 out_srcu:
	srcu_read_unlock(&kvm->srcu, srcu_idx);
2310 2311 2312
 out:
	mutex_unlock(&kvm->lock);
	return err;
2313

2314 2315
 up_out:
	up_read(&current->mm->mmap_sem);
2316
	goto out_srcu;
2317 2318
}

2319
static int kvmppc_core_init_vm_hv(struct kvm *kvm)
2320
{
2321
	unsigned long lpcr, lpid;
2322

2323 2324 2325
	/* Allocate the guest's logical partition ID */

	lpid = kvmppc_alloc_lpid();
2326
	if ((long)lpid < 0)
2327 2328
		return -ENOMEM;
	kvm->arch.lpid = lpid;
2329

2330 2331 2332 2333 2334 2335 2336
	/*
	 * Since we don't flush the TLB when tearing down a VM,
	 * and this lpid might have previously been used,
	 * make sure we flush on each core before running the new VM.
	 */
	cpumask_setall(&kvm->arch.need_tlb_flush);

2337 2338 2339 2340
	/* Start out with the default set of hcalls enabled */
	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
	       sizeof(kvm->arch.enabled_hcalls));

2341
	kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
2342

2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353
	/* Init LPCR for virtual RMA mode */
	kvm->arch.host_lpid = mfspr(SPRN_LPID);
	kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
	lpcr &= LPCR_PECE | LPCR_LPES;
	lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
		LPCR_VPM0 | LPCR_VPM1;
	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
		(VRMA_VSID << SLB_VSID_SHIFT_1T);
	/* On POWER8 turn on online bit to enable PURR/SPURR */
	if (cpu_has_feature(CPU_FTR_ARCH_207S))
		lpcr |= LPCR_ONL;
2354
	kvm->arch.lpcr = lpcr;
2355

2356
	/*
2357 2358
	 * Track that we now have a HV mode VM active. This blocks secondary
	 * CPU threads from coming online.
2359
	 */
2360
	kvm_hv_vm_activated();
2361

2362
	return 0;
2363 2364
}

2365 2366 2367 2368
static void kvmppc_free_vcores(struct kvm *kvm)
{
	long int i;

2369 2370 2371 2372 2373 2374
	for (i = 0; i < KVM_MAX_VCORES; ++i) {
		if (kvm->arch.vcores[i] && kvm->arch.vcores[i]->mpp_buffer) {
			struct kvmppc_vcore *vc = kvm->arch.vcores[i];
			free_pages((unsigned long)vc->mpp_buffer,
				   MPP_BUFFER_ORDER);
		}
2375
		kfree(kvm->arch.vcores[i]);
2376
	}
2377 2378 2379
	kvm->arch.online_vcores = 0;
}

2380
static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
2381
{
2382
	kvm_hv_vm_deactivated();
2383

2384
	kvmppc_free_vcores(kvm);
2385

2386 2387 2388
	kvmppc_free_hpt(kvm);
}

2389 2390 2391
/* We don't need to emulate any privileged instructions or dcbz */
static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
				     unsigned int inst, int *advance)
2392
{
2393
	return EMULATE_FAIL;
2394 2395
}

2396 2397
static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
					ulong spr_val)
2398 2399 2400 2401
{
	return EMULATE_FAIL;
}

2402 2403
static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
					ulong *spr_val)
2404 2405 2406 2407
{
	return EMULATE_FAIL;
}

2408
static int kvmppc_core_check_processor_compat_hv(void)
2409
{
2410 2411
	if (!cpu_has_feature(CPU_FTR_HVMODE) ||
	    !cpu_has_feature(CPU_FTR_ARCH_206))
2412 2413
		return -EIO;
	return 0;
2414 2415
}

2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
static long kvm_arch_vm_ioctl_hv(struct file *filp,
				 unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm __maybe_unused = filp->private_data;
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {

	case KVM_PPC_ALLOCATE_HTAB: {
		u32 htab_order;

		r = -EFAULT;
		if (get_user(htab_order, (u32 __user *)argp))
			break;
		r = kvmppc_alloc_reset_hpt(kvm, &htab_order);
		if (r)
			break;
		r = -EFAULT;
		if (put_user(htab_order, (u32 __user *)argp))
			break;
		r = 0;
		break;
	}

	case KVM_PPC_GET_HTAB_FD: {
		struct kvm_get_htab_fd ghf;

		r = -EFAULT;
		if (copy_from_user(&ghf, argp, sizeof(ghf)))
			break;
		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
		break;
	}

	default:
		r = -ENOTTY;
	}

	return r;
}

2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491
/*
 * List of hcall numbers to enable by default.
 * For compatibility with old userspace, we enable by default
 * all hcalls that were implemented before the hcall-enabling
 * facility was added.  Note this list should not include H_RTAS.
 */
static unsigned int default_hcall_list[] = {
	H_REMOVE,
	H_ENTER,
	H_READ,
	H_PROTECT,
	H_BULK_REMOVE,
	H_GET_TCE,
	H_PUT_TCE,
	H_SET_DABR,
	H_SET_XDABR,
	H_CEDE,
	H_PROD,
	H_CONFER,
	H_REGISTER_VPA,
#ifdef CONFIG_KVM_XICS
	H_EOI,
	H_CPPR,
	H_IPI,
	H_IPOLL,
	H_XIRR,
	H_XIRR_X,
#endif
	0
};

static void init_default_hcalls(void)
{
	int i;
2492
	unsigned int hcall;
2493

2494 2495 2496 2497 2498
	for (i = 0; default_hcall_list[i]; ++i) {
		hcall = default_hcall_list[i];
		WARN_ON(!kvmppc_hcall_impl_hv(hcall));
		__set_bit(hcall / 4, default_enabled_hcalls);
	}
2499 2500
}

2501
static struct kvmppc_ops kvm_ops_hv = {
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
	.get_one_reg = kvmppc_get_one_reg_hv,
	.set_one_reg = kvmppc_set_one_reg_hv,
	.vcpu_load   = kvmppc_core_vcpu_load_hv,
	.vcpu_put    = kvmppc_core_vcpu_put_hv,
	.set_msr     = kvmppc_set_msr_hv,
	.vcpu_run    = kvmppc_vcpu_run_hv,
	.vcpu_create = kvmppc_core_vcpu_create_hv,
	.vcpu_free   = kvmppc_core_vcpu_free_hv,
	.check_requests = kvmppc_core_check_requests_hv,
	.get_dirty_log  = kvm_vm_ioctl_get_dirty_log_hv,
	.flush_memslot  = kvmppc_core_flush_memslot_hv,
	.prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
	.commit_memory_region  = kvmppc_core_commit_memory_region_hv,
	.unmap_hva = kvm_unmap_hva_hv,
	.unmap_hva_range = kvm_unmap_hva_range_hv,
	.age_hva  = kvm_age_hva_hv,
	.test_age_hva = kvm_test_age_hva_hv,
	.set_spte_hva = kvm_set_spte_hva_hv,
	.mmu_destroy  = kvmppc_mmu_destroy_hv,
	.free_memslot = kvmppc_core_free_memslot_hv,
	.create_memslot = kvmppc_core_create_memslot_hv,
	.init_vm =  kvmppc_core_init_vm_hv,
	.destroy_vm = kvmppc_core_destroy_vm_hv,
	.get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
	.emulate_op = kvmppc_core_emulate_op_hv,
	.emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
	.emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
	.fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
	.arch_vm_ioctl  = kvm_arch_vm_ioctl_hv,
2533
	.hcall_implemented = kvmppc_hcall_impl_hv,
2534 2535 2536
};

static int kvmppc_book3s_init_hv(void)
2537 2538
{
	int r;
2539 2540 2541 2542 2543
	/*
	 * FIXME!! Do we need to check on all cpus ?
	 */
	r = kvmppc_core_check_processor_compat_hv();
	if (r < 0)
2544
		return -ENODEV;
2545

2546 2547
	kvm_ops_hv.owner = THIS_MODULE;
	kvmppc_hv_ops = &kvm_ops_hv;
2548

2549 2550
	init_default_hcalls();

2551
	r = kvmppc_mmu_hv_init();
2552 2553 2554
	return r;
}

2555
static void kvmppc_book3s_exit_hv(void)
2556
{
2557
	kvmppc_hv_ops = NULL;
2558 2559
}

2560 2561
module_init(kvmppc_book3s_init_hv);
module_exit(kvmppc_book3s_exit_hv);
2562
MODULE_LICENSE("GPL");
2563 2564
MODULE_ALIAS_MISCDEV(KVM_MINOR);
MODULE_ALIAS("devname:kvm");