book3s_hv_rmhandlers.S 66.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 *
 * Derived from book3s_rmhandlers.S and other files, which are:
 *
 * Copyright SUSE Linux Products GmbH 2009
 *
 * Authors: Alexander Graf <agraf@suse.de>
 */

#include <asm/ppc_asm.h>
#include <asm/kvm_asm.h>
#include <asm/reg.h>
23
#include <asm/mmu.h>
24
#include <asm/page.h>
25 26
#include <asm/ptrace.h>
#include <asm/hvcall.h>
27 28
#include <asm/asm-offsets.h>
#include <asm/exception-64s.h>
29
#include <asm/kvm_book3s_asm.h>
30
#include <asm/book3s/64/mmu-hash.h>
31
#include <asm/tm.h>
32
#include <asm/opal.h>
33 34

#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
35

36 37 38 39
/* Values in HSTATE_NAPPING(r13) */
#define NAPPING_CEDE	1
#define NAPPING_NOVCPU	2

40
/*
41
 * Call kvmppc_hv_entry in real mode.
42 43 44 45 46 47
 * Must be called with interrupts hard-disabled.
 *
 * Input Registers:
 *
 * LR = return address to continue at after eventually re-enabling MMU
 */
48
_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
49 50 51
	mflr	r0
	std	r0, PPC_LR_STKOFF(r1)
	stdu	r1, -112(r1)
52
	mfmsr	r10
53
	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
54 55 56 57 58 59 60 61 62
	li	r0,MSR_RI
	andc	r0,r10,r0
	li	r6,MSR_IR | MSR_DR
	andc	r6,r10,r6
	mtmsrd	r0,1		/* clear RI in MSR */
	mtsrr0	r5
	mtsrr1	r6
	RFI

63
kvmppc_call_hv_entry:
64
	ld	r4, HSTATE_KVM_VCPU(r13)
65 66 67 68
	bl	kvmppc_hv_entry

	/* Back from guest - restore host state and return to caller */

69
BEGIN_FTR_SECTION
70 71 72 73 74
	/* Restore host DABR and DABRX */
	ld	r5,HSTATE_DABR(r13)
	li	r6,7
	mtspr	SPRN_DABR,r5
	mtspr	SPRN_DABRX,r6
75
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
76 77

	/* Restore SPRG3 */
S
Scott Wood 已提交
78 79
	ld	r3,PACA_SPRG_VDSO(r13)
	mtspr	SPRN_SPRG_VDSO_WRITE,r3
80 81 82 83 84 85

	/* Reload the host's PMU registers */
	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
	lbz	r4, LPPACA_PMCINUSE(r3)
	cmpwi	r4, 0
	beq	23f			/* skip if not */
86
BEGIN_FTR_SECTION
87
	ld	r3, HSTATE_MMCR0(r13)
88 89 90 91
	andi.	r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
	cmpwi	r4, MMCR0_PMAO
	beql	kvmppc_fix_pmao
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
92 93 94 95 96 97
	lwz	r3, HSTATE_PMC1(r13)
	lwz	r4, HSTATE_PMC2(r13)
	lwz	r5, HSTATE_PMC3(r13)
	lwz	r6, HSTATE_PMC4(r13)
	lwz	r8, HSTATE_PMC5(r13)
	lwz	r9, HSTATE_PMC6(r13)
98 99 100 101 102 103
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r4
	mtspr	SPRN_PMC3, r5
	mtspr	SPRN_PMC4, r6
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
104 105 106 107 108
	ld	r3, HSTATE_MMCR0(r13)
	ld	r4, HSTATE_MMCR1(r13)
	ld	r5, HSTATE_MMCRA(r13)
	ld	r6, HSTATE_SIAR(r13)
	ld	r7, HSTATE_SDAR(r13)
109 110
	mtspr	SPRN_MMCR1, r4
	mtspr	SPRN_MMCRA, r5
111 112 113
	mtspr	SPRN_SIAR, r6
	mtspr	SPRN_SDAR, r7
BEGIN_FTR_SECTION
114 115
	ld	r8, HSTATE_MMCR2(r13)
	ld	r9, HSTATE_SIER(r13)
116 117 118
	mtspr	SPRN_MMCR2, r8
	mtspr	SPRN_SIER, r9
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
119 120 121 122
	mtspr	SPRN_MMCR0, r3
	isync
23:

123 124 125 126 127 128 129 130 131
	/*
	 * Reload DEC.  HDEC interrupts were disabled when
	 * we reloaded the host's LPCR value.
	 */
	ld	r3, HSTATE_DECEXP(r13)
	mftb	r4
	subf	r4, r4, r3
	mtspr	SPRN_DEC, r4

132 133 134 135
	/* hwthread_req may have got set by cede or no vcpu, so clear it */
	li	r0, 0
	stb	r0, HSTATE_HWTHREAD_REQ(r13)

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
	/*
	 * For external and machine check interrupts, we need
	 * to call the Linux handler to process the interrupt.
	 * We do that by jumping to absolute address 0x500 for
	 * external interrupts, or the machine_check_fwnmi label
	 * for machine checks (since firmware might have patched
	 * the vector area at 0x200).  The [h]rfid at the end of the
	 * handler will return to the book3s_hv_interrupts.S code.
	 * For other interrupts we do the rfid to get back
	 * to the book3s_hv_interrupts.S code here.
	 */
	ld	r8, 112+PPC_LR_STKOFF(r1)
	addi	r1, r1, 112
	ld	r7, HSTATE_HOST_MSR(r13)

	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
	beq	11f
154 155
	cmpwi	r12, BOOK3S_INTERRUPT_H_DOORBELL
	beq 	15f	/* Invoke the H_DOORBELL handler */
156 157
	cmpwi	cr2, r12, BOOK3S_INTERRUPT_HMI
	beq	cr2, 14f			/* HMI check */
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175

	/* RFI into the highmem handler, or branch to interrupt handler */
	mfmsr	r6
	li	r0, MSR_RI
	andc	r6, r6, r0
	mtmsrd	r6, 1			/* Clear RI in MSR */
	mtsrr0	r8
	mtsrr1	r7
	beq	cr1, 13f		/* machine check */
	RFI

	/* On POWER7, we have external interrupts set to use HSRR0/1 */
11:	mtspr	SPRN_HSRR0, r8
	mtspr	SPRN_HSRR1, r7
	ba	0x500

13:	b	machine_check_fwnmi

176 177 178 179
14:	mtspr	SPRN_HSRR0, r8
	mtspr	SPRN_HSRR1, r7
	b	hmi_exception_after_realmode

180 181 182 183
15:	mtspr SPRN_HSRR0, r8
	mtspr SPRN_HSRR1, r7
	ba    0xe80

184 185
kvmppc_primary_no_guest:
	/* We handle this much like a ceded vcpu */
186 187 188
	/* put the HDEC into the DEC, since HDEC interrupts don't wake us */
	mfspr	r3, SPRN_HDEC
	mtspr	SPRN_DEC, r3
189 190 191 192 193 194 195 196 197 198 199 200 201
	/*
	 * Make sure the primary has finished the MMU switch.
	 * We should never get here on a secondary thread, but
	 * check it for robustness' sake.
	 */
	ld	r5, HSTATE_KVM_VCORE(r13)
65:	lbz	r0, VCORE_IN_GUEST(r5)
	cmpwi	r0, 0
	beq	65b
	/* Set LPCR. */
	ld	r8,VCORE_LPCR(r5)
	mtspr	SPRN_LPCR,r8
	isync
202 203 204 205 206 207 208 209 210 211
	/* set our bit in napping_threads */
	ld	r5, HSTATE_KVM_VCORE(r13)
	lbz	r7, HSTATE_PTID(r13)
	li	r0, 1
	sld	r0, r0, r7
	addi	r6, r5, VCORE_NAPPING_THREADS
1:	lwarx	r3, 0, r6
	or	r3, r3, r0
	stwcx.	r3, 0, r6
	bne	1b
212
	/* order napping_threads update vs testing entry_exit_map */
213 214 215 216 217 218 219 220
	isync
	li	r12, 0
	lwz	r7, VCORE_ENTRY_EXIT(r5)
	cmpwi	r7, 0x100
	bge	kvm_novcpu_exit	/* another thread already exiting */
	li	r3, NAPPING_NOVCPU
	stb	r3, HSTATE_NAPPING(r13)

221
	li	r3, 0		/* Don't wake on privileged (OS) doorbell */
222 223
	b	kvm_do_nap

224 225 226 227 228 229 230
/*
 * kvm_novcpu_wakeup
 *	Entered from kvm_start_guest if kvm_hstate.napping is set
 *	to NAPPING_NOVCPU
 *		r2 = kernel TOC
 *		r13 = paca
 */
231 232 233 234 235 236
kvm_novcpu_wakeup:
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r5, HSTATE_KVM_VCORE(r13)
	li	r0, 0
	stb	r0, HSTATE_NAPPING(r13)

237 238
	/* check the wake reason */
	bl	kvmppc_check_wake_reason
239

240 241 242 243 244 245 246
	/*
	 * Restore volatile registers since we could have called
	 * a C routine in kvmppc_check_wake_reason.
	 *	r5 = VCORE
	 */
	ld	r5, HSTATE_KVM_VCORE(r13)

247 248 249 250 251 252 253 254 255 256
	/* see if any other thread is already exiting */
	lwz	r0, VCORE_ENTRY_EXIT(r5)
	cmpwi	r0, 0x100
	bge	kvm_novcpu_exit

	/* clear our bit in napping_threads */
	lbz	r7, HSTATE_PTID(r13)
	li	r0, 1
	sld	r0, r0, r7
	addi	r6, r5, VCORE_NAPPING_THREADS
257 258 259
4:	lwarx	r7, 0, r6
	andc	r7, r7, r0
	stwcx.	r7, 0, r6
260 261
	bne	4b

262
	/* See if the wake reason means we need to exit */
263 264 265
	cmpdi	r3, 0
	bge	kvm_novcpu_exit

266 267 268 269 270 271
	/* See if our timeslice has expired (HDEC is negative) */
	mfspr	r0, SPRN_HDEC
	li	r12, BOOK3S_INTERRUPT_HV_DECREMENTER
	cmpwi	r0, 0
	blt	kvm_novcpu_exit

272 273 274
	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
	ld	r4, HSTATE_KVM_VCPU(r13)
	cmpdi	r4, 0
275 276 277 278 279 280 281
	beq	kvmppc_primary_no_guest

#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	addi	r3, r4, VCPU_TB_RMENTRY
	bl	kvmhv_start_timing
#endif
	b	kvmppc_got_guest
282 283

kvm_novcpu_exit:
284 285 286 287 288 289 290
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	ld	r4, HSTATE_KVM_VCPU(r13)
	cmpdi	r4, 0
	beq	13f
	addi	r3, r4, VCPU_TB_RMEXIT
	bl	kvmhv_accumulate_time
#endif
291 292 293 294 295
13:	mr	r3, r12
	stw	r12, 112-4(r1)
	bl	kvmhv_commence_exit
	nop
	lwz	r12, 112-4(r1)
296
	b	kvmhv_switch_to_host
297

298
/*
299
 * We come in here when wakened from nap mode.
300 301 302 303 304
 * Relocation is off and most register values are lost.
 * r13 points to the PACA.
 */
	.globl	kvm_start_guest
kvm_start_guest:
305 306

	/* Set runlatch bit the minute you wake up from nap */
307 308 309
	mfspr	r0, SPRN_CTRLF
	ori 	r0, r0, 1
	mtspr	SPRN_CTRLT, r0
310

311 312
	ld	r2,PACATOC(r13)

313 314
	li	r0,KVM_HWTHREAD_IN_KVM
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
315

316 317 318
	/* NV GPR values from power7_idle() will no longer be valid */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)
319

320 321
	/* were we napping due to cede? */
	lbz	r0,HSTATE_NAPPING(r13)
322 323 324 325 326 327 328
	cmpwi	r0,NAPPING_CEDE
	beq	kvm_end_cede
	cmpwi	r0,NAPPING_NOVCPU
	beq	kvm_novcpu_wakeup

	ld	r1,PACAEMERGSP(r13)
	subi	r1,r1,STACK_FRAME_OVERHEAD
329 330 331 332 333 334 335

	/*
	 * We weren't napping due to cede, so this must be a secondary
	 * thread being woken up to run a guest, or being woken up due
	 * to a stray IPI.  (Or due to some machine check or hypervisor
	 * maintenance interrupt while the core is in KVM.)
	 */
336 337

	/* Check the wake reason in SRR1 to see why we got here */
338
	bl	kvmppc_check_wake_reason
339 340 341 342 343
	/*
	 * kvmppc_check_wake_reason could invoke a C routine, but we
	 * have no volatile registers to restore when we return.
	 */

344 345
	cmpdi	r3, 0
	bge	kvm_no_guest
346

347 348 349 350
	/* get vcore pointer, NULL if we have nothing to run */
	ld	r5,HSTATE_KVM_VCORE(r13)
	cmpdi	r5,0
	/* if we have no vcore to run, go back to sleep */
351
	beq	kvm_no_guest
352

353 354
kvm_secondary_got_guest:

355
	/* Set HSTATE_DSCR(r13) to something sensible */
356
	ld	r6, PACA_DSCR_DEFAULT(r13)
357
	std	r6, HSTATE_DSCR(r13)
358

359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
	/* On thread 0 of a subcore, set HDEC to max */
	lbz	r4, HSTATE_PTID(r13)
	cmpwi	r4, 0
	bne	63f
	lis	r6, 0x7fff
	ori	r6, r6, 0xffff
	mtspr	SPRN_HDEC, r6
	/* and set per-LPAR registers, if doing dynamic micro-threading */
	ld	r6, HSTATE_SPLIT_MODE(r13)
	cmpdi	r6, 0
	beq	63f
	ld	r0, KVM_SPLIT_RPR(r6)
	mtspr	SPRN_RPR, r0
	ld	r0, KVM_SPLIT_PMMAR(r6)
	mtspr	SPRN_PMMAR, r0
	ld	r0, KVM_SPLIT_LDBAR(r6)
	mtspr	SPRN_LDBAR, r0
	isync
63:
	/* Order load of vcpu after load of vcore */
379
	lwsync
380
	ld	r4, HSTATE_KVM_VCPU(r13)
381
	bl	kvmppc_hv_entry
382 383

	/* Back from the guest, go back to nap */
384
	/* Clear our vcpu and vcore pointers so we don't come back in early */
385
	li	r0, 0
386
	std	r0, HSTATE_KVM_VCPU(r13)
387
	/*
388
	 * Once we clear HSTATE_KVM_VCORE(r13), the code in
389 390 391
	 * kvmppc_run_core() is going to assume that all our vcpu
	 * state is visible in memory.  This lwsync makes sure
	 * that that is true.
392
	 */
393
	lwsync
394
	std	r0, HSTATE_KVM_VCORE(r13)
395

396 397 398 399 400 401 402 403 404 405 406 407
	/*
	 * All secondaries exiting guest will fall through this path.
	 * Before proceeding, just check for HMI interrupt and
	 * invoke opal hmi handler. By now we are sure that the
	 * primary thread on this core/subcore has already made partition
	 * switch/TB resync and we are good to call opal hmi handler.
	 */
	cmpwi	r12, BOOK3S_INTERRUPT_HMI
	bne	kvm_no_guest

	li	r3,0			/* NULL argument */
	bl	hmi_exception_realmode
408 409 410 411 412 413
/*
 * At this point we have finished executing in the guest.
 * We need to wait for hwthread_req to become zero, since
 * we may not turn on the MMU while hwthread_req is non-zero.
 * While waiting we also need to check if we get given a vcpu to run.
 */
414
kvm_no_guest:
415 416 417 418 419
	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r3, 0
	bne	53f
	HMT_MEDIUM
	li	r0, KVM_HWTHREAD_IN_KERNEL
420
	stb	r0, HSTATE_HWTHREAD_STATE(r13)
421 422 423 424 425 426
	/* need to recheck hwthread_req after a barrier, to avoid race */
	sync
	lbz	r3, HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r3, 0
	bne	54f
/*
427
 * We jump to pnv_wakeup_loss, which will return to the caller
428 429 430
 * of power7_nap in the powernv cpu offline loop.  The value we
 * put in r3 becomes the return value for power7_nap.
 */
431 432 433 434
	li	r3, LPCR_PECE0
	mfspr	r4, SPRN_LPCR
	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
	mtspr	SPRN_LPCR, r4
435
	li	r3, 0
436
	b	pnv_wakeup_loss
437 438

53:	HMT_LOW
439 440 441 442 443 444 445 446
	ld	r5, HSTATE_KVM_VCORE(r13)
	cmpdi	r5, 0
	bne	60f
	ld	r3, HSTATE_SPLIT_MODE(r13)
	cmpdi	r3, 0
	beq	kvm_no_guest
	lbz	r0, KVM_SPLIT_DO_NAP(r3)
	cmpwi	r0, 0
447 448
	beq	kvm_no_guest
	HMT_MEDIUM
449 450
	b	kvm_unsplit_nap
60:	HMT_MEDIUM
451 452 453 454 455
	b	kvm_secondary_got_guest

54:	li	r0, KVM_HWTHREAD_IN_KVM
	stb	r0, HSTATE_HWTHREAD_STATE(r13)
	b	kvm_no_guest
456

457 458 459 460 461
/*
 * Here the primary thread is trying to return the core to
 * whole-core mode, so we need to nap.
 */
kvm_unsplit_nap:
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
	/*
	 * When secondaries are napping in kvm_unsplit_nap() with
	 * hwthread_req = 1, HMI goes ignored even though subcores are
	 * already exited the guest. Hence HMI keeps waking up secondaries
	 * from nap in a loop and secondaries always go back to nap since
	 * no vcore is assigned to them. This makes impossible for primary
	 * thread to get hold of secondary threads resulting into a soft
	 * lockup in KVM path.
	 *
	 * Let us check if HMI is pending and handle it before we go to nap.
	 */
	cmpwi	r12, BOOK3S_INTERRUPT_HMI
	bne	55f
	li	r3, 0			/* NULL argument */
	bl	hmi_exception_realmode
55:
478 479 480 481 482 483 484 485
	/*
	 * Ensure that secondary doesn't nap when it has
	 * its vcore pointer set.
	 */
	sync		/* matches smp_mb() before setting split_info.do_nap */
	ld	r0, HSTATE_KVM_VCORE(r13)
	cmpdi	r0, 0
	bne	kvm_no_guest
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
	/* clear any pending message */
BEGIN_FTR_SECTION
	lis	r6, (PPC_DBELL_SERVER << (63-36))@h
	PPC_MSGCLR(6)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
	/* Set kvm_split_mode.napped[tid] = 1 */
	ld	r3, HSTATE_SPLIT_MODE(r13)
	li	r0, 1
	lhz	r4, PACAPACAINDEX(r13)
	clrldi	r4, r4, 61	/* micro-threading => P8 => 8 threads/core */
	addi	r4, r4, KVM_SPLIT_NAPPED
	stbx	r0, r3, r4
	/* Check the do_nap flag again after setting napped[] */
	sync
	lbz	r0, KVM_SPLIT_DO_NAP(r3)
	cmpwi	r0, 0
	beq	57f
	li	r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
	mfspr	r4, SPRN_LPCR
	rlwimi	r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
	mtspr	SPRN_LPCR, r4
	isync
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

57:	li	r0, 0
	stbx	r0, r3, r4
	b	kvm_no_guest

520 521 522 523 524 525
/******************************************************************************
 *                                                                            *
 *                               Entry code                                   *
 *                                                                            *
 *****************************************************************************/

526 527 528 529 530
.global kvmppc_hv_entry
kvmppc_hv_entry:

	/* Required state:
	 *
531
	 * R4 = vcpu pointer (or NULL)
532 533 534
	 * MSR = ~IR|DR
	 * R13 = PACA
	 * R1 = host R1
535
	 * R2 = TOC
536 537 538
	 * all other volatile GPRS = free
	 */
	mflr	r0
539 540
	std	r0, PPC_LR_STKOFF(r1)
	stdu	r1, -112(r1)
541 542 543 544

	/* Save R1 in the PACA */
	std	r1, HSTATE_HOST_R1(r13)

545 546 547
	li	r6, KVM_GUEST_MODE_HOST_HV
	stb	r6, HSTATE_IN_GUEST(r13)

548 549 550 551 552 553 554 555
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	/* Store initial timestamp */
	cmpdi	r4, 0
	beq	1f
	addi	r3, r4, VCPU_TB_RMENTRY
	bl	kvmhv_start_timing
1:
#endif
556 557 558 559 560 561
	/* Clear out SLB */
	li	r6,0
	slbmte	r6,r6
	slbia
	ptesync

562
	/*
563
	 * POWER7/POWER8 host -> guest partition switch code.
564 565 566
	 * We don't have to lock against concurrent tlbies,
	 * but we do have to coordinate across hardware threads.
	 */
567 568 569 570 571 572 573 574
	/* Set bit in entry map iff exit map is zero. */
	ld	r5, HSTATE_KVM_VCORE(r13)
	li	r7, 1
	lbz	r6, HSTATE_PTID(r13)
	sld	r7, r7, r6
	addi	r9, r5, VCORE_ENTRY_EXIT
21:	lwarx	r3, 0, r9
	cmpwi	r3, 0x100		/* any threads starting to exit? */
575
	bge	secondary_too_late	/* if so we're too late to the party */
576 577
	or	r3, r3, r7
	stwcx.	r3, 0, r9
578 579 580
	bne	21b

	/* Primary thread switches to guest partition. */
581
	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
582
	cmpwi	r6,0
583
	bne	10f
584 585 586 587 588 589 590 591
	ld	r6,KVM_SDR1(r9)
	lwz	r7,KVM_LPID(r9)
	li	r0,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r0
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync
592 593 594 595 596 597 598 599

	/* See if we need to flush the TLB */
	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
	srdi	r6,r6,6			/* doubleword number */
	sldi	r6,r6,3			/* address offset */
	add	r6,r6,r9
	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
600
	li	r0,1
601 602 603 604 605 606 607 608
	sld	r0,r0,r7
	ld	r7,0(r6)
	and.	r7,r7,r0
	beq	22f
23:	ldarx	r7,0,r6			/* if set, clear the bit */
	andc	r7,r7,r0
	stdcx.	r7,0,r6
	bne	23b
609 610 611 612 613 614 615
	/* Flush the TLB of any entries for this LPID */
	/* use arch 2.07S as a proxy for POWER8 */
BEGIN_FTR_SECTION
	li	r6,512			/* POWER8 has 512 sets */
FTR_SECTION_ELSE
	li	r6,128			/* POWER7 has 128 sets */
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
616 617 618 619 620 621 622 623
	mtctr	r6
	li	r7,0x800		/* IS field = 0b10 */
	ptesync
28:	tlbiel	r7
	addi	r7,r7,0x1000
	bdnz	28b
	ptesync

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
	/* Add timebase offset onto timebase */
22:	ld	r8,VCORE_TB_OFFSET(r5)
	cmpdi	r8,0
	beq	37f
	mftb	r6		/* current host timebase */
	add	r8,r8,r6
	mtspr	SPRN_TBU40,r8	/* update upper 40 bits */
	mftb	r7		/* check if lower 24 bits overflowed */
	clrldi	r6,r6,40
	clrldi	r7,r7,40
	cmpld	r7,r6
	bge	37f
	addis	r8,r8,0x100	/* if so, increment upper 40 bits */
	mtspr	SPRN_TBU40,r8

639 640 641 642 643 644
	/* Load guest PCR value to select appropriate compat mode */
37:	ld	r7, VCORE_PCR(r5)
	cmpdi	r7, 0
	beq	38f
	mtspr	SPRN_PCR, r7
38:
645 646 647 648 649 650 651

BEGIN_FTR_SECTION
	/* DPDES is shared between threads */
	ld	r8, VCORE_DPDES(r5)
	mtspr	SPRN_DPDES, r8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)

652 653 654 655 656
	/* Mark the subcore state as inside guest */
	bl	kvmppc_subcore_enter_guest
	nop
	ld	r5, HSTATE_KVM_VCORE(r13)
	ld	r4, HSTATE_KVM_VCPU(r13)
657
	li	r0,1
658
	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
659

660
	/* Do we have a guest vcpu to run? */
661
10:	cmpdi	r4, 0
662 663
	beq	kvmppc_primary_no_guest
kvmppc_got_guest:
664 665

	/* Load up guest SLB entries */
666
	lwz	r5,VCPU_SLB_MAX(r4)
667 668 669 670 671 672 673 674 675 676
	cmpwi	r5,0
	beq	9f
	mtctr	r5
	addi	r6,r4,VCPU_SLB
1:	ld	r8,VCPU_SLB_E(r6)
	ld	r9,VCPU_SLB_V(r6)
	slbmte	r9,r8
	addi	r6,r6,VCPU_SLB_SIZE
	bdnz	1b
9:
677 678 679 680
	/* Increment yield count if they have a VPA */
	ld	r3, VCPU_VPA(r4)
	cmpdi	r3, 0
	beq	25f
681 682
	li	r6, LPPACA_YIELDCOUNT
	LWZX_BE	r5, r3, r6
683
	addi	r5, r5, 1
684
	STWX_BE	r5, r3, r6
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
	li	r6, 1
	stb	r6, VCPU_VPA_DIRTY(r4)
25:

	/* Save purr/spurr */
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	std	r5,HSTATE_PURR(r13)
	std	r6,HSTATE_SPURR(r13)
	ld	r7,VCPU_PURR(r4)
	ld	r8,VCPU_SPURR(r4)
	mtspr	SPRN_PURR,r7
	mtspr	SPRN_SPURR,r8

BEGIN_FTR_SECTION
	/* Set partition DABR */
	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
702
	lwz	r5,VCPU_DABRX(r4)
703 704 705 706 707 708
	ld	r6,VCPU_DABR(r4)
	mtspr	SPRN_DABRX,r5
	mtspr	SPRN_DABR,r6
	isync
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)

709 710
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
711 712
	bl	kvmppc_restore_tm
END_FTR_SECTION_IFSET(CPU_FTR_TM)
713 714
#endif

715 716 717 718 719 720
	/* Load guest PMU registers */
	/* R4 is live here (vcpu pointer) */
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	isync
721 722 723 724 725 726
BEGIN_FTR_SECTION
	ld	r3, VCPU_MMCR(r4)
	andi.	r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
	cmpwi	r5, MMCR0_PMAO
	beql	kvmppc_fix_pmao
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
	lwz	r6, VCPU_PMC + 8(r4)
	lwz	r7, VCPU_PMC + 12(r4)
	lwz	r8, VCPU_PMC + 16(r4)
	lwz	r9, VCPU_PMC + 20(r4)
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r5
	mtspr	SPRN_PMC3, r6
	mtspr	SPRN_PMC4, r7
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
	ld	r3, VCPU_MMCR(r4)
	ld	r5, VCPU_MMCR + 8(r4)
	ld	r6, VCPU_MMCR + 16(r4)
	ld	r7, VCPU_SIAR(r4)
	ld	r8, VCPU_SDAR(r4)
	mtspr	SPRN_MMCR1, r5
	mtspr	SPRN_MMCRA, r6
	mtspr	SPRN_SIAR, r7
	mtspr	SPRN_SDAR, r8
748 749 750 751 752 753 754 755 756 757 758 759
BEGIN_FTR_SECTION
	ld	r5, VCPU_MMCR + 24(r4)
	ld	r6, VCPU_SIER(r4)
	lwz	r7, VCPU_PMC + 24(r4)
	lwz	r8, VCPU_PMC + 28(r4)
	ld	r9, VCPU_MMCR + 32(r4)
	mtspr	SPRN_MMCR2, r5
	mtspr	SPRN_SIER, r6
	mtspr	SPRN_SPMC1, r7
	mtspr	SPRN_SPMC2, r8
	mtspr	SPRN_MMCRS, r9
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
	mtspr	SPRN_MMCR0, r3
	isync

	/* Load up FP, VMX and VSX registers */
	bl	kvmppc_load_fp

	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)

	/* Switch DSCR to guest value */
	ld	r5, VCPU_DSCR(r4)
	mtspr	SPRN_DSCR, r5

789
BEGIN_FTR_SECTION
790
	/* Skip next section on POWER7 */
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
	b	8f
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
	/* Load up POWER8-specific registers */
	ld	r5, VCPU_IAMR(r4)
	lwz	r6, VCPU_PSPB(r4)
	ld	r7, VCPU_FSCR(r4)
	mtspr	SPRN_IAMR, r5
	mtspr	SPRN_PSPB, r6
	mtspr	SPRN_FSCR, r7
	ld	r5, VCPU_DAWR(r4)
	ld	r6, VCPU_DAWRX(r4)
	ld	r7, VCPU_CIABR(r4)
	ld	r8, VCPU_TAR(r4)
	mtspr	SPRN_DAWR, r5
	mtspr	SPRN_DAWRX, r6
	mtspr	SPRN_CIABR, r7
	mtspr	SPRN_TAR, r8
	ld	r5, VCPU_IC(r4)
	ld	r6, VCPU_VTB(r4)
	mtspr	SPRN_IC, r5
	mtspr	SPRN_VTB, r6
812
	ld	r8, VCPU_EBBHR(r4)
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
	mtspr	SPRN_EBBHR, r8
	ld	r5, VCPU_EBBRR(r4)
	ld	r6, VCPU_BESCR(r4)
	ld	r7, VCPU_CSIGR(r4)
	ld	r8, VCPU_TACR(r4)
	mtspr	SPRN_EBBRR, r5
	mtspr	SPRN_BESCR, r6
	mtspr	SPRN_CSIGR, r7
	mtspr	SPRN_TACR, r8
	ld	r5, VCPU_TCSCR(r4)
	ld	r6, VCPU_ACOP(r4)
	lwz	r7, VCPU_GUEST_PID(r4)
	ld	r8, VCPU_WORT(r4)
	mtspr	SPRN_TCSCR, r5
	mtspr	SPRN_ACOP, r6
	mtspr	SPRN_PID, r7
	mtspr	SPRN_WORT, r8
8:

832 833 834 835
	/*
	 * Set the decrementer to the guest decrementer.
	 */
	ld	r8,VCPU_DEC_EXPIRES(r4)
836 837 838 839
	/* r8 is a host timebase value here, convert to guest TB */
	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r6,VCORE_TB_OFFSET(r5)
	add	r8,r8,r6
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
	mftb	r7
	subf	r3,r7,r8
	mtspr	SPRN_DEC,r3
	stw	r3,VCPU_DEC(r4)

	ld	r5, VCPU_SPRG0(r4)
	ld	r6, VCPU_SPRG1(r4)
	ld	r7, VCPU_SPRG2(r4)
	ld	r8, VCPU_SPRG3(r4)
	mtspr	SPRN_SPRG0, r5
	mtspr	SPRN_SPRG1, r6
	mtspr	SPRN_SPRG2, r7
	mtspr	SPRN_SPRG3, r8

	/* Load up DAR and DSISR */
	ld	r5, VCPU_DAR(r4)
	lwz	r6, VCPU_DSISR(r4)
	mtspr	SPRN_DAR, r5
	mtspr	SPRN_DSISR, r6

	/* Restore AMR and UAMOR, set AMOR to all 1s */
	ld	r5,VCPU_AMR(r4)
	ld	r6,VCPU_UAMOR(r4)
	li	r7,-1
	mtspr	SPRN_AMR,r5
	mtspr	SPRN_UAMOR,r6
	mtspr	SPRN_AMOR,r7
867 868 869 870 871 872 873 874 875

	/* Restore state of CTRL run bit; assume 1 on entry */
	lwz	r5,VCPU_CTRL(r4)
	andi.	r5,r5,1
	bne	4f
	mfspr	r6,SPRN_CTRLF
	clrrdi	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
876 877 878 879 880 881 882 883 884
	/* Secondary threads wait for primary to have done partition switch */
	ld	r5, HSTATE_KVM_VCORE(r13)
	lbz	r6, HSTATE_PTID(r13)
	cmpwi	r6, 0
	beq	21f
	lbz	r0, VCORE_IN_GUEST(r5)
	cmpwi	r0, 0
	bne	21f
	HMT_LOW
885 886 887 888
20:	lwz	r3, VCORE_ENTRY_EXIT(r5)
	cmpwi	r3, 0x100
	bge	no_switch_exit
	lbz	r0, VCORE_IN_GUEST(r5)
889 890 891 892 893 894 895 896 897 898 899 900 901 902
	cmpwi	r0, 0
	beq	20b
	HMT_MEDIUM
21:
	/* Set LPCR. */
	ld	r8,VCORE_LPCR(r5)
	mtspr	SPRN_LPCR,r8
	isync

	/* Check if HDEC expires soon */
	mfspr	r3, SPRN_HDEC
	cmpwi	r3, 512		/* 1 microsecond */
	blt	hdec_soon

903
deliver_guest_interrupt:
904
	ld	r6, VCPU_CTR(r4)
905
	ld	r7, VCPU_XER(r4)
906 907 908 909

	mtctr	r6
	mtxer	r7

910
kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
911 912
	ld	r10, VCPU_PC(r4)
	ld	r11, VCPU_MSR(r4)
913 914
	ld	r6, VCPU_SRR0(r4)
	ld	r7, VCPU_SRR1(r4)
915 916
	mtspr	SPRN_SRR0, r6
	mtspr	SPRN_SRR1, r7
917

918
	/* r11 = vcpu->arch.msr & ~MSR_HV */
919 920 921 922
	rldicl	r11, r11, 63 - MSR_HV_LG, 1
	rotldi	r11, r11, 1 + MSR_HV_LG
	ori	r11, r11, MSR_ME

923
	/* Check if we can deliver an external or decrementer interrupt now */
924 925 926 927 928 929 930 931
	ld	r0, VCPU_PENDING_EXC(r4)
	rldicl	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
	cmpdi	cr1, r0, 0
	andi.	r8, r11, MSR_EE
	mfspr	r8, SPRN_LPCR
	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
	mtspr	SPRN_LPCR, r8
932 933
	isync
	beq	5f
934 935 936 937 938 939
	li	r0, BOOK3S_INTERRUPT_EXTERNAL
	bne	cr1, 12f
	mfspr	r0, SPRN_DEC
	cmpwi	r0, 0
	li	r0, BOOK3S_INTERRUPT_DECREMENTER
	bge	5f
940

941
12:	mtspr	SPRN_SRR0, r10
942
	mr	r10,r0
943
	mtspr	SPRN_SRR1, r11
944 945
	mr	r9, r4
	bl	kvmppc_msr_interrupt
946
5:
947

948 949 950 951 952 953 954
/*
 * Required state:
 * R4 = vcpu
 * R10: value for HSRR0
 * R11: value for HSRR1
 * R13 = PACA
 */
955
fast_guest_return:
956 957
	li	r0,0
	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
958 959 960 961
	mtspr	SPRN_HSRR0,r10
	mtspr	SPRN_HSRR1,r11

	/* Activate guest mode, so faults get handled by KVM */
962
	li	r9, KVM_GUEST_MODE_GUEST_HV
963 964
	stb	r9, HSTATE_IN_GUEST(r13)

965 966 967 968 969 970
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	/* Accumulate timing */
	addi	r3, r4, VCPU_TB_GUEST
	bl	kvmhv_accumulate_time
#endif

971 972
	/* Enter guest */

973 974 975 976
BEGIN_FTR_SECTION
	ld	r5, VCPU_CFAR(r4)
	mtspr	SPRN_CFAR, r5
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
977 978 979
BEGIN_FTR_SECTION
	ld	r0, VCPU_PPR(r4)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
980

981 982 983 984 985
	ld	r5, VCPU_LR(r4)
	lwz	r6, VCPU_CR(r4)
	mtlr	r5
	mtcr	r6

986 987 988 989 990 991 992 993 994 995 996 997 998
	ld	r1, VCPU_GPR(R1)(r4)
	ld	r2, VCPU_GPR(R2)(r4)
	ld	r3, VCPU_GPR(R3)(r4)
	ld	r5, VCPU_GPR(R5)(r4)
	ld	r6, VCPU_GPR(R6)(r4)
	ld	r7, VCPU_GPR(R7)(r4)
	ld	r8, VCPU_GPR(R8)(r4)
	ld	r9, VCPU_GPR(R9)(r4)
	ld	r10, VCPU_GPR(R10)(r4)
	ld	r11, VCPU_GPR(R11)(r4)
	ld	r12, VCPU_GPR(R12)(r4)
	ld	r13, VCPU_GPR(R13)(r4)

999 1000 1001 1002
BEGIN_FTR_SECTION
	mtspr	SPRN_PPR, r0
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
	ld	r0, VCPU_GPR(R0)(r4)
1003
	ld	r4, VCPU_GPR(R4)(r4)
1004 1005 1006 1007

	hrfid
	b	.

1008
secondary_too_late:
1009
	li	r12, 0
1010 1011
	cmpdi	r4, 0
	beq	11f
1012 1013
	stw	r12, VCPU_TRAP(r4)
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1014 1015
	addi	r3, r4, VCPU_TB_RMEXIT
	bl	kvmhv_accumulate_time
1016
#endif
1017 1018
11:	b	kvmhv_switch_to_host

1019 1020 1021 1022
no_switch_exit:
	HMT_MEDIUM
	li	r12, 0
	b	12f
1023
hdec_soon:
1024
	li	r12, BOOK3S_INTERRUPT_HV_DECREMENTER
1025
12:	stw	r12, VCPU_TRAP(r4)
1026 1027
	mr	r9, r4
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1028 1029 1030
	addi	r3, r4, VCPU_TB_RMEXIT
	bl	kvmhv_accumulate_time
#endif
1031
	b	guest_exit_cont
1032

1033 1034 1035 1036 1037 1038 1039 1040 1041
/******************************************************************************
 *                                                                            *
 *                               Exit code                                    *
 *                                                                            *
 *****************************************************************************/

/*
 * We come here from the first-level interrupt handlers.
 */
1042 1043
	.globl	kvmppc_interrupt_hv
kvmppc_interrupt_hv:
1044 1045 1046 1047 1048 1049 1050
	/*
	 * Register contents:
	 * R12		= interrupt vector
	 * R13		= PACA
	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
	 * guest R13 saved in SPRN_SCRATCH0
	 */
1051
	std	r9, HSTATE_SCRATCH2(r13)
1052 1053 1054 1055

	lbz	r9, HSTATE_IN_GUEST(r13)
	cmpwi	r9, KVM_GUEST_MODE_HOST_HV
	beq	kvmppc_bad_host_intr
1056 1057
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
	cmpwi	r9, KVM_GUEST_MODE_GUEST
1058
	ld	r9, HSTATE_SCRATCH2(r13)
1059 1060
	beq	kvmppc_interrupt_pr
#endif
1061 1062 1063 1064
	/* We're now back in the host but in guest MMU context */
	li	r9, KVM_GUEST_MODE_HOST_HV
	stb	r9, HSTATE_IN_GUEST(r13)

1065 1066 1067 1068
	ld	r9, HSTATE_KVM_VCPU(r13)

	/* Save registers */

1069 1070 1071 1072 1073 1074 1075 1076 1077
	std	r0, VCPU_GPR(R0)(r9)
	std	r1, VCPU_GPR(R1)(r9)
	std	r2, VCPU_GPR(R2)(r9)
	std	r3, VCPU_GPR(R3)(r9)
	std	r4, VCPU_GPR(R4)(r9)
	std	r5, VCPU_GPR(R5)(r9)
	std	r6, VCPU_GPR(R6)(r9)
	std	r7, VCPU_GPR(R7)(r9)
	std	r8, VCPU_GPR(R8)(r9)
1078
	ld	r0, HSTATE_SCRATCH2(r13)
1079 1080 1081
	std	r0, VCPU_GPR(R9)(r9)
	std	r10, VCPU_GPR(R10)(r9)
	std	r11, VCPU_GPR(R11)(r9)
1082 1083
	ld	r3, HSTATE_SCRATCH0(r13)
	lwz	r4, HSTATE_SCRATCH1(r13)
1084
	std	r3, VCPU_GPR(R12)(r9)
1085
	stw	r4, VCPU_CR(r9)
1086 1087 1088 1089
BEGIN_FTR_SECTION
	ld	r3, HSTATE_CFAR(r13)
	std	r3, VCPU_CFAR(r9)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1090 1091 1092 1093
BEGIN_FTR_SECTION
	ld	r4, HSTATE_PPR(r13)
	std	r4, VCPU_PPR(r9)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112

	/* Restore R1/R2 so we can handle faults */
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATOC(r13)

	mfspr	r10, SPRN_SRR0
	mfspr	r11, SPRN_SRR1
	std	r10, VCPU_SRR0(r9)
	std	r11, VCPU_SRR1(r9)
	andi.	r0, r12, 2		/* need to read HSRR0/1? */
	beq	1f
	mfspr	r10, SPRN_HSRR0
	mfspr	r11, SPRN_HSRR1
	clrrdi	r12, r12, 2
1:	std	r10, VCPU_PC(r9)
	std	r11, VCPU_MSR(r9)

	GET_SCRATCH0(r3)
	mflr	r4
1113
	std	r3, VCPU_GPR(R13)(r9)
1114 1115 1116 1117
	std	r4, VCPU_LR(r9)

	stw	r12,VCPU_TRAP(r9)

1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	addi	r3, r9, VCPU_TB_RMINTR
	mr	r4, r9
	bl	kvmhv_accumulate_time
	ld	r5, VCPU_GPR(R5)(r9)
	ld	r6, VCPU_GPR(R6)(r9)
	ld	r7, VCPU_GPR(R7)(r9)
	ld	r8, VCPU_GPR(R8)(r9)
#endif

1128
	/* Save HEIR (HV emulation assist reg) in emul_inst
1129 1130
	   if this is an HEI (HV emulation interrupt, e40) */
	li	r3,KVM_INST_FETCH_FAILED
1131
	stw	r3,VCPU_LAST_INST(r9)
1132 1133 1134
	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
	bne	11f
	mfspr	r3,SPRN_HEIR
1135
11:	stw	r3,VCPU_HEIR(r9)
1136 1137 1138 1139 1140

	/* these are volatile across C function calls */
	mfctr	r3
	mfxer	r4
	std	r3, VCPU_CTR(r9)
1141
	std	r4, VCPU_XER(r9)
1142 1143 1144 1145

	/* If this is a page table miss then see if it's theirs or ours */
	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	beq	kvmppc_hdsi
1146 1147
	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	beq	kvmppc_hisi
1148

1149 1150 1151 1152 1153
	/* See if this is a leftover HDEC interrupt */
	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	bne	2f
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,0
1154 1155
	mr	r4,r9
	bge	fast_guest_return
1156
2:
1157
	/* See if this is an hcall we can handle in real mode */
1158 1159
	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
	beq	hcall_try_real_mode
1160

1161 1162 1163 1164
	/* Hypervisor doorbell - exit only if host IPI flag set */
	cmpwi	r12, BOOK3S_INTERRUPT_H_DOORBELL
	bne	3f
	lbz	r0, HSTATE_HOST_IPI(r13)
1165
	cmpwi	r0, 0
1166 1167 1168
	beq	4f
	b	guest_exit_cont
3:
1169 1170
	/* External interrupt ? */
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
1171
	bne+	guest_exit_cont
1172 1173 1174 1175

	/* External interrupt, first check for host_ipi. If this is
	 * set, we know the host wants us out so let's do it now
	 */
1176
	bl	kvmppc_read_intr
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202

	/*
	 * Restore the active volatile registers after returning from
	 * a C function.
	 */
	ld	r9, HSTATE_KVM_VCPU(r13)
	li	r12, BOOK3S_INTERRUPT_EXTERNAL

	/*
	 * kvmppc_read_intr return codes:
	 *
	 * Exit to host (r3 > 0)
	 *   1 An interrupt is pending that needs to be handled by the host
	 *     Exit guest and return to host by branching to guest_exit_cont
	 *
	 * Before returning to guest, we check if any CPU is heading out
	 * to the host and if so, we head out also. If no CPUs are heading
	 * check return values <= 0.
	 *
	 * Return to guest (r3 <= 0)
	 *  0 No external interrupt is pending
	 * -1 A guest wakeup IPI (which has now been cleared)
	 *    In either case, we return to guest to deliver any pending
	 *    guest interrupts.
	 */

1203
	cmpdi	r3, 0
1204
	bgt	guest_exit_cont
1205

1206
	/* Return code <= 0 */
1207
4:	ld	r5, HSTATE_KVM_VCORE(r13)
1208 1209
	lwz	r0, VCORE_ENTRY_EXIT(r5)
	cmpwi	r0, 0x100
1210
	mr	r4, r9
1211
	blt	deliver_guest_interrupt
1212

1213
guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
1214 1215 1216 1217 1218
	/* Save more register state  */
	mfdar	r6
	mfdsisr	r7
	std	r6, VCPU_DAR(r9)
	stw	r7, VCPU_DSISR(r9)
1219
	/* don't overwrite fault_dar/fault_dsisr if HDSI */
1220
	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1221
	beq	mc_cont
1222
	std	r6, VCPU_FAULT_DAR(r9)
1223 1224
	stw	r7, VCPU_FAULT_DSISR(r9)

1225 1226 1227 1228
	/* See if it is a machine check */
	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	machine_check_realmode
mc_cont:
1229 1230 1231 1232 1233
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	addi	r3, r9, VCPU_TB_RMEXIT
	mr	r4, r9
	bl	kvmhv_accumulate_time
#endif
1234

1235
	mr 	r3, r12
1236 1237
	/* Increment exit count, poke other threads to exit */
	bl	kvmhv_commence_exit
1238 1239 1240
	nop
	ld	r9, HSTATE_KVM_VCPU(r13)
	lwz	r12, VCPU_TRAP(r9)
1241

1242 1243 1244 1245 1246
	/* Stop others sending VCPU interrupts to this physical CPU */
	li	r0, -1
	stw	r0, VCPU_CPU(r9)
	stw	r0, VCPU_THREAD_CPU(r9)

1247
	/* Save guest CTRL register, set runlatch to 1 */
1248
	mfspr	r6,SPRN_CTRLF
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
	stw	r6,VCPU_CTRL(r9)
	andi.	r0,r6,1
	bne	4f
	ori	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
	/* Read the guest SLB and save it away */
	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
	mtctr	r0
	li	r6,0
	addi	r7,r9,VCPU_SLB
	li	r5,0
1:	slbmfee	r8,r6
	andis.	r0,r8,SLB_ESID_V@h
	beq	2f
	add	r8,r8,r6		/* put index in */
	slbmfev	r3,r6
	std	r8,VCPU_SLB_E(r7)
	std	r3,VCPU_SLB_V(r7)
	addi	r7,r7,VCPU_SLB_SIZE
	addi	r5,r5,1
2:	addi	r6,r6,1
	bdnz	1b
	stw	r5,VCPU_SLB_MAX(r9)

	/*
	 * Save the guest PURR/SPURR
	 */
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	ld	r7,VCPU_PURR(r9)
	ld	r8,VCPU_SPURR(r9)
	std	r5,VCPU_PURR(r9)
	std	r6,VCPU_SPURR(r9)
	subf	r5,r7,r5
	subf	r6,r8,r6

	/*
	 * Restore host PURR/SPURR and add guest times
	 * so that the time in the guest gets accounted.
	 */
	ld	r3,HSTATE_PURR(r13)
	ld	r4,HSTATE_SPURR(r13)
	add	r3,r3,r5
	add	r4,r4,r6
	mtspr	SPRN_PURR,r3
	mtspr	SPRN_SPURR,r4

1297 1298 1299 1300 1301
	/* Save DEC */
	mfspr	r5,SPRN_DEC
	mftb	r6
	extsw	r5,r5
	add	r5,r5,r6
1302 1303 1304 1305
	/* r5 is a guest timebase value here, convert to host TB */
	ld	r3,HSTATE_KVM_VCORE(r13)
	ld	r4,VCORE_TB_OFFSET(r3)
	subf	r5,r4,r5
1306 1307
	std	r5,VCPU_DEC_EXPIRES(r9)

1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
BEGIN_FTR_SECTION
	b	8f
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
	/* Save POWER8-specific registers */
	mfspr	r5, SPRN_IAMR
	mfspr	r6, SPRN_PSPB
	mfspr	r7, SPRN_FSCR
	std	r5, VCPU_IAMR(r9)
	stw	r6, VCPU_PSPB(r9)
	std	r7, VCPU_FSCR(r9)
	mfspr	r5, SPRN_IC
	mfspr	r6, SPRN_VTB
	mfspr	r7, SPRN_TAR
	std	r5, VCPU_IC(r9)
	std	r6, VCPU_VTB(r9)
	std	r7, VCPU_TAR(r9)
1324
	mfspr	r8, SPRN_EBBHR
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
	std	r8, VCPU_EBBHR(r9)
	mfspr	r5, SPRN_EBBRR
	mfspr	r6, SPRN_BESCR
	mfspr	r7, SPRN_CSIGR
	mfspr	r8, SPRN_TACR
	std	r5, VCPU_EBBRR(r9)
	std	r6, VCPU_BESCR(r9)
	std	r7, VCPU_CSIGR(r9)
	std	r8, VCPU_TACR(r9)
	mfspr	r5, SPRN_TCSCR
	mfspr	r6, SPRN_ACOP
	mfspr	r7, SPRN_PID
	mfspr	r8, SPRN_WORT
	std	r5, VCPU_TCSCR(r9)
	std	r6, VCPU_ACOP(r9)
	stw	r7, VCPU_GUEST_PID(r9)
	std	r8, VCPU_WORT(r9)
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
	/*
	 * Restore various registers to 0, where non-zero values
	 * set by the guest could disrupt the host.
	 */
	li	r0, 0
	mtspr	SPRN_IAMR, r0
	mtspr	SPRN_CIABR, r0
	mtspr	SPRN_DAWRX, r0
	mtspr	SPRN_TCSCR, r0
	mtspr	SPRN_WORT, r0
	/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
	li	r0, 1
	sldi	r0, r0, 31
	mtspr	SPRN_MMCRS, r0
1356 1357
8:

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
	/* Save and reset AMR and UAMOR before turning on the MMU */
	mfspr	r5,SPRN_AMR
	mfspr	r6,SPRN_UAMOR
	std	r5,VCPU_AMR(r9)
	std	r6,VCPU_UAMOR(r9)
	li	r6,0
	mtspr	SPRN_AMR,r6

	/* Switch DSCR back to host value */
	mfspr	r8, SPRN_DSCR
	ld	r7, HSTATE_DSCR(r13)
	std	r8, VCPU_DSCR(r9)
	mtspr	SPRN_DSCR, r7

	/* Save non-volatile GPRs */
	std	r14, VCPU_GPR(R14)(r9)
	std	r15, VCPU_GPR(R15)(r9)
	std	r16, VCPU_GPR(R16)(r9)
	std	r17, VCPU_GPR(R17)(r9)
	std	r18, VCPU_GPR(R18)(r9)
	std	r19, VCPU_GPR(R19)(r9)
	std	r20, VCPU_GPR(R20)(r9)
	std	r21, VCPU_GPR(R21)(r9)
	std	r22, VCPU_GPR(R22)(r9)
	std	r23, VCPU_GPR(R23)(r9)
	std	r24, VCPU_GPR(R24)(r9)
	std	r25, VCPU_GPR(R25)(r9)
	std	r26, VCPU_GPR(R26)(r9)
	std	r27, VCPU_GPR(R27)(r9)
	std	r28, VCPU_GPR(R28)(r9)
	std	r29, VCPU_GPR(R29)(r9)
	std	r30, VCPU_GPR(R30)(r9)
	std	r31, VCPU_GPR(R31)(r9)

	/* Save SPRGs */
	mfspr	r3, SPRN_SPRG0
	mfspr	r4, SPRN_SPRG1
	mfspr	r5, SPRN_SPRG2
	mfspr	r6, SPRN_SPRG3
	std	r3, VCPU_SPRG0(r9)
	std	r4, VCPU_SPRG1(r9)
	std	r5, VCPU_SPRG2(r9)
	std	r6, VCPU_SPRG3(r9)

	/* save FP state */
	mr	r3, r9
	bl	kvmppc_save_fp
1405

1406 1407
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
1408 1409
	bl	kvmppc_save_tm
END_FTR_SECTION_IFSET(CPU_FTR_TM)
1410 1411
#endif

1412 1413 1414 1415
	/* Increment yield count if they have a VPA */
	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
	cmpdi	r8, 0
	beq	25f
1416 1417
	li	r4, LPPACA_YIELDCOUNT
	LWZX_BE	r3, r8, r4
1418
	addi	r3, r3, 1
1419
	STWX_BE	r3, r8, r4
1420 1421 1422 1423 1424
	li	r3, 1
	stb	r3, VCPU_VPA_DIRTY(r9)
25:
	/* Save PMU registers if requested */
	/* r8 and cr0.eq are live here */
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
BEGIN_FTR_SECTION
	/*
	 * POWER8 seems to have a hardware bug where setting
	 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
	 * when some counters are already negative doesn't seem
	 * to cause a performance monitor alert (and hence interrupt).
	 * The effect of this is that when saving the PMU state,
	 * if there is no PMU alert pending when we read MMCR0
	 * before freezing the counters, but one becomes pending
	 * before we read the counters, we lose it.
	 * To work around this, we need a way to freeze the counters
	 * before reading MMCR0.  Normally, freezing the counters
	 * is done by writing MMCR0 (to set MMCR0[FC]) which
	 * unavoidably writes MMCR0[PMA0] as well.  On POWER8,
	 * we can also freeze the counters using MMCR2, by writing
	 * 1s to all the counter freeze condition bits (there are
	 * 9 bits each for 6 counters).
	 */
	li	r3, -1			/* set all freeze bits */
	clrrdi	r3, r3, 10
	mfspr	r10, SPRN_MMCR2
	mtspr	SPRN_MMCR2, r3
	isync
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1449 1450 1451 1452 1453
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	mfspr	r6, SPRN_MMCRA
1454
	/* Clear MMCRA in order to disable SDAR updates */
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469
	li	r7, 0
	mtspr	SPRN_MMCRA, r7
	isync
	beq	21f			/* if no VPA, save PMU stuff anyway */
	lbz	r7, LPPACA_PMCINUSE(r8)
	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
	bne	21f
	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
	b	22f
21:	mfspr	r5, SPRN_MMCR1
	mfspr	r7, SPRN_SIAR
	mfspr	r8, SPRN_SDAR
	std	r4, VCPU_MMCR(r9)
	std	r5, VCPU_MMCR + 8(r9)
	std	r6, VCPU_MMCR + 16(r9)
1470 1471 1472
BEGIN_FTR_SECTION
	std	r10, VCPU_MMCR + 24(r9)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
	std	r7, VCPU_SIAR(r9)
	std	r8, VCPU_SDAR(r9)
	mfspr	r3, SPRN_PMC1
	mfspr	r4, SPRN_PMC2
	mfspr	r5, SPRN_PMC3
	mfspr	r6, SPRN_PMC4
	mfspr	r7, SPRN_PMC5
	mfspr	r8, SPRN_PMC6
	stw	r3, VCPU_PMC(r9)
	stw	r4, VCPU_PMC + 4(r9)
	stw	r5, VCPU_PMC + 8(r9)
	stw	r6, VCPU_PMC + 12(r9)
	stw	r7, VCPU_PMC + 16(r9)
	stw	r8, VCPU_PMC + 20(r9)
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
BEGIN_FTR_SECTION
	mfspr	r5, SPRN_SIER
	mfspr	r6, SPRN_SPMC1
	mfspr	r7, SPRN_SPMC2
	mfspr	r8, SPRN_MMCRS
	std	r5, VCPU_SIER(r9)
	stw	r6, VCPU_PMC + 24(r9)
	stw	r7, VCPU_PMC + 28(r9)
	std	r8, VCPU_MMCR + 32(r9)
	lis	r4, 0x8000
	mtspr	SPRN_MMCRS, r4
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1499
22:
1500 1501 1502 1503 1504 1505
	/* Clear out SLB */
	li	r5,0
	slbmte	r5,r5
	slbia
	ptesync

1506
	/*
1507
	 * POWER7/POWER8 guest -> host partition switch code.
1508 1509 1510
	 * We don't have to lock against tlbies but we do
	 * have to coordinate the hardware threads.
	 */
1511
kvmhv_switch_to_host:
1512
	/* Secondary threads wait for primary to do partition switch */
1513
	ld	r5,HSTATE_KVM_VCORE(r13)
1514 1515
	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
	lbz	r3,HSTATE_PTID(r13)
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
	cmpwi	r3,0
	beq	15f
	HMT_LOW
13:	lbz	r3,VCORE_IN_GUEST(r5)
	cmpwi	r3,0
	bne	13b
	HMT_MEDIUM
	b	16f

	/* Primary thread waits for all the secondaries to exit guest */
15:	lwz	r3,VCORE_ENTRY_EXIT(r5)
1527
	rlwinm	r0,r3,32-8,0xff
1528 1529 1530 1531 1532
	clrldi	r3,r3,56
	cmpw	r3,r0
	bne	15b
	isync

1533 1534 1535 1536 1537
	/* Did we actually switch to the guest at all? */
	lbz	r6, VCORE_IN_GUEST(r5)
	cmpwi	r6, 0
	beq	19f

1538
	/* Primary thread switches back to host partition */
1539 1540 1541 1542 1543 1544 1545 1546
	ld	r6,KVM_HOST_SDR1(r4)
	lwz	r7,KVM_HOST_LPID(r4)
	li	r8,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r8
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync
1547

1548 1549 1550 1551 1552 1553 1554 1555 1556
BEGIN_FTR_SECTION
	/* DPDES is shared between threads */
	mfspr	r7, SPRN_DPDES
	std	r7, VCORE_DPDES(r5)
	/* clear DPDES so we don't get guest doorbells in the host */
	li	r8, 0
	mtspr	SPRN_DPDES, r8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)

1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
	/* If HMI, call kvmppc_realmode_hmi_handler() */
	cmpwi	r12, BOOK3S_INTERRUPT_HMI
	bne	27f
	bl	kvmppc_realmode_hmi_handler
	nop
	li	r12, BOOK3S_INTERRUPT_HMI
	/*
	 * At this point kvmppc_realmode_hmi_handler would have resync-ed
	 * the TB. Hence it is not required to subtract guest timebase
	 * offset from timebase. So, skip it.
	 *
	 * Also, do not call kvmppc_subcore_exit_guest() because it has
	 * been invoked as part of kvmppc_realmode_hmi_handler().
	 */
	b	30f

27:
1574 1575 1576 1577
	/* Subtract timebase offset from timebase */
	ld	r8,VCORE_TB_OFFSET(r5)
	cmpdi	r8,0
	beq	17f
1578
	mftb	r6			/* current guest timebase */
1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
	subf	r8,r8,r6
	mtspr	SPRN_TBU40,r8		/* update upper 40 bits */
	mftb	r7			/* check if lower 24 bits overflowed */
	clrldi	r6,r6,40
	clrldi	r7,r7,40
	cmpld	r7,r6
	bge	17f
	addis	r8,r8,0x100		/* if so, increment upper 40 bits */
	mtspr	SPRN_TBU40,r8

1589 1590 1591 1592 1593
17:	bl	kvmppc_subcore_exit_guest
	nop
30:	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */

1594
	/* Reset PCR */
1595
	ld	r0, VCORE_PCR(r5)
1596 1597 1598 1599 1600
	cmpdi	r0, 0
	beq	18f
	li	r0, 0
	mtspr	SPRN_PCR, r0
18:
1601
	/* Signal secondary CPUs to continue */
1602
	stb	r0,VCORE_IN_GUEST(r5)
1603
19:	lis	r8,0x7fff		/* MAX_INT@h */
1604 1605
	mtspr	SPRN_HDEC,r8

1606
16:	ld	r8,KVM_HOST_LPCR(r4)
1607 1608 1609 1610
	mtspr	SPRN_LPCR,r8
	isync

	/* load host SLB entries */
1611
	ld	r8,PACA_SLBSHADOWPTR(r13)
1612 1613

	.rept	SLB_NUM_BOLTED
1614 1615 1616 1617
	li	r3, SLBSHADOW_SAVEAREA
	LDX_BE	r5, r8, r3
	addi	r3, r3, 8
	LDX_BE	r6, r8, r3
1618 1619 1620 1621 1622 1623
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r8,r8,16
	.endr

1624 1625 1626 1627 1628 1629 1630 1631 1632
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	/* Finish timing, if we have a vcpu */
	ld	r4, HSTATE_KVM_VCPU(r13)
	cmpdi	r4, 0
	li	r3, 0
	beq	2f
	bl	kvmhv_accumulate_time
2:
#endif
1633 1634 1635 1636
	/* Unset guest mode */
	li	r0, KVM_GUEST_MODE_NONE
	stb	r0, HSTATE_IN_GUEST(r13)

1637 1638 1639 1640
	ld	r0, 112+PPC_LR_STKOFF(r1)
	addi	r1, r1, 112
	mtlr	r0
	blr
1641

1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
/*
 * Check whether an HDSI is an HPTE not found fault or something else.
 * If it is an HPTE not found fault that is due to the guest accessing
 * a page that they have mapped but which we have paged out, then
 * we continue on with the guest exit path.  In all other cases,
 * reflect the HDSI to the guest as a DSI.
 */
kvmppc_hdsi:
	mfspr	r4, SPRN_HDAR
	mfspr	r6, SPRN_HDSISR
1652 1653
	/* HPTE not found fault or protection fault? */
	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1654 1655 1656 1657
	beq	1f			/* if not, send it to the guest */
	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
	beq	3f
	clrrdi	r0, r4, 28
1658
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1659 1660
	li	r0, BOOK3S_INTERRUPT_DATA_SEGMENT
	bne	7f			/* if no SLB entry found */
1661 1662 1663 1664 1665
4:	std	r4, VCPU_FAULT_DAR(r9)
	stw	r6, VCPU_FAULT_DSISR(r9)

	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
1666
	li	r7, 1			/* data fault */
1667
	bl	kvmppc_hpte_hv_fault
1668 1669 1670 1671 1672 1673 1674
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
	beq	6f
	cmpdi	r3, -1			/* handle in kernel mode */
1675
	beq	guest_exit_cont
1676 1677 1678
	cmpdi	r3, -2			/* MMIO emulation; need instr word */
	beq	2f

1679
	/* Synthesize a DSI (or DSegI) for the guest */
1680 1681
	ld	r4, VCPU_FAULT_DAR(r9)
	mr	r6, r3
1682
1:	li	r0, BOOK3S_INTERRUPT_DATA_STORAGE
1683
	mtspr	SPRN_DSISR, r6
1684
7:	mtspr	SPRN_DAR, r4
1685 1686
	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
1687
	mr	r10, r0
1688
	bl	kvmppc_msr_interrupt
1689
fast_interrupt_c_return:
1690
6:	ld	r7, VCPU_CTR(r9)
1691
	ld	r8, VCPU_XER(r9)
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
	mtctr	r7
	mtxer	r8
	mr	r4, r9
	b	fast_guest_return

3:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r5)
	b	4b

	/* If this is for emulated MMIO, load the instruction word */
2:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */

	/* Set guest mode to 'jump over instruction' so if lwz faults
	 * we'll just continue at the next IP. */
	li	r0, KVM_GUEST_MODE_SKIP
	stb	r0, HSTATE_IN_GUEST(r13)

	/* Do the access with MSR:DR enabled */
	mfmsr	r3
	ori	r4, r3, MSR_DR		/* Enable paging for data */
	mtmsrd	r4
	lwz	r8, 0(r10)
	mtmsrd	r3

	/* Store the result */
	stw	r8, VCPU_LAST_INST(r9)

	/* Unset guest mode. */
1720
	li	r0, KVM_GUEST_MODE_HOST_HV
1721
	stb	r0, HSTATE_IN_GUEST(r13)
1722
	b	guest_exit_cont
1723

1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
/*
 * Similarly for an HISI, reflect it to the guest as an ISI unless
 * it is an HPTE not found fault for a page that we have paged out.
 */
kvmppc_hisi:
	andis.	r0, r11, SRR1_ISI_NOPT@h
	beq	1f
	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
	beq	3f
	clrrdi	r0, r10, 28
1734
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1735 1736
	li	r0, BOOK3S_INTERRUPT_INST_SEGMENT
	bne	7f			/* if no SLB entry found */
1737 1738 1739 1740 1741 1742
4:
	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
	mr	r4, r10
	mr	r6, r11
	li	r7, 0			/* instruction fault */
1743
	bl	kvmppc_hpte_hv_fault
1744 1745 1746 1747 1748
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
1749
	beq	fast_interrupt_c_return
1750
	cmpdi	r3, -1			/* handle in kernel mode */
1751
	beq	guest_exit_cont
1752

1753
	/* Synthesize an ISI (or ISegI) for the guest */
1754
	mr	r11, r3
1755 1756
1:	li	r0, BOOK3S_INTERRUPT_INST_STORAGE
7:	mtspr	SPRN_SRR0, r10
1757
	mtspr	SPRN_SRR1, r11
1758
	mr	r10, r0
1759
	bl	kvmppc_msr_interrupt
1760
	b	fast_interrupt_c_return
1761 1762 1763 1764 1765

3:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r6)
	b	4b

1766 1767 1768 1769 1770
/*
 * Try to handle an hcall in real mode.
 * Returns to the guest if we handle it, or continues on up to
 * the kernel if we can't (i.e. if we don't have a handler for
 * it, or if the handler returns H_TOO_HARD).
1771 1772 1773
 *
 * r5 - r8 contain hcall args,
 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1774 1775
 */
hcall_try_real_mode:
1776
	ld	r3,VCPU_GPR(R3)(r9)
1777
	andi.	r0,r11,MSR_PR
1778 1779
	/* sc 1 from userspace - reflect to guest syscall */
	bne	sc_1_fast_return
1780 1781
	clrrdi	r3,r3,2
	cmpldi	r3,hcall_real_table_end - hcall_real_table
1782
	bge	guest_exit_cont
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
	/* See if this hcall is enabled for in-kernel handling */
	ld	r4, VCPU_KVM(r9)
	srdi	r0, r3, 8	/* r0 = (r3 / 4) >> 6 */
	sldi	r0, r0, 3	/* index into kvm->arch.enabled_hcalls[] */
	add	r4, r4, r0
	ld	r0, KVM_ENABLED_HCALLS(r4)
	rlwinm	r4, r3, 32-2, 0x3f	/* r4 = (r3 / 4) & 0x3f */
	srd	r0, r0, r4
	andi.	r0, r0, 1
	beq	guest_exit_cont
	/* Get pointer to handler, if any, and call it */
1794
	LOAD_REG_ADDR(r4, hcall_real_table)
1795
	lwax	r3,r3,r4
1796
	cmpwi	r3,0
1797
	beq	guest_exit_cont
1798 1799
	add	r12,r3,r4
	mtctr	r12
1800
	mr	r3,r9		/* get vcpu pointer */
1801
	ld	r4,VCPU_GPR(R4)(r9)
1802 1803 1804 1805
	bctrl
	cmpdi	r3,H_TOO_HARD
	beq	hcall_real_fallback
	ld	r4,HSTATE_KVM_VCPU(r13)
1806
	std	r3,VCPU_GPR(R3)(r4)
1807 1808 1809 1810
	ld	r10,VCPU_PC(r4)
	ld	r11,VCPU_MSR(r4)
	b	fast_guest_return

1811 1812 1813 1814
sc_1_fast_return:
	mtspr	SPRN_SRR0,r10
	mtspr	SPRN_SRR1,r11
	li	r10, BOOK3S_INTERRUPT_SYSCALL
1815
	bl	kvmppc_msr_interrupt
1816 1817 1818
	mr	r4,r9
	b	fast_guest_return

1819 1820 1821 1822 1823 1824 1825
	/* We've attempted a real mode hcall, but it's punted it back
	 * to userspace.  We need to restore some clobbered volatiles
	 * before resuming the pass-it-to-qemu path */
hcall_real_fallback:
	li	r12,BOOK3S_INTERRUPT_SYSCALL
	ld	r9, HSTATE_KVM_VCPU(r13)

1826
	b	guest_exit_cont
1827 1828 1829 1830

	.globl	hcall_real_table
hcall_real_table:
	.long	0		/* 0 - unused */
1831 1832 1833
	.long	DOTSYM(kvmppc_h_remove) - hcall_real_table
	.long	DOTSYM(kvmppc_h_enter) - hcall_real_table
	.long	DOTSYM(kvmppc_h_read) - hcall_real_table
1834 1835
	.long	DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
	.long	DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1836 1837
	.long	DOTSYM(kvmppc_h_protect) - hcall_real_table
	.long	DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1838
	.long	DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
1839
	.long	0		/* 0x24 - H_SET_SPRG0 */
1840
	.long	DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
	.long	0		/* 0x2c */
	.long	0		/* 0x30 */
	.long	0		/* 0x34 */
	.long	0		/* 0x38 */
	.long	0		/* 0x3c */
	.long	0		/* 0x40 */
	.long	0		/* 0x44 */
	.long	0		/* 0x48 */
	.long	0		/* 0x4c */
	.long	0		/* 0x50 */
	.long	0		/* 0x54 */
	.long	0		/* 0x58 */
	.long	0		/* 0x5c */
	.long	0		/* 0x60 */
1855
#ifdef CONFIG_KVM_XICS
1856 1857 1858
	.long	DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
	.long	DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
	.long	DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1859
	.long	0		/* 0x70 - H_IPOLL */
1860
	.long	DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1861 1862 1863 1864 1865 1866 1867
#else
	.long	0		/* 0x64 - H_EOI */
	.long	0		/* 0x68 - H_CPPR */
	.long	0		/* 0x6c - H_IPI */
	.long	0		/* 0x70 - H_IPOLL */
	.long	0		/* 0x74 - H_XIRR */
#endif
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
	.long	0		/* 0x78 */
	.long	0		/* 0x7c */
	.long	0		/* 0x80 */
	.long	0		/* 0x84 */
	.long	0		/* 0x88 */
	.long	0		/* 0x8c */
	.long	0		/* 0x90 */
	.long	0		/* 0x94 */
	.long	0		/* 0x98 */
	.long	0		/* 0x9c */
	.long	0		/* 0xa0 */
	.long	0		/* 0xa4 */
	.long	0		/* 0xa8 */
	.long	0		/* 0xac */
	.long	0		/* 0xb0 */
	.long	0		/* 0xb4 */
	.long	0		/* 0xb8 */
	.long	0		/* 0xbc */
	.long	0		/* 0xc0 */
	.long	0		/* 0xc4 */
	.long	0		/* 0xc8 */
	.long	0		/* 0xcc */
	.long	0		/* 0xd0 */
	.long	0		/* 0xd4 */
	.long	0		/* 0xd8 */
	.long	0		/* 0xdc */
1894
	.long	DOTSYM(kvmppc_h_cede) - hcall_real_table
1895
	.long	DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
	.long	0		/* 0xe8 */
	.long	0		/* 0xec */
	.long	0		/* 0xf0 */
	.long	0		/* 0xf4 */
	.long	0		/* 0xf8 */
	.long	0		/* 0xfc */
	.long	0		/* 0x100 */
	.long	0		/* 0x104 */
	.long	0		/* 0x108 */
	.long	0		/* 0x10c */
	.long	0		/* 0x110 */
	.long	0		/* 0x114 */
	.long	0		/* 0x118 */
	.long	0		/* 0x11c */
	.long	0		/* 0x120 */
1911
	.long	DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1912 1913 1914
	.long	0		/* 0x128 */
	.long	0		/* 0x12c */
	.long	0		/* 0x130 */
1915
	.long	DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1916
	.long	DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
1917
	.long	DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
	.long	0		/* 0x140 */
	.long	0		/* 0x144 */
	.long	0		/* 0x148 */
	.long	0		/* 0x14c */
	.long	0		/* 0x150 */
	.long	0		/* 0x154 */
	.long	0		/* 0x158 */
	.long	0		/* 0x15c */
	.long	0		/* 0x160 */
	.long	0		/* 0x164 */
	.long	0		/* 0x168 */
	.long	0		/* 0x16c */
	.long	0		/* 0x170 */
	.long	0		/* 0x174 */
	.long	0		/* 0x178 */
	.long	0		/* 0x17c */
	.long	0		/* 0x180 */
	.long	0		/* 0x184 */
	.long	0		/* 0x188 */
	.long	0		/* 0x18c */
	.long	0		/* 0x190 */
	.long	0		/* 0x194 */
	.long	0		/* 0x198 */
	.long	0		/* 0x19c */
	.long	0		/* 0x1a0 */
	.long	0		/* 0x1a4 */
	.long	0		/* 0x1a8 */
	.long	0		/* 0x1ac */
	.long	0		/* 0x1b0 */
	.long	0		/* 0x1b4 */
	.long	0		/* 0x1b8 */
	.long	0		/* 0x1bc */
	.long	0		/* 0x1c0 */
	.long	0		/* 0x1c4 */
	.long	0		/* 0x1c8 */
	.long	0		/* 0x1cc */
	.long	0		/* 0x1d0 */
	.long	0		/* 0x1d4 */
	.long	0		/* 0x1d8 */
	.long	0		/* 0x1dc */
	.long	0		/* 0x1e0 */
	.long	0		/* 0x1e4 */
	.long	0		/* 0x1e8 */
	.long	0		/* 0x1ec */
	.long	0		/* 0x1f0 */
	.long	0		/* 0x1f4 */
	.long	0		/* 0x1f8 */
	.long	0		/* 0x1fc */
	.long	0		/* 0x200 */
	.long	0		/* 0x204 */
	.long	0		/* 0x208 */
	.long	0		/* 0x20c */
	.long	0		/* 0x210 */
	.long	0		/* 0x214 */
	.long	0		/* 0x218 */
	.long	0		/* 0x21c */
	.long	0		/* 0x220 */
	.long	0		/* 0x224 */
	.long	0		/* 0x228 */
	.long	0		/* 0x22c */
	.long	0		/* 0x230 */
	.long	0		/* 0x234 */
	.long	0		/* 0x238 */
	.long	0		/* 0x23c */
	.long	0		/* 0x240 */
	.long	0		/* 0x244 */
	.long	0		/* 0x248 */
	.long	0		/* 0x24c */
	.long	0		/* 0x250 */
	.long	0		/* 0x254 */
	.long	0		/* 0x258 */
	.long	0		/* 0x25c */
	.long	0		/* 0x260 */
	.long	0		/* 0x264 */
	.long	0		/* 0x268 */
	.long	0		/* 0x26c */
	.long	0		/* 0x270 */
	.long	0		/* 0x274 */
	.long	0		/* 0x278 */
	.long	0		/* 0x27c */
	.long	0		/* 0x280 */
	.long	0		/* 0x284 */
	.long	0		/* 0x288 */
	.long	0		/* 0x28c */
	.long	0		/* 0x290 */
	.long	0		/* 0x294 */
	.long	0		/* 0x298 */
	.long	0		/* 0x29c */
	.long	0		/* 0x2a0 */
	.long	0		/* 0x2a4 */
	.long	0		/* 0x2a8 */
	.long	0		/* 0x2ac */
	.long	0		/* 0x2b0 */
	.long	0		/* 0x2b4 */
	.long	0		/* 0x2b8 */
	.long	0		/* 0x2bc */
	.long	0		/* 0x2c0 */
	.long	0		/* 0x2c4 */
	.long	0		/* 0x2c8 */
	.long	0		/* 0x2cc */
	.long	0		/* 0x2d0 */
	.long	0		/* 0x2d4 */
	.long	0		/* 0x2d8 */
	.long	0		/* 0x2dc */
	.long	0		/* 0x2e0 */
	.long	0		/* 0x2e4 */
	.long	0		/* 0x2e8 */
	.long	0		/* 0x2ec */
	.long	0		/* 0x2f0 */
	.long	0		/* 0x2f4 */
	.long	0		/* 0x2f8 */
	.long	0		/* 0x2fc */
	.long	DOTSYM(kvmppc_h_random) - hcall_real_table
2031
	.globl	hcall_real_table_end
2032 2033
hcall_real_table_end:

2034 2035 2036 2037 2038 2039 2040 2041 2042
_GLOBAL(kvmppc_h_set_xdabr)
	andi.	r0, r5, DABRX_USER | DABRX_KERNEL
	beq	6f
	li	r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
	andc.	r0, r5, r0
	beq	3f
6:	li	r3, H_PARAMETER
	blr

2043
_GLOBAL(kvmppc_h_set_dabr)
2044 2045
	li	r5, DABRX_USER | DABRX_KERNEL
3:
2046 2047 2048
BEGIN_FTR_SECTION
	b	2f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2049
	std	r4,VCPU_DABR(r3)
2050 2051
	stw	r5, VCPU_DABRX(r3)
	mtspr	SPRN_DABRX, r5
2052 2053 2054 2055 2056 2057
	/* Work around P7 bug where DABR can get corrupted on mtspr */
1:	mtspr	SPRN_DABR,r4
	mfspr	r5, SPRN_DABR
	cmpd	r4, r5
	bne	1b
	isync
2058 2059 2060
	li	r3,0
	blr

2061 2062
	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2:	rlwimi	r5, r4, 5, DAWRX_DR | DAWRX_DW
2063
	rlwimi	r5, r4, 2, DAWRX_WT
2064 2065 2066 2067 2068 2069
	clrrdi	r4, r4, 3
	std	r4, VCPU_DAWR(r3)
	std	r5, VCPU_DAWRX(r3)
	mtspr	SPRN_DAWR, r4
	mtspr	SPRN_DAWRX, r5
	li	r3, 0
2070 2071
	blr

2072
_GLOBAL(kvmppc_h_cede)		/* r3 = vcpu pointer, r11 = msr, r13 = paca */
2073 2074 2075 2076 2077 2078 2079
	ori	r11,r11,MSR_EE
	std	r11,VCPU_MSR(r3)
	li	r0,1
	stb	r0,VCPU_CEDED(r3)
	sync			/* order setting ceded vs. testing prodded */
	lbz	r5,VCPU_PRODDED(r3)
	cmpwi	r5,0
2080
	bne	kvm_cede_prodded
2081 2082
	li	r12,0		/* set trap to 0 to say hcall is handled */
	stw	r12,VCPU_TRAP(r3)
2083
	li	r0,H_SUCCESS
2084
	std	r0,VCPU_GPR(R3)(r3)
2085 2086 2087 2088 2089 2090 2091

	/*
	 * Set our bit in the bitmask of napping threads unless all the
	 * other threads are already napping, in which case we send this
	 * up to the host.
	 */
	ld	r5,HSTATE_KVM_VCORE(r13)
2092
	lbz	r6,HSTATE_PTID(r13)
2093 2094 2095 2096 2097 2098 2099
	lwz	r8,VCORE_ENTRY_EXIT(r5)
	clrldi	r8,r8,56
	li	r0,1
	sld	r0,r0,r6
	addi	r6,r5,VCORE_NAPPING_THREADS
31:	lwarx	r4,0,r6
	or	r4,r4,r0
2100 2101
	cmpw	r4,r8
	beq	kvm_cede_exit
2102 2103
	stwcx.	r4,0,r6
	bne	31b
2104
	/* order napping_threads update vs testing entry_exit_map */
2105
	isync
2106
	li	r0,NAPPING_CEDE
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
	stb	r0,HSTATE_NAPPING(r13)
	lwz	r7,VCORE_ENTRY_EXIT(r5)
	cmpwi	r7,0x100
	bge	33f		/* another thread already exiting */

/*
 * Although not specifically required by the architecture, POWER7
 * preserves the following registers in nap mode, even if an SMT mode
 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
 */
	/* Save non-volatile GPRs */
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
	std	r14, VCPU_GPR(R14)(r3)
	std	r15, VCPU_GPR(R15)(r3)
	std	r16, VCPU_GPR(R16)(r3)
	std	r17, VCPU_GPR(R17)(r3)
	std	r18, VCPU_GPR(R18)(r3)
	std	r19, VCPU_GPR(R19)(r3)
	std	r20, VCPU_GPR(R20)(r3)
	std	r21, VCPU_GPR(R21)(r3)
	std	r22, VCPU_GPR(R22)(r3)
	std	r23, VCPU_GPR(R23)(r3)
	std	r24, VCPU_GPR(R24)(r3)
	std	r25, VCPU_GPR(R25)(r3)
	std	r26, VCPU_GPR(R26)(r3)
	std	r27, VCPU_GPR(R27)(r3)
	std	r28, VCPU_GPR(R28)(r3)
	std	r29, VCPU_GPR(R29)(r3)
	std	r30, VCPU_GPR(R30)(r3)
	std	r31, VCPU_GPR(R31)(r3)
2137 2138

	/* save FP state */
2139
	bl	kvmppc_save_fp
2140

2141 2142 2143 2144 2145 2146 2147
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	ld	r9, HSTATE_KVM_VCPU(r13)
	bl	kvmppc_save_tm
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif

2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168
	/*
	 * Set DEC to the smaller of DEC and HDEC, so that we wake
	 * no later than the end of our timeslice (HDEC interrupts
	 * don't wake us from nap).
	 */
	mfspr	r3, SPRN_DEC
	mfspr	r4, SPRN_HDEC
	mftb	r5
	cmpw	r3, r4
	ble	67f
	mtspr	SPRN_DEC, r4
67:
	/* save expiry time of guest decrementer */
	extsw	r3, r3
	add	r3, r3, r5
	ld	r4, HSTATE_KVM_VCPU(r13)
	ld	r5, HSTATE_KVM_VCORE(r13)
	ld	r6, VCORE_TB_OFFSET(r5)
	subf	r3, r6, r3	/* convert to host TB value */
	std	r3, VCPU_DEC_EXPIRES(r4)

2169 2170 2171 2172 2173 2174
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	ld	r4, HSTATE_KVM_VCPU(r13)
	addi	r3, r4, VCPU_TB_CEDE
	bl	kvmhv_accumulate_time
#endif

2175 2176
	lis	r3, LPCR_PECEDP@h	/* Do wake on privileged doorbell */

2177
	/*
2178
	 * Take a nap until a decrementer or external or doobell interrupt
2179
	 * occurs, with PECE1 and PECE0 set in LPCR.
2180
	 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2181
	 * Also clear the runlatch bit before napping.
2182
	 */
2183
kvm_do_nap:
2184 2185 2186
	mfspr	r0, SPRN_CTRLF
	clrrdi	r0, r0, 1
	mtspr	SPRN_CTRLT, r0
2187

2188 2189
	li	r0,1
	stb	r0,HSTATE_HWTHREAD_REQ(r13)
2190 2191
	mfspr	r5,SPRN_LPCR
	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
2192
BEGIN_FTR_SECTION
2193
	ori	r5, r5, LPCR_PECEDH
2194
	rlwimi	r5, r3, 0, LPCR_PECEDP
2195
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
	mtspr	SPRN_LPCR,r5
	isync
	li	r0, 0
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

2207 2208 2209 2210 2211
33:	mr	r4, r3
	li	r3, 0
	li	r12, 0
	b	34f

2212
kvm_end_cede:
2213 2214 2215
	/* get vcpu pointer */
	ld	r4, HSTATE_KVM_VCPU(r13)

2216 2217 2218
	/* Woken by external or decrementer interrupt */
	ld	r1, HSTATE_HOST_R1(r13)

2219 2220 2221 2222 2223
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
	addi	r3, r4, VCPU_TB_RMINTR
	bl	kvmhv_accumulate_time
#endif

2224 2225 2226 2227 2228 2229
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	bl	kvmppc_restore_tm
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif

2230 2231 2232
	/* load up FP state */
	bl	kvmppc_load_fp

2233 2234 2235 2236 2237 2238 2239 2240 2241
	/* Restore guest decrementer */
	ld	r3, VCPU_DEC_EXPIRES(r4)
	ld	r5, HSTATE_KVM_VCORE(r13)
	ld	r6, VCORE_TB_OFFSET(r5)
	add	r3, r3, r6	/* convert host TB to guest TB value */
	mftb	r7
	subf	r3, r7, r3
	mtspr	SPRN_DEC, r3

2242
	/* Load NV GPRS */
2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)
2261

2262 2263
	/* Check the wake reason in SRR1 to see why we got here */
	bl	kvmppc_check_wake_reason
2264

2265 2266 2267 2268 2269 2270 2271 2272 2273 2274
	/*
	 * Restore volatile registers since we could have called a
	 * C routine in kvmppc_check_wake_reason
	 *	r4 = VCPU
	 * r3 tells us whether we need to return to host or not
	 * WARNING: it gets checked further down:
	 * should not modify r3 until this check is done.
	 */
	ld	r4, HSTATE_KVM_VCPU(r13)

2275
	/* clear our bit in vcore->napping_threads */
2276 2277
34:	ld	r5,HSTATE_KVM_VCORE(r13)
	lbz	r7,HSTATE_PTID(r13)
2278
	li	r0,1
2279
	sld	r0,r0,r7
2280 2281 2282 2283 2284 2285 2286 2287
	addi	r6,r5,VCORE_NAPPING_THREADS
32:	lwarx	r7,0,r6
	andc	r7,r7,r0
	stwcx.	r7,0,r6
	bne	32b
	li	r0,0
	stb	r0,HSTATE_NAPPING(r13)

2288
	/* See if the wake reason saved in r3 means we need to exit */
2289
	stw	r12, VCPU_TRAP(r4)
2290
	mr	r9, r4
2291 2292
	cmpdi	r3, 0
	bgt	guest_exit_cont
2293

2294 2295 2296
	/* see if any other thread is already exiting */
	lwz	r0,VCORE_ENTRY_EXIT(r5)
	cmpwi	r0,0x100
2297
	bge	guest_exit_cont
2298

2299
	b	kvmppc_cede_reentry	/* if not go back to guest */
2300 2301

	/* cede when already previously prodded case */
2302 2303
kvm_cede_prodded:
	li	r0,0
2304 2305 2306 2307 2308 2309 2310
	stb	r0,VCPU_PRODDED(r3)
	sync			/* order testing prodded vs. clearing ceded */
	stb	r0,VCPU_CEDED(r3)
	li	r3,H_SUCCESS
	blr

	/* we've ceded but we want to give control to the host */
2311
kvm_cede_exit:
2312 2313
	ld	r9, HSTATE_KVM_VCPU(r13)
	b	guest_exit_cont
2314

2315 2316 2317
	/* Try to handle a machine check in real mode */
machine_check_realmode:
	mr	r3, r9		/* get vcpu pointer */
2318
	bl	kvmppc_realmode_machine_check
2319 2320 2321
	nop
	ld	r9, HSTATE_KVM_VCPU(r13)
	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
	/*
	 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
	 * machine check interrupt (set HSRR0 to 0x200). And for handled
	 * errors (no-fatal), just go back to guest execution with current
	 * HSRR0 instead of exiting guest. This new approach will inject
	 * machine check to guest for fatal error causing guest to crash.
	 *
	 * The old code used to return to host for unhandled errors which
	 * was causing guest to hang with soft lockups inside guest and
	 * makes it difficult to recover guest instance.
2332 2333 2334
	 *
	 * if we receive machine check with MSR(RI=0) then deliver it to
	 * guest as machine check causing guest to crash.
2335 2336
	 */
	ld	r11, VCPU_MSR(r9)
2337 2338
	rldicl.	r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */
	bne	mc_cont			/* if so, exit to host */
2339 2340 2341 2342
	andi.	r10, r11, MSR_RI	/* check for unrecoverable exception */
	beq	1f			/* Deliver a machine check to guest */
	ld	r10, VCPU_PC(r9)
	cmpdi	r3, 0		/* Did we handle MCE ? */
2343
	bne	2f	/* Continue guest execution. */
2344
	/* If not, deliver a machine check.  SRR0/1 are already set */
2345
1:	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2346
	bl	kvmppc_msr_interrupt
2347
2:	b	fast_interrupt_c_return
2348

2349 2350
/*
 * Check the reason we woke from nap, and take appropriate action.
2351
 * Returns (in r3):
2352 2353
 *	0 if nothing needs to be done
 *	1 if something happened that needs to be handled by the host
2354
 *	-1 if there was a guest wakeup (IPI or msgsnd)
2355 2356 2357
 *
 * Also sets r12 to the interrupt vector for any interrupt that needs
 * to be handled now by the host (0x500 for external interrupt), or zero.
2358 2359 2360
 * Modifies all volatile registers (since it may call a C function).
 * This routine calls kvmppc_read_intr, a C function, if an external
 * interrupt is pending.
2361 2362 2363
 */
kvmppc_check_wake_reason:
	mfspr	r6, SPRN_SRR1
2364 2365 2366 2367 2368 2369
BEGIN_FTR_SECTION
	rlwinm	r6, r6, 45-31, 0xf	/* extract wake reason field (P8) */
FTR_SECTION_ELSE
	rlwinm	r6, r6, 45-31, 0xe	/* P7 wake reason field is 3 bits */
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
	cmpwi	r6, 8			/* was it an external interrupt? */
2370
	beq	7f			/* if so, see what it was */
2371 2372 2373 2374
	li	r3, 0
	li	r12, 0
	cmpwi	r6, 6			/* was it the decrementer? */
	beq	0f
2375 2376 2377
BEGIN_FTR_SECTION
	cmpwi	r6, 5			/* privileged doorbell? */
	beq	0f
2378 2379
	cmpwi	r6, 3			/* hypervisor doorbell? */
	beq	3f
2380
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2381 2382
	cmpwi	r6, 0xa			/* Hypervisor maintenance ? */
	beq	4f
2383 2384 2385
	li	r3, 1			/* anything else, return 1 */
0:	blr

2386 2387
	/* hypervisor doorbell */
3:	li	r12, BOOK3S_INTERRUPT_H_DOORBELL
2388 2389 2390 2391 2392 2393 2394

	/*
	 * Clear the doorbell as we will invoke the handler
	 * explicitly in the guest exit path.
	 */
	lis	r6, (PPC_DBELL_SERVER << (63-36))@h
	PPC_MSGCLR(6)
2395
	/* see if it's a host IPI */
2396
	li	r3, 1
2397 2398 2399
	lbz	r0, HSTATE_HOST_IPI(r13)
	cmpwi	r0, 0
	bnelr
2400
	/* if not, return -1 */
2401
	li	r3, -1
2402 2403
	blr

2404 2405 2406 2407 2408
	/* Woken up due to Hypervisor maintenance interrupt */
4:	li	r12, BOOK3S_INTERRUPT_HMI
	li	r3, 1
	blr

2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419
	/* external interrupt - create a stack frame so we can call C */
7:	mflr	r0
	std	r0, PPC_LR_STKOFF(r1)
	stdu	r1, -PPC_MIN_STKFRM(r1)
	bl	kvmppc_read_intr
	nop
	li	r12, BOOK3S_INTERRUPT_EXTERNAL
	ld	r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
	addi	r1, r1, PPC_MIN_STKFRM
	mtlr	r0
	blr
2420

2421 2422 2423
/*
 * Save away FP, VMX and VSX registers.
 * r3 = vcpu pointer
2424 2425
 * N.B. r30 and r31 are volatile across this function,
 * thus it is not callable from C.
2426
 */
2427 2428 2429
kvmppc_save_fp:
	mflr	r30
	mr	r31,r3
2430 2431
	mfmsr	r5
	ori	r8,r5,MSR_FP
2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
2443
	addi	r3,r3,VCPU_FPRS
2444
	bl	store_fp_state
2445 2446
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
2447
	addi	r3,r31,VCPU_VRS
2448
	bl	store_vr_state
2449 2450 2451
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
	mfspr	r6,SPRN_VRSAVE
2452
	stw	r6,VCPU_VRSAVE(r31)
2453
	mtlr	r30
2454 2455 2456 2457 2458
	blr

/*
 * Load up FP, VMX and VSX registers
 * r4 = vcpu pointer
2459 2460
 * N.B. r30 and r31 are volatile across this function,
 * thus it is not callable from C.
2461 2462
 */
kvmppc_load_fp:
2463 2464
	mflr	r30
	mr	r31,r4
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
	mfmsr	r9
	ori	r8,r9,MSR_FP
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
2478
	addi	r3,r4,VCPU_FPRS
2479
	bl	load_fp_state
2480 2481
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
2482
	addi	r3,r31,VCPU_VRS
2483
	bl	load_vr_state
2484 2485
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
2486
	lwz	r7,VCPU_VRSAVE(r31)
2487
	mtspr	SPRN_VRSAVE,r7
2488 2489
	mtlr	r30
	mr	r4,r31
2490
	blr
2491

2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
/*
 * Save transactional state and TM-related registers.
 * Called with r9 pointing to the vcpu struct.
 * This can modify all checkpointed registers, but
 * restores r1, r2 and r9 (vcpu pointer) before exit.
 */
kvmppc_save_tm:
	mflr	r0
	std	r0, PPC_LR_STKOFF(r1)

	/* Turn on TM. */
	mfmsr	r8
	li	r0, 1
	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
	mtmsrd	r8

	ld	r5, VCPU_MSR(r9)
	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
	beq	1f	/* TM not active in guest. */

	std	r1, HSTATE_HOST_R1(r13)
	li	r3, TM_CAUSE_KVM_RESCHED

	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
	li	r5, 0
	mtmsrd	r5, 1

	/* All GPRs are volatile at this point. */
	TRECLAIM(R3)

	/* Temporarily store r13 and r9 so we have some regs to play with */
	SET_SCRATCH0(r13)
	GET_PACA(r13)
	std	r9, PACATMSCRATCH(r13)
	ld	r9, HSTATE_KVM_VCPU(r13)

	/* Get a few more GPRs free. */
	std	r29, VCPU_GPRS_TM(29)(r9)
	std	r30, VCPU_GPRS_TM(30)(r9)
	std	r31, VCPU_GPRS_TM(31)(r9)

	/* Save away PPR and DSCR soon so don't run with user values. */
	mfspr	r31, SPRN_PPR
	HMT_MEDIUM
	mfspr	r30, SPRN_DSCR
	ld	r29, HSTATE_DSCR(r13)
	mtspr	SPRN_DSCR, r29

	/* Save all but r9, r13 & r29-r31 */
	reg = 0
	.rept	29
	.if (reg != 9) && (reg != 13)
	std	reg, VCPU_GPRS_TM(reg)(r9)
	.endif
	reg = reg + 1
	.endr
	/* ... now save r13 */
	GET_SCRATCH0(r4)
	std	r4, VCPU_GPRS_TM(13)(r9)
	/* ... and save r9 */
	ld	r4, PACATMSCRATCH(r13)
	std	r4, VCPU_GPRS_TM(9)(r9)

	/* Reload stack pointer and TOC. */
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATOC(r13)

	/* Set MSR RI now we have r1 and r13 back. */
	li	r5, MSR_RI
	mtmsrd	r5, 1

	/* Save away checkpinted SPRs. */
	std	r31, VCPU_PPR_TM(r9)
	std	r30, VCPU_DSCR_TM(r9)
	mflr	r5
	mfcr	r6
	mfctr	r7
	mfspr	r8, SPRN_AMR
	mfspr	r10, SPRN_TAR
	std	r5, VCPU_LR_TM(r9)
	stw	r6, VCPU_CR_TM(r9)
	std	r7, VCPU_CTR_TM(r9)
	std	r8, VCPU_AMR_TM(r9)
	std	r10, VCPU_TAR_TM(r9)

	/* Restore r12 as trap number. */
	lwz	r12, VCPU_TRAP(r9)

	/* Save FP/VSX. */
	addi	r3, r9, VCPU_FPRS_TM
	bl	store_fp_state
	addi	r3, r9, VCPU_VRS_TM
	bl	store_vr_state
	mfspr	r6, SPRN_VRSAVE
	stw	r6, VCPU_VRSAVE_TM(r9)
1:
	/*
	 * We need to save these SPRs after the treclaim so that the software
	 * error code is recorded correctly in the TEXASR.  Also the user may
	 * change these outside of a transaction, so they must always be
	 * context switched.
	 */
	mfspr	r5, SPRN_TFHAR
	mfspr	r6, SPRN_TFIAR
	mfspr	r7, SPRN_TEXASR
	std	r5, VCPU_TFHAR(r9)
	std	r6, VCPU_TFIAR(r9)
	std	r7, VCPU_TEXASR(r9)

	ld	r0, PPC_LR_STKOFF(r1)
	mtlr	r0
	blr

/*
 * Restore transactional state and TM-related registers.
 * Called with r4 pointing to the vcpu struct.
 * This potentially modifies all checkpointed registers.
 * It restores r1, r2, r4 from the PACA.
 */
kvmppc_restore_tm:
	mflr	r0
	std	r0, PPC_LR_STKOFF(r1)

	/* Turn on TM/FP/VSX/VMX so we can restore them. */
	mfmsr	r5
	li	r6, MSR_TM >> 32
	sldi	r6, r6, 32
	or	r5, r5, r6
	ori	r5, r5, MSR_FP
	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
	mtmsrd	r5

	/*
	 * The user may change these outside of a transaction, so they must
	 * always be context switched.
	 */
	ld	r5, VCPU_TFHAR(r4)
	ld	r6, VCPU_TFIAR(r4)
	ld	r7, VCPU_TEXASR(r4)
	mtspr	SPRN_TFHAR, r5
	mtspr	SPRN_TFIAR, r6
	mtspr	SPRN_TEXASR, r7

	ld	r5, VCPU_MSR(r4)
	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
	beqlr		/* TM not active in guest */
	std	r1, HSTATE_HOST_R1(r13)

	/* Make sure the failure summary is set, otherwise we'll program check
	 * when we trechkpt.  It's possible that this might have been not set
	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
	 * host.
	 */
	oris	r7, r7, (TEXASR_FS)@h
	mtspr	SPRN_TEXASR, r7

	/*
	 * We need to load up the checkpointed state for the guest.
	 * We need to do this early as it will blow away any GPRs, VSRs and
	 * some SPRs.
	 */

	mr	r31, r4
	addi	r3, r31, VCPU_FPRS_TM
	bl	load_fp_state
	addi	r3, r31, VCPU_VRS_TM
	bl	load_vr_state
	mr	r4, r31
	lwz	r7, VCPU_VRSAVE_TM(r4)
	mtspr	SPRN_VRSAVE, r7

	ld	r5, VCPU_LR_TM(r4)
	lwz	r6, VCPU_CR_TM(r4)
	ld	r7, VCPU_CTR_TM(r4)
	ld	r8, VCPU_AMR_TM(r4)
	ld	r9, VCPU_TAR_TM(r4)
	mtlr	r5
	mtcr	r6
	mtctr	r7
	mtspr	SPRN_AMR, r8
	mtspr	SPRN_TAR, r9

	/*
	 * Load up PPR and DSCR values but don't put them in the actual SPRs
	 * till the last moment to avoid running with userspace PPR and DSCR for
	 * too long.
	 */
	ld	r29, VCPU_DSCR_TM(r4)
	ld	r30, VCPU_PPR_TM(r4)

	std	r2, PACATMSCRATCH(r13) /* Save TOC */

	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
	li	r5, 0
	mtmsrd	r5, 1

	/* Load GPRs r0-r28 */
	reg = 0
	.rept	29
	ld	reg, VCPU_GPRS_TM(reg)(r31)
	reg = reg + 1
	.endr

	mtspr	SPRN_DSCR, r29
	mtspr	SPRN_PPR, r30

	/* Load final GPRs */
	ld	29, VCPU_GPRS_TM(29)(r31)
	ld	30, VCPU_GPRS_TM(30)(r31)
	ld	31, VCPU_GPRS_TM(31)(r31)

	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
	TRECHKPT

	/* Now let's get back the state we need. */
	HMT_MEDIUM
	GET_PACA(r13)
	ld	r29, HSTATE_DSCR(r13)
	mtspr	SPRN_DSCR, r29
	ld	r4, HSTATE_KVM_VCPU(r13)
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATMSCRATCH(r13)

	/* Set the MSR RI since we have our registers back. */
	li	r5, MSR_RI
	mtmsrd	r5, 1

	ld	r0, PPC_LR_STKOFF(r1)
	mtlr	r0
	blr
#endif

2725 2726 2727 2728 2729 2730 2731
/*
 * We come here if we get any exception or interrupt while we are
 * executing host real mode code while in guest MMU context.
 * For now just spin, but we should do something better.
 */
kvmppc_bad_host_intr:
	b	.
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748

/*
 * This mimics the MSR transition on IRQ delivery.  The new guest MSR is taken
 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
 *   r11 has the guest MSR value (in/out)
 *   r9 has a vcpu pointer (in)
 *   r0 is used as a scratch register
 */
kvmppc_msr_interrupt:
	rldicl	r0, r11, 64 - MSR_TS_S_LG, 62
	cmpwi	r0, 2 /* Check if we are in transactional state..  */
	ld	r11, VCPU_INTR_MSR(r9)
	bne	1f
	/* ... if transactional, change to suspended */
	li	r0, 1
1:	rldimi	r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
	blr
2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766

/*
 * This works around a hardware bug on POWER8E processors, where
 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
 * performance monitor interrupt.  Instead, when we need to have
 * an interrupt pending, we have to arrange for a counter to overflow.
 */
kvmppc_fix_pmao:
	li	r3, 0
	mtspr	SPRN_MMCR2, r3
	lis	r3, (MMCR0_PMXE | MMCR0_FCECE)@h
	ori	r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
	mtspr	SPRN_MMCR0, r3
	lis	r3, 0x7fff
	ori	r3, r3, 0xffff
	mtspr	SPRN_PMC6, r3
	isync
	blr
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825

#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
/*
 * Start timing an activity
 * r3 = pointer to time accumulation struct, r4 = vcpu
 */
kvmhv_start_timing:
	ld	r5, HSTATE_KVM_VCORE(r13)
	lbz	r6, VCORE_IN_GUEST(r5)
	cmpwi	r6, 0
	beq	5f				/* if in guest, need to */
	ld	r6, VCORE_TB_OFFSET(r5)		/* subtract timebase offset */
5:	mftb	r5
	subf	r5, r6, r5
	std	r3, VCPU_CUR_ACTIVITY(r4)
	std	r5, VCPU_ACTIVITY_START(r4)
	blr

/*
 * Accumulate time to one activity and start another.
 * r3 = pointer to new time accumulation struct, r4 = vcpu
 */
kvmhv_accumulate_time:
	ld	r5, HSTATE_KVM_VCORE(r13)
	lbz	r8, VCORE_IN_GUEST(r5)
	cmpwi	r8, 0
	beq	4f				/* if in guest, need to */
	ld	r8, VCORE_TB_OFFSET(r5)		/* subtract timebase offset */
4:	ld	r5, VCPU_CUR_ACTIVITY(r4)
	ld	r6, VCPU_ACTIVITY_START(r4)
	std	r3, VCPU_CUR_ACTIVITY(r4)
	mftb	r7
	subf	r7, r8, r7
	std	r7, VCPU_ACTIVITY_START(r4)
	cmpdi	r5, 0
	beqlr
	subf	r3, r6, r7
	ld	r8, TAS_SEQCOUNT(r5)
	cmpdi	r8, 0
	addi	r8, r8, 1
	std	r8, TAS_SEQCOUNT(r5)
	lwsync
	ld	r7, TAS_TOTAL(r5)
	add	r7, r7, r3
	std	r7, TAS_TOTAL(r5)
	ld	r6, TAS_MIN(r5)
	ld	r7, TAS_MAX(r5)
	beq	3f
	cmpd	r3, r6
	bge	1f
3:	std	r3, TAS_MIN(r5)
1:	cmpd	r3, r7
	ble	2f
	std	r3, TAS_MAX(r5)
2:	lwsync
	addi	r8, r8, 1
	std	r8, TAS_SEQCOUNT(r5)
	blr
#endif