book3s_hv_rmhandlers.S 58.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 *
 * Derived from book3s_rmhandlers.S and other files, which are:
 *
 * Copyright SUSE Linux Products GmbH 2009
 *
 * Authors: Alexander Graf <agraf@suse.de>
 */

#include <asm/ppc_asm.h>
#include <asm/kvm_asm.h>
#include <asm/reg.h>
23
#include <asm/mmu.h>
24
#include <asm/page.h>
25 26
#include <asm/ptrace.h>
#include <asm/hvcall.h>
27 28
#include <asm/asm-offsets.h>
#include <asm/exception-64s.h>
29
#include <asm/kvm_book3s_asm.h>
30
#include <asm/mmu-hash64.h>
31 32 33
#include <asm/tm.h>

#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
34

35 36 37 38
#ifdef __LITTLE_ENDIAN__
#error Need to fix lppaca and SLB shadow accesses in little endian mode
#endif

39 40 41 42
/* Values in HSTATE_NAPPING(r13) */
#define NAPPING_CEDE	1
#define NAPPING_NOVCPU	2

43
/*
44
 * Call kvmppc_hv_entry in real mode.
45 46 47 48 49 50 51
 * Must be called with interrupts hard-disabled.
 *
 * Input Registers:
 *
 * LR = return address to continue at after eventually re-enabling MMU
 */
_GLOBAL(kvmppc_hv_entry_trampoline)
52 53 54
	mflr	r0
	std	r0, PPC_LR_STKOFF(r1)
	stdu	r1, -112(r1)
55
	mfmsr	r10
56
	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
57 58 59 60 61 62 63 64 65
	li	r0,MSR_RI
	andc	r0,r10,r0
	li	r6,MSR_IR | MSR_DR
	andc	r6,r10,r6
	mtmsrd	r0,1		/* clear RI in MSR */
	mtsrr0	r5
	mtsrr1	r6
	RFI

66
kvmppc_call_hv_entry:
67
	ld	r4, HSTATE_KVM_VCPU(r13)
68 69 70 71
	bl	kvmppc_hv_entry

	/* Back from guest - restore host state and return to caller */

72
BEGIN_FTR_SECTION
73 74 75 76 77
	/* Restore host DABR and DABRX */
	ld	r5,HSTATE_DABR(r13)
	li	r6,7
	mtspr	SPRN_DABR,r5
	mtspr	SPRN_DABRX,r6
78
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
79 80

	/* Restore SPRG3 */
S
Scott Wood 已提交
81 82
	ld	r3,PACA_SPRG_VDSO(r13)
	mtspr	SPRN_SPRG_VDSO_WRITE,r3
83 84 85 86 87 88

	/* Reload the host's PMU registers */
	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
	lbz	r4, LPPACA_PMCINUSE(r3)
	cmpwi	r4, 0
	beq	23f			/* skip if not */
89 90 91 92 93 94
BEGIN_FTR_SECTION
	ld	r3, HSTATE_MMCR(r13)
	andi.	r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
	cmpwi	r4, MMCR0_PMAO
	beql	kvmppc_fix_pmao
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
	lwz	r3, HSTATE_PMC(r13)
	lwz	r4, HSTATE_PMC + 4(r13)
	lwz	r5, HSTATE_PMC + 8(r13)
	lwz	r6, HSTATE_PMC + 12(r13)
	lwz	r8, HSTATE_PMC + 16(r13)
	lwz	r9, HSTATE_PMC + 20(r13)
BEGIN_FTR_SECTION
	lwz	r10, HSTATE_PMC + 24(r13)
	lwz	r11, HSTATE_PMC + 28(r13)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r4
	mtspr	SPRN_PMC3, r5
	mtspr	SPRN_PMC4, r6
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
BEGIN_FTR_SECTION
	mtspr	SPRN_PMC7, r10
	mtspr	SPRN_PMC8, r11
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	ld	r3, HSTATE_MMCR(r13)
	ld	r4, HSTATE_MMCR + 8(r13)
	ld	r5, HSTATE_MMCR + 16(r13)
118 119
	ld	r6, HSTATE_MMCR + 24(r13)
	ld	r7, HSTATE_MMCR + 32(r13)
120 121
	mtspr	SPRN_MMCR1, r4
	mtspr	SPRN_MMCRA, r5
122 123 124 125 126 127 128 129
	mtspr	SPRN_SIAR, r6
	mtspr	SPRN_SDAR, r7
BEGIN_FTR_SECTION
	ld	r8, HSTATE_MMCR + 40(r13)
	ld	r9, HSTATE_MMCR + 48(r13)
	mtspr	SPRN_MMCR2, r8
	mtspr	SPRN_SIER, r9
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
130 131 132 133
	mtspr	SPRN_MMCR0, r3
	isync
23:

134 135 136 137 138 139 140 141 142
	/*
	 * Reload DEC.  HDEC interrupts were disabled when
	 * we reloaded the host's LPCR value.
	 */
	ld	r3, HSTATE_DECEXP(r13)
	mftb	r4
	subf	r4, r4, r3
	mtspr	SPRN_DEC, r4

143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
	/*
	 * For external and machine check interrupts, we need
	 * to call the Linux handler to process the interrupt.
	 * We do that by jumping to absolute address 0x500 for
	 * external interrupts, or the machine_check_fwnmi label
	 * for machine checks (since firmware might have patched
	 * the vector area at 0x200).  The [h]rfid at the end of the
	 * handler will return to the book3s_hv_interrupts.S code.
	 * For other interrupts we do the rfid to get back
	 * to the book3s_hv_interrupts.S code here.
	 */
	ld	r8, 112+PPC_LR_STKOFF(r1)
	addi	r1, r1, 112
	ld	r7, HSTATE_HOST_MSR(r13)

	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
BEGIN_FTR_SECTION
	beq	11f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* RFI into the highmem handler, or branch to interrupt handler */
	mfmsr	r6
	li	r0, MSR_RI
	andc	r6, r6, r0
	mtmsrd	r6, 1			/* Clear RI in MSR */
	mtsrr0	r8
	mtsrr1	r7
	beqa	0x500			/* external interrupt (PPC970) */
	beq	cr1, 13f		/* machine check */
	RFI

	/* On POWER7, we have external interrupts set to use HSRR0/1 */
11:	mtspr	SPRN_HSRR0, r8
	mtspr	SPRN_HSRR1, r7
	ba	0x500

13:	b	machine_check_fwnmi

182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
kvmppc_primary_no_guest:
	/* We handle this much like a ceded vcpu */
	/* set our bit in napping_threads */
	ld	r5, HSTATE_KVM_VCORE(r13)
	lbz	r7, HSTATE_PTID(r13)
	li	r0, 1
	sld	r0, r0, r7
	addi	r6, r5, VCORE_NAPPING_THREADS
1:	lwarx	r3, 0, r6
	or	r3, r3, r0
	stwcx.	r3, 0, r6
	bne	1b
	/* order napping_threads update vs testing entry_exit_count */
	isync
	li	r12, 0
	lwz	r7, VCORE_ENTRY_EXIT(r5)
	cmpwi	r7, 0x100
	bge	kvm_novcpu_exit	/* another thread already exiting */
	li	r3, NAPPING_NOVCPU
	stb	r3, HSTATE_NAPPING(r13)
	li	r3, 1
	stb	r3, HSTATE_HWTHREAD_REQ(r13)

	b	kvm_do_nap

kvm_novcpu_wakeup:
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r5, HSTATE_KVM_VCORE(r13)
	li	r0, 0
	stb	r0, HSTATE_NAPPING(r13)
	stb	r0, HSTATE_HWTHREAD_REQ(r13)

214 215 216
	/* check the wake reason */
	bl	kvmppc_check_wake_reason
	
217 218 219 220 221 222 223 224 225 226
	/* see if any other thread is already exiting */
	lwz	r0, VCORE_ENTRY_EXIT(r5)
	cmpwi	r0, 0x100
	bge	kvm_novcpu_exit

	/* clear our bit in napping_threads */
	lbz	r7, HSTATE_PTID(r13)
	li	r0, 1
	sld	r0, r0, r7
	addi	r6, r5, VCORE_NAPPING_THREADS
227 228 229
4:	lwarx	r7, 0, r6
	andc	r7, r7, r0
	stwcx.	r7, 0, r6
230 231
	bne	4b

232
	/* See if the wake reason means we need to exit */
233 234 235 236 237 238 239 240 241 242 243
	cmpdi	r3, 0
	bge	kvm_novcpu_exit

	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
	ld	r4, HSTATE_KVM_VCPU(r13)
	cmpdi	r4, 0
	bne	kvmppc_got_guest

kvm_novcpu_exit:
	b	hdec_soon

244
/*
245
 * We come in here when wakened from nap mode.
246 247 248 249 250
 * Relocation is off and most register values are lost.
 * r13 points to the PACA.
 */
	.globl	kvm_start_guest
kvm_start_guest:
251 252 253 254 255 256

	/* Set runlatch bit the minute you wake up from nap */
	mfspr	r1, SPRN_CTRLF
	ori 	r1, r1, 1
	mtspr	SPRN_CTRLT, r1

257 258
	ld	r2,PACATOC(r13)

259 260
	li	r0,KVM_HWTHREAD_IN_KVM
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
261

262 263 264
	/* NV GPR values from power7_idle() will no longer be valid */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)
265

266 267
	/* were we napping due to cede? */
	lbz	r0,HSTATE_NAPPING(r13)
268 269 270 271 272 273 274
	cmpwi	r0,NAPPING_CEDE
	beq	kvm_end_cede
	cmpwi	r0,NAPPING_NOVCPU
	beq	kvm_novcpu_wakeup

	ld	r1,PACAEMERGSP(r13)
	subi	r1,r1,STACK_FRAME_OVERHEAD
275 276 277 278 279 280 281

	/*
	 * We weren't napping due to cede, so this must be a secondary
	 * thread being woken up to run a guest, or being woken up due
	 * to a stray IPI.  (Or due to some machine check or hypervisor
	 * maintenance interrupt while the core is in KVM.)
	 */
282 283

	/* Check the wake reason in SRR1 to see why we got here */
284 285 286
	bl	kvmppc_check_wake_reason
	cmpdi	r3, 0
	bge	kvm_no_guest
287

288
	/* get vcpu pointer, NULL if we have no vcpu to run */
289 290
	ld	r4,HSTATE_KVM_VCPU(r13)
	cmpdi	r4,0
291
	/* if we have no vcpu to run, go back to sleep */
292
	beq	kvm_no_guest
293

294
	/* Set HSTATE_DSCR(r13) to something sensible */
295
	ld	r6, PACA_DSCR(r13)
296
	std	r6, HSTATE_DSCR(r13)
297

298
	bl	kvmppc_hv_entry
299 300 301 302 303

	/* Back from the guest, go back to nap */
	/* Clear our vcpu pointer so we don't come back in early */
	li	r0, 0
	std	r0, HSTATE_KVM_VCPU(r13)
304 305 306 307 308
	/*
	 * Make sure we clear HSTATE_KVM_VCPU(r13) before incrementing
	 * the nap_count, because once the increment to nap_count is
	 * visible we could be given another vcpu.
	 */
309 310 311 312 313 314 315 316 317 318 319 320 321
	lwsync

	/* increment the nap count and then go to nap mode */
	ld	r4, HSTATE_KVM_VCORE(r13)
	addi	r4, r4, VCORE_NAP_COUNT
51:	lwarx	r3, 0, r4
	addi	r3, r3, 1
	stwcx.	r3, 0, r4
	bne	51b

kvm_no_guest:
	li	r0, KVM_HWTHREAD_IN_NAP
	stb	r0, HSTATE_HWTHREAD_STATE(r13)
322
kvm_do_nap:
323 324 325 326 327
	/* Clear the runlatch bit before napping */
	mfspr	r2, SPRN_CTRLF
	clrrdi	r2, r2, 1
	mtspr	SPRN_CTRLT, r2

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	li	r3, LPCR_PECE0
	mfspr	r4, SPRN_LPCR
	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
	mtspr	SPRN_LPCR, r4
	isync
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

/******************************************************************************
 *                                                                            *
 *                               Entry code                                   *
 *                                                                            *
 *****************************************************************************/

347 348 349 350 351
.global kvmppc_hv_entry
kvmppc_hv_entry:

	/* Required state:
	 *
352
	 * R4 = vcpu pointer (or NULL)
353 354 355 356 357 358
	 * MSR = ~IR|DR
	 * R13 = PACA
	 * R1 = host R1
	 * all other volatile GPRS = free
	 */
	mflr	r0
359 360
	std	r0, PPC_LR_STKOFF(r1)
	stdu	r1, -112(r1)
361 362 363 364

	/* Save R1 in the PACA */
	std	r1, HSTATE_HOST_R1(r13)

365 366 367
	li	r6, KVM_GUEST_MODE_HOST_HV
	stb	r6, HSTATE_IN_GUEST(r13)

368 369 370 371 372 373
	/* Clear out SLB */
	li	r6,0
	slbmte	r6,r6
	slbia
	ptesync

374 375 376 377 378 379 380 381
BEGIN_FTR_SECTION
	b	30f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	/*
	 * POWER7 host -> guest partition switch code.
	 * We don't have to lock against concurrent tlbies,
	 * but we do have to coordinate across hardware threads.
	 */
382 383 384 385 386 387 388 389 390 391 392
	/* Increment entry count iff exit count is zero. */
	ld	r5,HSTATE_KVM_VCORE(r13)
	addi	r9,r5,VCORE_ENTRY_EXIT
21:	lwarx	r3,0,r9
	cmpwi	r3,0x100		/* any threads starting to exit? */
	bge	secondary_too_late	/* if so we're too late to the party */
	addi	r3,r3,1
	stwcx.	r3,0,r9
	bne	21b

	/* Primary thread switches to guest partition. */
393 394
	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
	lbz	r6,HSTATE_PTID(r13)
395 396
	cmpwi	r6,0
	bne	20f
397 398 399 400 401 402 403 404
	ld	r6,KVM_SDR1(r9)
	lwz	r7,KVM_LPID(r9)
	li	r0,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r0
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync
405 406 407 408 409 410 411 412

	/* See if we need to flush the TLB */
	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
	srdi	r6,r6,6			/* doubleword number */
	sldi	r6,r6,3			/* address offset */
	add	r6,r6,r9
	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
413
	li	r0,1
414 415 416 417 418 419 420 421
	sld	r0,r0,r7
	ld	r7,0(r6)
	and.	r7,r7,r0
	beq	22f
23:	ldarx	r7,0,r6			/* if set, clear the bit */
	andc	r7,r7,r0
	stdcx.	r7,0,r6
	bne	23b
422 423 424 425 426 427 428
	/* Flush the TLB of any entries for this LPID */
	/* use arch 2.07S as a proxy for POWER8 */
BEGIN_FTR_SECTION
	li	r6,512			/* POWER8 has 512 sets */
FTR_SECTION_ELSE
	li	r6,128			/* POWER7 has 128 sets */
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
429 430 431 432 433 434 435 436
	mtctr	r6
	li	r7,0x800		/* IS field = 0b10 */
	ptesync
28:	tlbiel	r7
	addi	r7,r7,0x1000
	bdnz	28b
	ptesync

437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
	/* Add timebase offset onto timebase */
22:	ld	r8,VCORE_TB_OFFSET(r5)
	cmpdi	r8,0
	beq	37f
	mftb	r6		/* current host timebase */
	add	r8,r8,r6
	mtspr	SPRN_TBU40,r8	/* update upper 40 bits */
	mftb	r7		/* check if lower 24 bits overflowed */
	clrldi	r6,r6,40
	clrldi	r7,r7,40
	cmpld	r7,r6
	bge	37f
	addis	r8,r8,0x100	/* if so, increment upper 40 bits */
	mtspr	SPRN_TBU40,r8

452 453 454 455 456 457
	/* Load guest PCR value to select appropriate compat mode */
37:	ld	r7, VCORE_PCR(r5)
	cmpdi	r7, 0
	beq	38f
	mtspr	SPRN_PCR, r7
38:
458 459 460 461 462 463 464

BEGIN_FTR_SECTION
	/* DPDES is shared between threads */
	ld	r8, VCORE_DPDES(r5)
	mtspr	SPRN_DPDES, r8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)

465
	li	r0,1
466 467 468 469 470 471 472
	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
	b	10f

	/* Secondary threads wait for primary to have done partition switch */
20:	lbz	r0,VCORE_IN_GUEST(r5)
	cmpwi	r0,0
	beq	20b
473

474
	/* Set LPCR and RMOR. */
475
10:	ld	r8,VCORE_LPCR(r5)
476
	mtspr	SPRN_LPCR,r8
477 478
	ld	r8,KVM_RMOR(r9)
	mtspr	SPRN_RMOR,r8
479 480 481 482
	isync

	/* Check if HDEC expires soon */
	mfspr	r3,SPRN_HDEC
483
	cmpwi	r3,512		/* 1 microsecond */
484 485
	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	blt	hdec_soon
486 487 488 489 490 491 492 493 494 495
	b	31f

	/*
	 * PPC970 host -> guest partition switch code.
	 * We have to lock against concurrent tlbies,
	 * using native_tlbie_lock to lock against host tlbies
	 * and kvm->arch.tlbie_lock to lock against guest tlbies.
	 * We also have to invalidate the TLB since its
	 * entries aren't tagged with the LPID.
	 */
496 497
30:	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
498 499 500 501 502 503 504

	/* first take native_tlbie_lock */
	.section ".toc","aw"
toc_tlbie_lock:
	.tc	native_tlbie_lock[TC],native_tlbie_lock
	.previous
	ld	r3,toc_tlbie_lock@toc(2)
505
#ifdef __BIG_ENDIAN__
506
	lwz	r8,PACA_LOCK_TOKEN(r13)
507 508 509
#else
	lwz	r8,PACAPACAINDEX(r13)
#endif
510 511 512 513 514 515 516
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

517 518
	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r7,VCORE_LPCR(r5)	/* use vcore->lpcr to store HID4 */
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
	li	r0,0x18f
	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
	or	r0,r7,r0
	ptesync
	sync
	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop native_tlbie_lock */

	/* invalidate the whole TLB */
	li	r0,256
	mtctr	r0
	li	r6,0
25:	tlbiel	r6
	addi	r6,r6,0x1000
	bdnz	25b
	ptesync

	/* Take the guest's tlbie_lock */
	addi	r3,r9,KVM_TLBIE_LOCK
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync
	ld	r6,KVM_SDR1(r9)
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */

	/* Set up HID4 with the guest's LPID etc. */
	sync
	mtspr	SPRN_HID4,r7
	isync

	/* drop the guest's tlbie_lock */
	li	r0,0
	stw	r0,0(r3)

	/* Check if HDEC expires soon */
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,10
	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	blt	hdec_soon

	/* Enable HDEC interrupts */
	mfspr	r0,SPRN_HID0
	li	r3,1
	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
	sync
	mtspr	SPRN_HID0,r0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
576 577 578 579 580
31:
	/* Do we have a guest vcpu to run? */
	cmpdi	r4, 0
	beq	kvmppc_primary_no_guest
kvmppc_got_guest:
581 582

	/* Load up guest SLB entries */
583
	lwz	r5,VCPU_SLB_MAX(r4)
584 585 586 587 588 589 590 591 592 593
	cmpwi	r5,0
	beq	9f
	mtctr	r5
	addi	r6,r4,VCPU_SLB
1:	ld	r8,VCPU_SLB_E(r6)
	ld	r9,VCPU_SLB_V(r6)
	slbmte	r9,r8
	addi	r6,r6,VCPU_SLB_SIZE
	bdnz	1b
9:
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
	/* Increment yield count if they have a VPA */
	ld	r3, VCPU_VPA(r4)
	cmpdi	r3, 0
	beq	25f
	lwz	r5, LPPACA_YIELDCOUNT(r3)
	addi	r5, r5, 1
	stw	r5, LPPACA_YIELDCOUNT(r3)
	li	r6, 1
	stb	r6, VCPU_VPA_DIRTY(r4)
25:

BEGIN_FTR_SECTION
	/* Save purr/spurr */
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	std	r5,HSTATE_PURR(r13)
	std	r6,HSTATE_SPURR(r13)
	ld	r7,VCPU_PURR(r4)
	ld	r8,VCPU_SPURR(r4)
	mtspr	SPRN_PURR,r7
	mtspr	SPRN_SPURR,r8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

BEGIN_FTR_SECTION
	/* Set partition DABR */
	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
620
	lwz	r5,VCPU_DABRX(r4)
621 622 623 624 625 626 627 628
	ld	r6,VCPU_DABR(r4)
	mtspr	SPRN_DABRX,r5
	mtspr	SPRN_DABR,r6
 BEGIN_FTR_SECTION_NESTED(89)
	isync
 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_206, CPU_FTR_ARCH_206, 89)
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	b	skip_tm
END_FTR_SECTION_IFCLR(CPU_FTR_TM)

	/* Turn on TM/FP/VSX/VMX so we can restore them. */
	mfmsr	r5
	li	r6, MSR_TM >> 32
	sldi	r6, r6, 32
	or	r5, r5, r6
	ori	r5, r5, MSR_FP
	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
	mtmsrd	r5

	/*
	 * The user may change these outside of a transaction, so they must
	 * always be context switched.
	 */
	ld	r5, VCPU_TFHAR(r4)
	ld	r6, VCPU_TFIAR(r4)
	ld	r7, VCPU_TEXASR(r4)
	mtspr	SPRN_TFHAR, r5
	mtspr	SPRN_TFIAR, r6
	mtspr	SPRN_TEXASR, r7

	ld	r5, VCPU_MSR(r4)
	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
	beq	skip_tm	/* TM not active in guest */

	/* Make sure the failure summary is set, otherwise we'll program check
	 * when we trechkpt.  It's possible that this might have been not set
	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
	 * host.
	 */
	oris	r7, r7, (TEXASR_FS)@h
	mtspr	SPRN_TEXASR, r7

	/*
	 * We need to load up the checkpointed state for the guest.
	 * We need to do this early as it will blow away any GPRs, VSRs and
	 * some SPRs.
	 */

	mr	r31, r4
	addi	r3, r31, VCPU_FPRS_TM
	bl	.load_fp_state
	addi	r3, r31, VCPU_VRS_TM
	bl	.load_vr_state
	mr	r4, r31
	lwz	r7, VCPU_VRSAVE_TM(r4)
	mtspr	SPRN_VRSAVE, r7

	ld	r5, VCPU_LR_TM(r4)
	lwz	r6, VCPU_CR_TM(r4)
	ld	r7, VCPU_CTR_TM(r4)
	ld	r8, VCPU_AMR_TM(r4)
	ld	r9, VCPU_TAR_TM(r4)
	mtlr	r5
	mtcr	r6
	mtctr	r7
	mtspr	SPRN_AMR, r8
	mtspr	SPRN_TAR, r9

	/*
	 * Load up PPR and DSCR values but don't put them in the actual SPRs
	 * till the last moment to avoid running with userspace PPR and DSCR for
	 * too long.
	 */
	ld	r29, VCPU_DSCR_TM(r4)
	ld	r30, VCPU_PPR_TM(r4)

	std	r2, PACATMSCRATCH(r13) /* Save TOC */

	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
	li	r5, 0
	mtmsrd	r5, 1

	/* Load GPRs r0-r28 */
	reg = 0
	.rept	29
	ld	reg, VCPU_GPRS_TM(reg)(r31)
	reg = reg + 1
	.endr

	mtspr	SPRN_DSCR, r29
	mtspr	SPRN_PPR, r30

	/* Load final GPRs */
	ld	29, VCPU_GPRS_TM(29)(r31)
	ld	30, VCPU_GPRS_TM(30)(r31)
	ld	31, VCPU_GPRS_TM(31)(r31)

	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
	TRECHKPT

	/* Now let's get back the state we need. */
	HMT_MEDIUM
	GET_PACA(r13)
	ld	r29, HSTATE_DSCR(r13)
	mtspr	SPRN_DSCR, r29
	ld	r4, HSTATE_KVM_VCPU(r13)
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATMSCRATCH(r13)

	/* Set the MSR RI since we have our registers back. */
	li	r5, MSR_RI
	mtmsrd	r5, 1
skip_tm:
#endif

739 740 741 742 743 744
	/* Load guest PMU registers */
	/* R4 is live here (vcpu pointer) */
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	isync
745 746 747 748 749 750
BEGIN_FTR_SECTION
	ld	r3, VCPU_MMCR(r4)
	andi.	r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
	cmpwi	r5, MMCR0_PMAO
	beql	kvmppc_fix_pmao
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
	lwz	r6, VCPU_PMC + 8(r4)
	lwz	r7, VCPU_PMC + 12(r4)
	lwz	r8, VCPU_PMC + 16(r4)
	lwz	r9, VCPU_PMC + 20(r4)
BEGIN_FTR_SECTION
	lwz	r10, VCPU_PMC + 24(r4)
	lwz	r11, VCPU_PMC + 28(r4)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r5
	mtspr	SPRN_PMC3, r6
	mtspr	SPRN_PMC4, r7
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
BEGIN_FTR_SECTION
	mtspr	SPRN_PMC7, r10
	mtspr	SPRN_PMC8, r11
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	ld	r3, VCPU_MMCR(r4)
	ld	r5, VCPU_MMCR + 8(r4)
	ld	r6, VCPU_MMCR + 16(r4)
	ld	r7, VCPU_SIAR(r4)
	ld	r8, VCPU_SDAR(r4)
	mtspr	SPRN_MMCR1, r5
	mtspr	SPRN_MMCRA, r6
	mtspr	SPRN_SIAR, r7
	mtspr	SPRN_SDAR, r8
780 781 782 783 784 785 786 787 788 789 790 791
BEGIN_FTR_SECTION
	ld	r5, VCPU_MMCR + 24(r4)
	ld	r6, VCPU_SIER(r4)
	lwz	r7, VCPU_PMC + 24(r4)
	lwz	r8, VCPU_PMC + 28(r4)
	ld	r9, VCPU_MMCR + 32(r4)
	mtspr	SPRN_MMCR2, r5
	mtspr	SPRN_SIER, r6
	mtspr	SPRN_SPMC1, r7
	mtspr	SPRN_SPMC2, r8
	mtspr	SPRN_MMCRS, r9
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
	mtspr	SPRN_MMCR0, r3
	isync

	/* Load up FP, VMX and VSX registers */
	bl	kvmppc_load_fp

	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)

BEGIN_FTR_SECTION
	/* Switch DSCR to guest value */
	ld	r5, VCPU_DSCR(r4)
	mtspr	SPRN_DSCR, r5
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
BEGIN_FTR_SECTION
	/* Skip next section on POWER7 or PPC970 */
	b	8f
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
	/* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
	mfmsr	r8
	li	r0, 1
	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
	mtmsrd	r8

	/* Load up POWER8-specific registers */
	ld	r5, VCPU_IAMR(r4)
	lwz	r6, VCPU_PSPB(r4)
	ld	r7, VCPU_FSCR(r4)
	mtspr	SPRN_IAMR, r5
	mtspr	SPRN_PSPB, r6
	mtspr	SPRN_FSCR, r7
	ld	r5, VCPU_DAWR(r4)
	ld	r6, VCPU_DAWRX(r4)
	ld	r7, VCPU_CIABR(r4)
	ld	r8, VCPU_TAR(r4)
	mtspr	SPRN_DAWR, r5
	mtspr	SPRN_DAWRX, r6
	mtspr	SPRN_CIABR, r7
	mtspr	SPRN_TAR, r8
	ld	r5, VCPU_IC(r4)
	ld	r6, VCPU_VTB(r4)
	mtspr	SPRN_IC, r5
	mtspr	SPRN_VTB, r6
852
	ld	r8, VCPU_EBBHR(r4)
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
	mtspr	SPRN_EBBHR, r8
	ld	r5, VCPU_EBBRR(r4)
	ld	r6, VCPU_BESCR(r4)
	ld	r7, VCPU_CSIGR(r4)
	ld	r8, VCPU_TACR(r4)
	mtspr	SPRN_EBBRR, r5
	mtspr	SPRN_BESCR, r6
	mtspr	SPRN_CSIGR, r7
	mtspr	SPRN_TACR, r8
	ld	r5, VCPU_TCSCR(r4)
	ld	r6, VCPU_ACOP(r4)
	lwz	r7, VCPU_GUEST_PID(r4)
	ld	r8, VCPU_WORT(r4)
	mtspr	SPRN_TCSCR, r5
	mtspr	SPRN_ACOP, r6
	mtspr	SPRN_PID, r7
	mtspr	SPRN_WORT, r8
8:

872 873 874 875
	/*
	 * Set the decrementer to the guest decrementer.
	 */
	ld	r8,VCPU_DEC_EXPIRES(r4)
876 877 878 879
	/* r8 is a host timebase value here, convert to guest TB */
	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r6,VCORE_TB_OFFSET(r5)
	add	r8,r8,r6
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908
	mftb	r7
	subf	r3,r7,r8
	mtspr	SPRN_DEC,r3
	stw	r3,VCPU_DEC(r4)

	ld	r5, VCPU_SPRG0(r4)
	ld	r6, VCPU_SPRG1(r4)
	ld	r7, VCPU_SPRG2(r4)
	ld	r8, VCPU_SPRG3(r4)
	mtspr	SPRN_SPRG0, r5
	mtspr	SPRN_SPRG1, r6
	mtspr	SPRN_SPRG2, r7
	mtspr	SPRN_SPRG3, r8

	/* Load up DAR and DSISR */
	ld	r5, VCPU_DAR(r4)
	lwz	r6, VCPU_DSISR(r4)
	mtspr	SPRN_DAR, r5
	mtspr	SPRN_DSISR, r6

BEGIN_FTR_SECTION
	/* Restore AMR and UAMOR, set AMOR to all 1s */
	ld	r5,VCPU_AMR(r4)
	ld	r6,VCPU_UAMOR(r4)
	li	r7,-1
	mtspr	SPRN_AMR,r5
	mtspr	SPRN_UAMOR,r6
	mtspr	SPRN_AMOR,r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923

	/* Restore state of CTRL run bit; assume 1 on entry */
	lwz	r5,VCPU_CTRL(r4)
	andi.	r5,r5,1
	bne	4f
	mfspr	r6,SPRN_CTRLF
	clrrdi	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
	ld	r6, VCPU_CTR(r4)
	lwz	r7, VCPU_XER(r4)

	mtctr	r6
	mtxer	r7

924
kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
925 926
	ld	r10, VCPU_PC(r4)
	ld	r11, VCPU_MSR(r4)
927 928
	ld	r6, VCPU_SRR0(r4)
	ld	r7, VCPU_SRR1(r4)
929 930
	mtspr	SPRN_SRR0, r6
	mtspr	SPRN_SRR1, r7
931

932
deliver_guest_interrupt:
933
	/* r11 = vcpu->arch.msr & ~MSR_HV */
934 935 936 937
	rldicl	r11, r11, 63 - MSR_HV_LG, 1
	rotldi	r11, r11, 1 + MSR_HV_LG
	ori	r11, r11, MSR_ME

938
	/* Check if we can deliver an external or decrementer interrupt now */
939 940 941 942
	ld	r0, VCPU_PENDING_EXC(r4)
	rldicl	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
	cmpdi	cr1, r0, 0
	andi.	r8, r11, MSR_EE
943
BEGIN_FTR_SECTION
944 945 946 947
	mfspr	r8, SPRN_LPCR
	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
	mtspr	SPRN_LPCR, r8
948 949 950
	isync
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
	beq	5f
951 952 953 954 955 956
	li	r0, BOOK3S_INTERRUPT_EXTERNAL
	bne	cr1, 12f
	mfspr	r0, SPRN_DEC
	cmpwi	r0, 0
	li	r0, BOOK3S_INTERRUPT_DECREMENTER
	bge	5f
957

958
12:	mtspr	SPRN_SRR0, r10
959
	mr	r10,r0
960
	mtspr	SPRN_SRR1, r11
961 962
	mr	r9, r4
	bl	kvmppc_msr_interrupt
963
5:
964

965 966 967 968 969 970 971
/*
 * Required state:
 * R4 = vcpu
 * R10: value for HSRR0
 * R11: value for HSRR1
 * R13 = PACA
 */
972
fast_guest_return:
973 974
	li	r0,0
	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
975 976 977 978
	mtspr	SPRN_HSRR0,r10
	mtspr	SPRN_HSRR1,r11

	/* Activate guest mode, so faults get handled by KVM */
979
	li	r9, KVM_GUEST_MODE_GUEST_HV
980 981 982 983
	stb	r9, HSTATE_IN_GUEST(r13)

	/* Enter guest */

984 985 986 987
BEGIN_FTR_SECTION
	ld	r5, VCPU_CFAR(r4)
	mtspr	SPRN_CFAR, r5
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
988 989 990
BEGIN_FTR_SECTION
	ld	r0, VCPU_PPR(r4)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
991

992 993 994 995 996
	ld	r5, VCPU_LR(r4)
	lwz	r6, VCPU_CR(r4)
	mtlr	r5
	mtcr	r6

997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	ld	r1, VCPU_GPR(R1)(r4)
	ld	r2, VCPU_GPR(R2)(r4)
	ld	r3, VCPU_GPR(R3)(r4)
	ld	r5, VCPU_GPR(R5)(r4)
	ld	r6, VCPU_GPR(R6)(r4)
	ld	r7, VCPU_GPR(R7)(r4)
	ld	r8, VCPU_GPR(R8)(r4)
	ld	r9, VCPU_GPR(R9)(r4)
	ld	r10, VCPU_GPR(R10)(r4)
	ld	r11, VCPU_GPR(R11)(r4)
	ld	r12, VCPU_GPR(R12)(r4)
	ld	r13, VCPU_GPR(R13)(r4)

1010 1011 1012 1013
BEGIN_FTR_SECTION
	mtspr	SPRN_PPR, r0
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
	ld	r0, VCPU_GPR(R0)(r4)
1014
	ld	r4, VCPU_GPR(R4)(r4)
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027

	hrfid
	b	.

/******************************************************************************
 *                                                                            *
 *                               Exit code                                    *
 *                                                                            *
 *****************************************************************************/

/*
 * We come here from the first-level interrupt handlers.
 */
1028 1029
	.globl	kvmppc_interrupt_hv
kvmppc_interrupt_hv:
1030 1031 1032 1033 1034 1035 1036
	/*
	 * Register contents:
	 * R12		= interrupt vector
	 * R13		= PACA
	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
	 * guest R13 saved in SPRN_SCRATCH0
	 */
1037
	std	r9, HSTATE_SCRATCH2(r13)
1038 1039 1040 1041

	lbz	r9, HSTATE_IN_GUEST(r13)
	cmpwi	r9, KVM_GUEST_MODE_HOST_HV
	beq	kvmppc_bad_host_intr
1042 1043
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
	cmpwi	r9, KVM_GUEST_MODE_GUEST
1044
	ld	r9, HSTATE_SCRATCH2(r13)
1045 1046
	beq	kvmppc_interrupt_pr
#endif
1047 1048 1049 1050
	/* We're now back in the host but in guest MMU context */
	li	r9, KVM_GUEST_MODE_HOST_HV
	stb	r9, HSTATE_IN_GUEST(r13)

1051 1052 1053 1054
	ld	r9, HSTATE_KVM_VCPU(r13)

	/* Save registers */

1055 1056 1057 1058 1059 1060 1061 1062 1063
	std	r0, VCPU_GPR(R0)(r9)
	std	r1, VCPU_GPR(R1)(r9)
	std	r2, VCPU_GPR(R2)(r9)
	std	r3, VCPU_GPR(R3)(r9)
	std	r4, VCPU_GPR(R4)(r9)
	std	r5, VCPU_GPR(R5)(r9)
	std	r6, VCPU_GPR(R6)(r9)
	std	r7, VCPU_GPR(R7)(r9)
	std	r8, VCPU_GPR(R8)(r9)
1064
	ld	r0, HSTATE_SCRATCH2(r13)
1065 1066 1067
	std	r0, VCPU_GPR(R9)(r9)
	std	r10, VCPU_GPR(R10)(r9)
	std	r11, VCPU_GPR(R11)(r9)
1068 1069
	ld	r3, HSTATE_SCRATCH0(r13)
	lwz	r4, HSTATE_SCRATCH1(r13)
1070
	std	r3, VCPU_GPR(R12)(r9)
1071
	stw	r4, VCPU_CR(r9)
1072 1073 1074 1075
BEGIN_FTR_SECTION
	ld	r3, HSTATE_CFAR(r13)
	std	r3, VCPU_CFAR(r9)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1076 1077 1078 1079
BEGIN_FTR_SECTION
	ld	r4, HSTATE_PPR(r13)
	std	r4, VCPU_PPR(r9)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098

	/* Restore R1/R2 so we can handle faults */
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATOC(r13)

	mfspr	r10, SPRN_SRR0
	mfspr	r11, SPRN_SRR1
	std	r10, VCPU_SRR0(r9)
	std	r11, VCPU_SRR1(r9)
	andi.	r0, r12, 2		/* need to read HSRR0/1? */
	beq	1f
	mfspr	r10, SPRN_HSRR0
	mfspr	r11, SPRN_HSRR1
	clrrdi	r12, r12, 2
1:	std	r10, VCPU_PC(r9)
	std	r11, VCPU_MSR(r9)

	GET_SCRATCH0(r3)
	mflr	r4
1099
	std	r3, VCPU_GPR(R13)(r9)
1100 1101 1102 1103
	std	r4, VCPU_LR(r9)

	stw	r12,VCPU_TRAP(r9)

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
	/* Save HEIR (HV emulation assist reg) in last_inst
	   if this is an HEI (HV emulation interrupt, e40) */
	li	r3,KVM_INST_FETCH_FAILED
BEGIN_FTR_SECTION
	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
	bne	11f
	mfspr	r3,SPRN_HEIR
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
11:	stw	r3,VCPU_LAST_INST(r9)

	/* these are volatile across C function calls */
	mfctr	r3
	mfxer	r4
	std	r3, VCPU_CTR(r9)
	stw	r4, VCPU_XER(r9)

BEGIN_FTR_SECTION
	/* If this is a page table miss then see if it's theirs or ours */
	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	beq	kvmppc_hdsi
1124 1125
	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	beq	kvmppc_hisi
1126 1127
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

1128 1129 1130 1131 1132 1133 1134
	/* See if this is a leftover HDEC interrupt */
	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	bne	2f
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,0
	bge	ignore_hdec
2:
1135
	/* See if this is an hcall we can handle in real mode */
1136 1137
	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
	beq	hcall_try_real_mode
1138

1139
	/* Only handle external interrupts here on arch 206 and later */
1140
BEGIN_FTR_SECTION
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
	b	ext_interrupt_to_host
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)

	/* External interrupt ? */
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
	bne+	ext_interrupt_to_host

	/* External interrupt, first check for host_ipi. If this is
	 * set, we know the host wants us out so let's do it now
	 */
1151 1152 1153
	bl	kvmppc_read_intr
	cmpdi	r3, 0
	bgt	ext_interrupt_to_host
1154

1155 1156 1157 1158 1159 1160
	/* Check if any CPU is heading out to the host, if so head out too */
	ld	r5, HSTATE_KVM_VCORE(r13)
	lwz	r0, VCORE_ENTRY_EXIT(r5)
	cmpwi	r0, 0x100
	bge	ext_interrupt_to_host

1161 1162 1163
	/* Return to guest after delivering any pending interrupt */
	mr	r4, r9
	b	deliver_guest_interrupt
1164 1165

ext_interrupt_to_host:
1166

1167
guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
1168 1169 1170 1171 1172
	/* Save more register state  */
	mfdar	r6
	mfdsisr	r7
	std	r6, VCPU_DAR(r9)
	stw	r7, VCPU_DSISR(r9)
1173
BEGIN_FTR_SECTION
1174
	/* don't overwrite fault_dar/fault_dsisr if HDSI */
1175 1176
	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
	beq	6f
1177
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
1178
	std	r6, VCPU_FAULT_DAR(r9)
1179 1180
	stw	r7, VCPU_FAULT_DSISR(r9)

1181 1182 1183 1184 1185
	/* See if it is a machine check */
	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	machine_check_realmode
mc_cont:

1186
	/* Save guest CTRL register, set runlatch to 1 */
1187
6:	mfspr	r6,SPRN_CTRLF
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
	stw	r6,VCPU_CTRL(r9)
	andi.	r0,r6,1
	bne	4f
	ori	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
	/* Read the guest SLB and save it away */
	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
	mtctr	r0
	li	r6,0
	addi	r7,r9,VCPU_SLB
	li	r5,0
1:	slbmfee	r8,r6
	andis.	r0,r8,SLB_ESID_V@h
	beq	2f
	add	r8,r8,r6		/* put index in */
	slbmfev	r3,r6
	std	r8,VCPU_SLB_E(r7)
	std	r3,VCPU_SLB_V(r7)
	addi	r7,r7,VCPU_SLB_SIZE
	addi	r5,r5,1
2:	addi	r6,r6,1
	bdnz	1b
	stw	r5,VCPU_SLB_MAX(r9)

	/*
	 * Save the guest PURR/SPURR
	 */
1216
BEGIN_FTR_SECTION
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	ld	r7,VCPU_PURR(r9)
	ld	r8,VCPU_SPURR(r9)
	std	r5,VCPU_PURR(r9)
	std	r6,VCPU_SPURR(r9)
	subf	r5,r7,r5
	subf	r6,r8,r6

	/*
	 * Restore host PURR/SPURR and add guest times
	 * so that the time in the guest gets accounted.
	 */
	ld	r3,HSTATE_PURR(r13)
	ld	r4,HSTATE_SPURR(r13)
	add	r3,r3,r5
	add	r4,r4,r6
	mtspr	SPRN_PURR,r3
	mtspr	SPRN_SPURR,r4
1236
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
1237

1238 1239 1240 1241 1242
	/* Save DEC */
	mfspr	r5,SPRN_DEC
	mftb	r6
	extsw	r5,r5
	add	r5,r5,r6
1243 1244 1245 1246
	/* r5 is a guest timebase value here, convert to host TB */
	ld	r3,HSTATE_KVM_VCORE(r13)
	ld	r4,VCORE_TB_OFFSET(r3)
	subf	r5,r4,r5
1247 1248
	std	r5,VCPU_DEC_EXPIRES(r9)

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
BEGIN_FTR_SECTION
	b	8f
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
	/* Save POWER8-specific registers */
	mfspr	r5, SPRN_IAMR
	mfspr	r6, SPRN_PSPB
	mfspr	r7, SPRN_FSCR
	std	r5, VCPU_IAMR(r9)
	stw	r6, VCPU_PSPB(r9)
	std	r7, VCPU_FSCR(r9)
	mfspr	r5, SPRN_IC
	mfspr	r6, SPRN_VTB
	mfspr	r7, SPRN_TAR
	std	r5, VCPU_IC(r9)
	std	r6, VCPU_VTB(r9)
	std	r7, VCPU_TAR(r9)
1265
	mfspr	r8, SPRN_EBBHR
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
	std	r8, VCPU_EBBHR(r9)
	mfspr	r5, SPRN_EBBRR
	mfspr	r6, SPRN_BESCR
	mfspr	r7, SPRN_CSIGR
	mfspr	r8, SPRN_TACR
	std	r5, VCPU_EBBRR(r9)
	std	r6, VCPU_BESCR(r9)
	std	r7, VCPU_CSIGR(r9)
	std	r8, VCPU_TACR(r9)
	mfspr	r5, SPRN_TCSCR
	mfspr	r6, SPRN_ACOP
	mfspr	r7, SPRN_PID
	mfspr	r8, SPRN_WORT
	std	r5, VCPU_TCSCR(r9)
	std	r6, VCPU_ACOP(r9)
	stw	r7, VCPU_GUEST_PID(r9)
	std	r8, VCPU_WORT(r9)
8:

1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
	/* Save and reset AMR and UAMOR before turning on the MMU */
BEGIN_FTR_SECTION
	mfspr	r5,SPRN_AMR
	mfspr	r6,SPRN_UAMOR
	std	r5,VCPU_AMR(r9)
	std	r6,VCPU_UAMOR(r9)
	li	r6,0
	mtspr	SPRN_AMR,r6
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Switch DSCR back to host value */
BEGIN_FTR_SECTION
	mfspr	r8, SPRN_DSCR
	ld	r7, HSTATE_DSCR(r13)
	std	r8, VCPU_DSCR(r9)
	mtspr	SPRN_DSCR, r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Save non-volatile GPRs */
	std	r14, VCPU_GPR(R14)(r9)
	std	r15, VCPU_GPR(R15)(r9)
	std	r16, VCPU_GPR(R16)(r9)
	std	r17, VCPU_GPR(R17)(r9)
	std	r18, VCPU_GPR(R18)(r9)
	std	r19, VCPU_GPR(R19)(r9)
	std	r20, VCPU_GPR(R20)(r9)
	std	r21, VCPU_GPR(R21)(r9)
	std	r22, VCPU_GPR(R22)(r9)
	std	r23, VCPU_GPR(R23)(r9)
	std	r24, VCPU_GPR(R24)(r9)
	std	r25, VCPU_GPR(R25)(r9)
	std	r26, VCPU_GPR(R26)(r9)
	std	r27, VCPU_GPR(R27)(r9)
	std	r28, VCPU_GPR(R28)(r9)
	std	r29, VCPU_GPR(R29)(r9)
	std	r30, VCPU_GPR(R30)(r9)
	std	r31, VCPU_GPR(R31)(r9)

	/* Save SPRGs */
	mfspr	r3, SPRN_SPRG0
	mfspr	r4, SPRN_SPRG1
	mfspr	r5, SPRN_SPRG2
	mfspr	r6, SPRN_SPRG3
	std	r3, VCPU_SPRG0(r9)
	std	r4, VCPU_SPRG1(r9)
	std	r5, VCPU_SPRG2(r9)
	std	r6, VCPU_SPRG3(r9)

	/* save FP state */
	mr	r3, r9
	bl	kvmppc_save_fp
1336

1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	b	2f
END_FTR_SECTION_IFCLR(CPU_FTR_TM)
	/* Turn on TM. */
	mfmsr	r8
	li	r0, 1
	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
	mtmsrd	r8

	ld	r5, VCPU_MSR(r9)
	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
	beq	1f	/* TM not active in guest. */

	li	r3, TM_CAUSE_KVM_RESCHED

	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
	li	r5, 0
	mtmsrd	r5, 1

	/* All GPRs are volatile at this point. */
	TRECLAIM(R3)

	/* Temporarily store r13 and r9 so we have some regs to play with */
	SET_SCRATCH0(r13)
	GET_PACA(r13)
	std	r9, PACATMSCRATCH(r13)
	ld	r9, HSTATE_KVM_VCPU(r13)

	/* Get a few more GPRs free. */
	std	r29, VCPU_GPRS_TM(29)(r9)
	std	r30, VCPU_GPRS_TM(30)(r9)
	std	r31, VCPU_GPRS_TM(31)(r9)

	/* Save away PPR and DSCR soon so don't run with user values. */
	mfspr	r31, SPRN_PPR
	HMT_MEDIUM
	mfspr	r30, SPRN_DSCR
	ld	r29, HSTATE_DSCR(r13)
	mtspr	SPRN_DSCR, r29

	/* Save all but r9, r13 & r29-r31 */
	reg = 0
	.rept	29
	.if (reg != 9) && (reg != 13)
	std	reg, VCPU_GPRS_TM(reg)(r9)
	.endif
	reg = reg + 1
	.endr
	/* ... now save r13 */
	GET_SCRATCH0(r4)
	std	r4, VCPU_GPRS_TM(13)(r9)
	/* ... and save r9 */
	ld	r4, PACATMSCRATCH(r13)
	std	r4, VCPU_GPRS_TM(9)(r9)

	/* Reload stack pointer and TOC. */
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATOC(r13)

	/* Set MSR RI now we have r1 and r13 back. */
	li	r5, MSR_RI
	mtmsrd	r5, 1

	/* Save away checkpinted SPRs. */
	std	r31, VCPU_PPR_TM(r9)
	std	r30, VCPU_DSCR_TM(r9)
	mflr	r5
	mfcr	r6
	mfctr	r7
	mfspr	r8, SPRN_AMR
	mfspr	r10, SPRN_TAR
	std	r5, VCPU_LR_TM(r9)
	stw	r6, VCPU_CR_TM(r9)
	std	r7, VCPU_CTR_TM(r9)
	std	r8, VCPU_AMR_TM(r9)
	std	r10, VCPU_TAR_TM(r9)

	/* Restore r12 as trap number. */
	lwz	r12, VCPU_TRAP(r9)

	/* Save FP/VSX. */
	addi	r3, r9, VCPU_FPRS_TM
	bl	.store_fp_state
	addi	r3, r9, VCPU_VRS_TM
	bl	.store_vr_state
	mfspr	r6, SPRN_VRSAVE
	stw	r6, VCPU_VRSAVE_TM(r9)
1:
	/*
	 * We need to save these SPRs after the treclaim so that the software
	 * error code is recorded correctly in the TEXASR.  Also the user may
	 * change these outside of a transaction, so they must always be
	 * context switched.
	 */
	mfspr	r5, SPRN_TFHAR
	mfspr	r6, SPRN_TFIAR
	mfspr	r7, SPRN_TEXASR
	std	r5, VCPU_TFHAR(r9)
	std	r6, VCPU_TFIAR(r9)
	std	r7, VCPU_TEXASR(r9)
2:
#endif

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
	/* Increment yield count if they have a VPA */
	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
	cmpdi	r8, 0
	beq	25f
	lwz	r3, LPPACA_YIELDCOUNT(r8)
	addi	r3, r3, 1
	stw	r3, LPPACA_YIELDCOUNT(r8)
	li	r3, 1
	stb	r3, VCPU_VPA_DIRTY(r9)
25:
	/* Save PMU registers if requested */
	/* r8 and cr0.eq are live here */
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
BEGIN_FTR_SECTION
	/*
	 * POWER8 seems to have a hardware bug where setting
	 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
	 * when some counters are already negative doesn't seem
	 * to cause a performance monitor alert (and hence interrupt).
	 * The effect of this is that when saving the PMU state,
	 * if there is no PMU alert pending when we read MMCR0
	 * before freezing the counters, but one becomes pending
	 * before we read the counters, we lose it.
	 * To work around this, we need a way to freeze the counters
	 * before reading MMCR0.  Normally, freezing the counters
	 * is done by writing MMCR0 (to set MMCR0[FC]) which
	 * unavoidably writes MMCR0[PMA0] as well.  On POWER8,
	 * we can also freeze the counters using MMCR2, by writing
	 * 1s to all the counter freeze condition bits (there are
	 * 9 bits each for 6 counters).
	 */
	li	r3, -1			/* set all freeze bits */
	clrrdi	r3, r3, 10
	mfspr	r10, SPRN_MMCR2
	mtspr	SPRN_MMCR2, r3
	isync
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	mfspr	r6, SPRN_MMCRA
BEGIN_FTR_SECTION
	/* On P7, clear MMCRA in order to disable SDAR updates */
	li	r7, 0
	mtspr	SPRN_MMCRA, r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
	isync
	beq	21f			/* if no VPA, save PMU stuff anyway */
	lbz	r7, LPPACA_PMCINUSE(r8)
	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
	bne	21f
	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
	b	22f
21:	mfspr	r5, SPRN_MMCR1
	mfspr	r7, SPRN_SIAR
	mfspr	r8, SPRN_SDAR
	std	r4, VCPU_MMCR(r9)
	std	r5, VCPU_MMCR + 8(r9)
	std	r6, VCPU_MMCR + 16(r9)
1500 1501 1502
BEGIN_FTR_SECTION
	std	r10, VCPU_MMCR + 24(r9)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
	std	r7, VCPU_SIAR(r9)
	std	r8, VCPU_SDAR(r9)
	mfspr	r3, SPRN_PMC1
	mfspr	r4, SPRN_PMC2
	mfspr	r5, SPRN_PMC3
	mfspr	r6, SPRN_PMC4
	mfspr	r7, SPRN_PMC5
	mfspr	r8, SPRN_PMC6
BEGIN_FTR_SECTION
	mfspr	r10, SPRN_PMC7
	mfspr	r11, SPRN_PMC8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	stw	r3, VCPU_PMC(r9)
	stw	r4, VCPU_PMC + 4(r9)
	stw	r5, VCPU_PMC + 8(r9)
	stw	r6, VCPU_PMC + 12(r9)
	stw	r7, VCPU_PMC + 16(r9)
	stw	r8, VCPU_PMC + 20(r9)
BEGIN_FTR_SECTION
	stw	r10, VCPU_PMC + 24(r9)
	stw	r11, VCPU_PMC + 28(r9)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
BEGIN_FTR_SECTION
	mfspr	r5, SPRN_SIER
	mfspr	r6, SPRN_SPMC1
	mfspr	r7, SPRN_SPMC2
	mfspr	r8, SPRN_MMCRS
	std	r5, VCPU_SIER(r9)
	stw	r6, VCPU_PMC + 24(r9)
	stw	r7, VCPU_PMC + 28(r9)
	std	r8, VCPU_MMCR + 32(r9)
	lis	r4, 0x8000
	mtspr	SPRN_MMCRS, r4
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1537
22:
1538 1539 1540 1541 1542 1543
	/* Clear out SLB */
	li	r5,0
	slbmte	r5,r5
	slbia
	ptesync

1544
hdec_soon:			/* r12 = trap, r13 = paca */
1545 1546 1547 1548 1549 1550 1551 1552
BEGIN_FTR_SECTION
	b	32f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	/*
	 * POWER7 guest -> host partition switch code.
	 * We don't have to lock against tlbies but we do
	 * have to coordinate the hardware threads.
	 */
1553 1554 1555 1556 1557 1558 1559 1560
	/* Increment the threads-exiting-guest count in the 0xff00
	   bits of vcore->entry_exit_count */
	ld	r5,HSTATE_KVM_VCORE(r13)
	addi	r6,r5,VCORE_ENTRY_EXIT
41:	lwarx	r3,0,r6
	addi	r0,r3,0x100
	stwcx.	r0,0,r6
	bne	41b
1561
	isync		/* order stwcx. vs. reading napping_threads */
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574

	/*
	 * At this point we have an interrupt that we have to pass
	 * up to the kernel or qemu; we can't handle it in real mode.
	 * Thus we have to do a partition switch, so we have to
	 * collect the other threads, if we are the first thread
	 * to take an interrupt.  To do this, we set the HDEC to 0,
	 * which causes an HDEC interrupt in all threads within 2ns
	 * because the HDEC register is shared between all 4 threads.
	 * However, we don't need to bother if this is an HDEC
	 * interrupt, since the other threads will already be on their
	 * way here in that case.
	 */
1575 1576
	cmpwi	r3,0x100	/* Are we the first here? */
	bge	43f
1577 1578 1579 1580 1581
	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	beq	40f
	li	r0,0
	mtspr	SPRN_HDEC,r0
40:
1582 1583 1584 1585 1586
	/*
	 * Send an IPI to any napping threads, since an HDEC interrupt
	 * doesn't wake CPUs up from nap.
	 */
	lwz	r3,VCORE_NAPPING_THREADS(r5)
1587
	lbz	r4,HSTATE_PTID(r13)
1588
	li	r0,1
1589
	sld	r0,r0,r4
1590 1591
	andc.	r3,r3,r0		/* no sense IPI'ing ourselves */
	beq	43f
1592 1593
	/* Order entry/exit update vs. IPIs */
	sync
1594 1595 1596 1597 1598 1599
	mulli	r4,r4,PACA_SIZE		/* get paca for thread 0 */
	subf	r6,r4,r13
42:	andi.	r0,r3,1
	beq	44f
	ld	r8,HSTATE_XICS_PHYS(r6)	/* get thread's XICS reg addr */
	li	r0,IPI_PRIORITY
1600
	li	r7,XICS_MFRR
1601 1602 1603 1604
	stbcix	r0,r7,r8		/* trigger the IPI */
44:	srdi.	r3,r3,1
	addi	r6,r6,PACA_SIZE
	bne	42b
1605

1606
secondary_too_late:
1607
	/* Secondary threads wait for primary to do partition switch */
1608 1609 1610
43:	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
	lbz	r3,HSTATE_PTID(r13)
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
	cmpwi	r3,0
	beq	15f
	HMT_LOW
13:	lbz	r3,VCORE_IN_GUEST(r5)
	cmpwi	r3,0
	bne	13b
	HMT_MEDIUM
	b	16f

	/* Primary thread waits for all the secondaries to exit guest */
15:	lwz	r3,VCORE_ENTRY_EXIT(r5)
	srwi	r0,r3,8
	clrldi	r3,r3,56
	cmpw	r3,r0
	bne	15b
	isync

	/* Primary thread switches back to host partition */
1629 1630 1631 1632 1633 1634 1635 1636
	ld	r6,KVM_HOST_SDR1(r4)
	lwz	r7,KVM_HOST_LPID(r4)
	li	r8,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r8
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync
1637

1638 1639 1640 1641 1642 1643 1644 1645 1646
BEGIN_FTR_SECTION
	/* DPDES is shared between threads */
	mfspr	r7, SPRN_DPDES
	std	r7, VCORE_DPDES(r5)
	/* clear DPDES so we don't get guest doorbells in the host */
	li	r8, 0
	mtspr	SPRN_DPDES, r8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)

1647 1648 1649 1650
	/* Subtract timebase offset from timebase */
	ld	r8,VCORE_TB_OFFSET(r5)
	cmpdi	r8,0
	beq	17f
1651
	mftb	r6			/* current guest timebase */
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	subf	r8,r8,r6
	mtspr	SPRN_TBU40,r8		/* update upper 40 bits */
	mftb	r7			/* check if lower 24 bits overflowed */
	clrldi	r6,r6,40
	clrldi	r7,r7,40
	cmpld	r7,r6
	bge	17f
	addis	r8,r8,0x100		/* if so, increment upper 40 bits */
	mtspr	SPRN_TBU40,r8

1662 1663 1664 1665 1666 1667 1668
	/* Reset PCR */
17:	ld	r0, VCORE_PCR(r5)
	cmpdi	r0, 0
	beq	18f
	li	r0, 0
	mtspr	SPRN_PCR, r0
18:
1669
	/* Signal secondary CPUs to continue */
1670
	stb	r0,VCORE_IN_GUEST(r5)
1671 1672 1673
	lis	r8,0x7fff		/* MAX_INT@h */
	mtspr	SPRN_HDEC,r8

1674
16:	ld	r8,KVM_HOST_LPCR(r4)
1675 1676
	mtspr	SPRN_LPCR,r8
	isync
1677 1678 1679 1680 1681 1682 1683
	b	33f

	/*
	 * PPC970 guest -> host partition switch code.
	 * We have to lock against concurrent tlbies, and
	 * we have to flush the whole TLB.
	 */
1684 1685
32:	ld	r5,HSTATE_KVM_VCORE(r13)
	ld	r4,VCORE_KVM(r5)	/* pointer to struct kvm */
1686 1687

	/* Take the guest's tlbie_lock */
1688
#ifdef __BIG_ENDIAN__
1689
	lwz	r8,PACA_LOCK_TOKEN(r13)
1690 1691 1692
#else
	lwz	r8,PACAPACAINDEX(r13)
#endif
1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
	addi	r3,r4,KVM_TLBIE_LOCK
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

	ld	r7,KVM_HOST_LPCR(r4)	/* use kvm->arch.host_lpcr for HID4 */
	li	r0,0x18f
	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
	or	r0,r7,r0
	ptesync
	sync
	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop guest tlbie_lock */

	/* invalidate the whole TLB */
	li	r0,256
	mtctr	r0
	li	r6,0
25:	tlbiel	r6
	addi	r6,r6,0x1000
	bdnz	25b
	ptesync

	/* take native_tlbie_lock */
	ld	r3,toc_tlbie_lock@toc(2)
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

	ld	r6,KVM_HOST_SDR1(r4)
	mtspr	SPRN_SDR1,r6		/* switch to host page table */

	/* Set up host HID4 value */
	sync
	mtspr	SPRN_HID4,r7
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop native_tlbie_lock */

	lis	r8,0x7fff		/* MAX_INT@h */
	mtspr	SPRN_HDEC,r8

	/* Disable HDEC interrupts */
	mfspr	r0,SPRN_HID0
	li	r3,0
	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
	sync
	mtspr	SPRN_HID0,r0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
1755 1756

	/* load host SLB entries */
1757
33:	ld	r8,PACA_SLBSHADOWPTR(r13)
1758 1759 1760 1761 1762 1763 1764 1765 1766 1767

	.rept	SLB_NUM_BOLTED
	ld	r5,SLBSHADOW_SAVEAREA(r8)
	ld	r6,SLBSHADOW_SAVEAREA+8(r8)
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r8,r8,16
	.endr

1768 1769 1770 1771
	/* Unset guest mode */
	li	r0, KVM_GUEST_MODE_NONE
	stb	r0, HSTATE_IN_GUEST(r13)

1772 1773 1774 1775
	ld	r0, 112+PPC_LR_STKOFF(r1)
	addi	r1, r1, 112
	mtlr	r0
	blr
1776

1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
/*
 * Check whether an HDSI is an HPTE not found fault or something else.
 * If it is an HPTE not found fault that is due to the guest accessing
 * a page that they have mapped but which we have paged out, then
 * we continue on with the guest exit path.  In all other cases,
 * reflect the HDSI to the guest as a DSI.
 */
kvmppc_hdsi:
	mfspr	r4, SPRN_HDAR
	mfspr	r6, SPRN_HDSISR
1787 1788
	/* HPTE not found fault or protection fault? */
	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1789 1790 1791 1792
	beq	1f			/* if not, send it to the guest */
	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
	beq	3f
	clrrdi	r0, r4, 28
1793
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1794 1795 1796 1797 1798 1799
	bne	1f			/* if no SLB entry found */
4:	std	r4, VCPU_FAULT_DAR(r9)
	stw	r6, VCPU_FAULT_DSISR(r9)

	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
1800
	li	r7, 1			/* data fault */
1801
	bl	kvmppc_hpte_hv_fault
1802 1803 1804 1805 1806 1807 1808
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
	beq	6f
	cmpdi	r3, -1			/* handle in kernel mode */
1809
	beq	guest_exit_cont
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
	cmpdi	r3, -2			/* MMIO emulation; need instr word */
	beq	2f

	/* Synthesize a DSI for the guest */
	ld	r4, VCPU_FAULT_DAR(r9)
	mr	r6, r3
1:	mtspr	SPRN_DAR, r4
	mtspr	SPRN_DSISR, r6
	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
	li	r10, BOOK3S_INTERRUPT_DATA_STORAGE
1821
	bl	kvmppc_msr_interrupt
1822
fast_interrupt_c_return:
1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852
6:	ld	r7, VCPU_CTR(r9)
	lwz	r8, VCPU_XER(r9)
	mtctr	r7
	mtxer	r8
	mr	r4, r9
	b	fast_guest_return

3:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r5)
	b	4b

	/* If this is for emulated MMIO, load the instruction word */
2:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */

	/* Set guest mode to 'jump over instruction' so if lwz faults
	 * we'll just continue at the next IP. */
	li	r0, KVM_GUEST_MODE_SKIP
	stb	r0, HSTATE_IN_GUEST(r13)

	/* Do the access with MSR:DR enabled */
	mfmsr	r3
	ori	r4, r3, MSR_DR		/* Enable paging for data */
	mtmsrd	r4
	lwz	r8, 0(r10)
	mtmsrd	r3

	/* Store the result */
	stw	r8, VCPU_LAST_INST(r9)

	/* Unset guest mode. */
1853
	li	r0, KVM_GUEST_MODE_HOST_HV
1854
	stb	r0, HSTATE_IN_GUEST(r13)
1855
	b	guest_exit_cont
1856

1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
/*
 * Similarly for an HISI, reflect it to the guest as an ISI unless
 * it is an HPTE not found fault for a page that we have paged out.
 */
kvmppc_hisi:
	andis.	r0, r11, SRR1_ISI_NOPT@h
	beq	1f
	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
	beq	3f
	clrrdi	r0, r10, 28
1867
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
1868 1869 1870 1871 1872 1873 1874
	bne	1f			/* if no SLB entry found */
4:
	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
	mr	r4, r10
	mr	r6, r11
	li	r7, 0			/* instruction fault */
1875
	bl	kvmppc_hpte_hv_fault
1876 1877 1878 1879 1880
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
1881
	beq	fast_interrupt_c_return
1882
	cmpdi	r3, -1			/* handle in kernel mode */
1883
	beq	guest_exit_cont
1884 1885 1886 1887 1888 1889

	/* Synthesize an ISI for the guest */
	mr	r11, r3
1:	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
	li	r10, BOOK3S_INTERRUPT_INST_STORAGE
1890
	bl	kvmppc_msr_interrupt
1891
	b	fast_interrupt_c_return
1892 1893 1894 1895 1896

3:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r6)
	b	4b

1897 1898 1899 1900 1901 1902 1903 1904
/*
 * Try to handle an hcall in real mode.
 * Returns to the guest if we handle it, or continues on up to
 * the kernel if we can't (i.e. if we don't have a handler for
 * it, or if the handler returns H_TOO_HARD).
 */
	.globl	hcall_try_real_mode
hcall_try_real_mode:
1905
	ld	r3,VCPU_GPR(R3)(r9)
1906
	andi.	r0,r11,MSR_PR
1907 1908
	/* sc 1 from userspace - reflect to guest syscall */
	bne	sc_1_fast_return
1909 1910
	clrrdi	r3,r3,2
	cmpldi	r3,hcall_real_table_end - hcall_real_table
1911
	bge	guest_exit_cont
1912
	LOAD_REG_ADDR(r4, hcall_real_table)
1913
	lwax	r3,r3,r4
1914
	cmpwi	r3,0
1915
	beq	guest_exit_cont
1916 1917
	add	r12,r3,r4
	mtctr	r12
1918
	mr	r3,r9		/* get vcpu pointer */
1919
	ld	r4,VCPU_GPR(R4)(r9)
1920 1921 1922 1923
	bctrl
	cmpdi	r3,H_TOO_HARD
	beq	hcall_real_fallback
	ld	r4,HSTATE_KVM_VCPU(r13)
1924
	std	r3,VCPU_GPR(R3)(r4)
1925 1926 1927 1928
	ld	r10,VCPU_PC(r4)
	ld	r11,VCPU_MSR(r4)
	b	fast_guest_return

1929 1930 1931 1932
sc_1_fast_return:
	mtspr	SPRN_SRR0,r10
	mtspr	SPRN_SRR1,r11
	li	r10, BOOK3S_INTERRUPT_SYSCALL
1933
	bl	kvmppc_msr_interrupt
1934 1935 1936
	mr	r4,r9
	b	fast_guest_return

1937 1938 1939 1940 1941 1942 1943
	/* We've attempted a real mode hcall, but it's punted it back
	 * to userspace.  We need to restore some clobbered volatiles
	 * before resuming the pass-it-to-qemu path */
hcall_real_fallback:
	li	r12,BOOK3S_INTERRUPT_SYSCALL
	ld	r9, HSTATE_KVM_VCPU(r13)

1944
	b	guest_exit_cont
1945 1946 1947 1948

	.globl	hcall_real_table
hcall_real_table:
	.long	0		/* 0 - unused */
1949 1950 1951
	.long	DOTSYM(kvmppc_h_remove) - hcall_real_table
	.long	DOTSYM(kvmppc_h_enter) - hcall_real_table
	.long	DOTSYM(kvmppc_h_read) - hcall_real_table
1952 1953
	.long	0		/* 0x10 - H_CLEAR_MOD */
	.long	0		/* 0x14 - H_CLEAR_REF */
1954 1955 1956
	.long	DOTSYM(kvmppc_h_protect) - hcall_real_table
	.long	DOTSYM(kvmppc_h_get_tce) - hcall_real_table
	.long	DOTSYM(kvmppc_h_put_tce) - hcall_real_table
1957
	.long	0		/* 0x24 - H_SET_SPRG0 */
1958
	.long	DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
	.long	0		/* 0x2c */
	.long	0		/* 0x30 */
	.long	0		/* 0x34 */
	.long	0		/* 0x38 */
	.long	0		/* 0x3c */
	.long	0		/* 0x40 */
	.long	0		/* 0x44 */
	.long	0		/* 0x48 */
	.long	0		/* 0x4c */
	.long	0		/* 0x50 */
	.long	0		/* 0x54 */
	.long	0		/* 0x58 */
	.long	0		/* 0x5c */
	.long	0		/* 0x60 */
1973
#ifdef CONFIG_KVM_XICS
1974 1975 1976
	.long	DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
	.long	DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
	.long	DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1977
	.long	0		/* 0x70 - H_IPOLL */
1978
	.long	DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1979 1980 1981 1982 1983 1984 1985
#else
	.long	0		/* 0x64 - H_EOI */
	.long	0		/* 0x68 - H_CPPR */
	.long	0		/* 0x6c - H_IPI */
	.long	0		/* 0x70 - H_IPOLL */
	.long	0		/* 0x74 - H_XIRR */
#endif
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
	.long	0		/* 0x78 */
	.long	0		/* 0x7c */
	.long	0		/* 0x80 */
	.long	0		/* 0x84 */
	.long	0		/* 0x88 */
	.long	0		/* 0x8c */
	.long	0		/* 0x90 */
	.long	0		/* 0x94 */
	.long	0		/* 0x98 */
	.long	0		/* 0x9c */
	.long	0		/* 0xa0 */
	.long	0		/* 0xa4 */
	.long	0		/* 0xa8 */
	.long	0		/* 0xac */
	.long	0		/* 0xb0 */
	.long	0		/* 0xb4 */
	.long	0		/* 0xb8 */
	.long	0		/* 0xbc */
	.long	0		/* 0xc0 */
	.long	0		/* 0xc4 */
	.long	0		/* 0xc8 */
	.long	0		/* 0xcc */
	.long	0		/* 0xd0 */
	.long	0		/* 0xd4 */
	.long	0		/* 0xd8 */
	.long	0		/* 0xdc */
2012
	.long	DOTSYM(kvmppc_h_cede) - hcall_real_table
2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028
	.long	0		/* 0xe4 */
	.long	0		/* 0xe8 */
	.long	0		/* 0xec */
	.long	0		/* 0xf0 */
	.long	0		/* 0xf4 */
	.long	0		/* 0xf8 */
	.long	0		/* 0xfc */
	.long	0		/* 0x100 */
	.long	0		/* 0x104 */
	.long	0		/* 0x108 */
	.long	0		/* 0x10c */
	.long	0		/* 0x110 */
	.long	0		/* 0x114 */
	.long	0		/* 0x118 */
	.long	0		/* 0x11c */
	.long	0		/* 0x120 */
2029
	.long	DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
2030 2031 2032
	.long	0		/* 0x128 */
	.long	0		/* 0x12c */
	.long	0		/* 0x130 */
2033
	.long	DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
2034 2035
hcall_real_table_end:

2036 2037 2038 2039
ignore_hdec:
	mr	r4,r9
	b	fast_guest_return

2040 2041 2042 2043 2044 2045 2046 2047 2048
_GLOBAL(kvmppc_h_set_xdabr)
	andi.	r0, r5, DABRX_USER | DABRX_KERNEL
	beq	6f
	li	r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
	andc.	r0, r5, r0
	beq	3f
6:	li	r3, H_PARAMETER
	blr

2049
_GLOBAL(kvmppc_h_set_dabr)
2050 2051
	li	r5, DABRX_USER | DABRX_KERNEL
3:
2052 2053 2054
BEGIN_FTR_SECTION
	b	2f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2055
	std	r4,VCPU_DABR(r3)
2056 2057
	stw	r5, VCPU_DABRX(r3)
	mtspr	SPRN_DABRX, r5
2058 2059 2060 2061 2062 2063
	/* Work around P7 bug where DABR can get corrupted on mtspr */
1:	mtspr	SPRN_DABR,r4
	mfspr	r5, SPRN_DABR
	cmpd	r4, r5
	bne	1b
	isync
2064 2065 2066
	li	r3,0
	blr

2067 2068 2069 2070 2071 2072 2073 2074 2075
	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2:	rlwimi	r5, r4, 5, DAWRX_DR | DAWRX_DW
	rlwimi	r5, r4, 1, DAWRX_WT
	clrrdi	r4, r4, 3
	std	r4, VCPU_DAWR(r3)
	std	r5, VCPU_DAWRX(r3)
	mtspr	SPRN_DAWR, r4
	mtspr	SPRN_DAWRX, r5
	li	r3, 0
2076 2077
	blr

2078 2079 2080 2081 2082 2083 2084 2085
_GLOBAL(kvmppc_h_cede)
	ori	r11,r11,MSR_EE
	std	r11,VCPU_MSR(r3)
	li	r0,1
	stb	r0,VCPU_CEDED(r3)
	sync			/* order setting ceded vs. testing prodded */
	lbz	r5,VCPU_PRODDED(r3)
	cmpwi	r5,0
2086
	bne	kvm_cede_prodded
2087 2088 2089
	li	r0,0		/* set trap to 0 to say hcall is handled */
	stw	r0,VCPU_TRAP(r3)
	li	r0,H_SUCCESS
2090
	std	r0,VCPU_GPR(R3)(r3)
2091
BEGIN_FTR_SECTION
2092
	b	kvm_cede_exit	/* just send it up to host on 970 */
2093 2094 2095 2096 2097 2098 2099 2100
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)

	/*
	 * Set our bit in the bitmask of napping threads unless all the
	 * other threads are already napping, in which case we send this
	 * up to the host.
	 */
	ld	r5,HSTATE_KVM_VCORE(r13)
2101
	lbz	r6,HSTATE_PTID(r13)
2102 2103 2104 2105 2106 2107 2108
	lwz	r8,VCORE_ENTRY_EXIT(r5)
	clrldi	r8,r8,56
	li	r0,1
	sld	r0,r0,r6
	addi	r6,r5,VCORE_NAPPING_THREADS
31:	lwarx	r4,0,r6
	or	r4,r4,r0
2109
	PPC_POPCNTW(R7,R4)
2110
	cmpw	r7,r8
2111
	bge	kvm_cede_exit
2112 2113
	stwcx.	r4,0,r6
	bne	31b
2114 2115
	/* order napping_threads update vs testing entry_exit_count */
	isync
2116
	li	r0,NAPPING_CEDE
2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
	stb	r0,HSTATE_NAPPING(r13)
	lwz	r7,VCORE_ENTRY_EXIT(r5)
	cmpwi	r7,0x100
	bge	33f		/* another thread already exiting */

/*
 * Although not specifically required by the architecture, POWER7
 * preserves the following registers in nap mode, even if an SMT mode
 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
 */
	/* Save non-volatile GPRs */
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
	std	r14, VCPU_GPR(R14)(r3)
	std	r15, VCPU_GPR(R15)(r3)
	std	r16, VCPU_GPR(R16)(r3)
	std	r17, VCPU_GPR(R17)(r3)
	std	r18, VCPU_GPR(R18)(r3)
	std	r19, VCPU_GPR(R19)(r3)
	std	r20, VCPU_GPR(R20)(r3)
	std	r21, VCPU_GPR(R21)(r3)
	std	r22, VCPU_GPR(R22)(r3)
	std	r23, VCPU_GPR(R23)(r3)
	std	r24, VCPU_GPR(R24)(r3)
	std	r25, VCPU_GPR(R25)(r3)
	std	r26, VCPU_GPR(R26)(r3)
	std	r27, VCPU_GPR(R27)(r3)
	std	r28, VCPU_GPR(R28)(r3)
	std	r29, VCPU_GPR(R29)(r3)
	std	r30, VCPU_GPR(R30)(r3)
	std	r31, VCPU_GPR(R31)(r3)
2147 2148

	/* save FP state */
2149
	bl	kvmppc_save_fp
2150 2151

	/*
2152
	 * Take a nap until a decrementer or external or doobell interrupt
2153 2154
	 * occurs, with PECE1, PECE0 and PECEDP set in LPCR. Also clear the
	 * runlatch bit before napping.
2155
	 */
2156 2157 2158 2159
	mfspr	r2, SPRN_CTRLF
	clrrdi	r2, r2, 1
	mtspr	SPRN_CTRLT, r2

2160 2161
	li	r0,1
	stb	r0,HSTATE_HWTHREAD_REQ(r13)
2162 2163
	mfspr	r5,SPRN_LPCR
	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
2164 2165 2166
BEGIN_FTR_SECTION
	oris	r5,r5,LPCR_PECEDP@h
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
	mtspr	SPRN_LPCR,r5
	isync
	li	r0, 0
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

2178 2179 2180 2181 2182
33:	mr	r4, r3
	li	r3, 0
	li	r12, 0
	b	34f

2183
kvm_end_cede:
2184 2185 2186
	/* get vcpu pointer */
	ld	r4, HSTATE_KVM_VCPU(r13)

2187 2188 2189 2190 2191 2192 2193
	/* Woken by external or decrementer interrupt */
	ld	r1, HSTATE_HOST_R1(r13)

	/* load up FP state */
	bl	kvmppc_load_fp

	/* Load NV GPRS */
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)
2212 2213 2214
 
	/* Check the wake reason in SRR1 to see why we got here */
	bl	kvmppc_check_wake_reason
2215 2216

	/* clear our bit in vcore->napping_threads */
2217 2218
34:	ld	r5,HSTATE_KVM_VCORE(r13)
	lbz	r7,HSTATE_PTID(r13)
2219
	li	r0,1
2220
	sld	r0,r0,r7
2221 2222 2223 2224 2225 2226 2227 2228
	addi	r6,r5,VCORE_NAPPING_THREADS
32:	lwarx	r7,0,r6
	andc	r7,r7,r0
	stwcx.	r7,0,r6
	bne	32b
	li	r0,0
	stb	r0,HSTATE_NAPPING(r13)

2229 2230
	/* See if the wake reason means we need to exit */
	stw	r12, VCPU_TRAP(r4)
2231
	mr	r9, r4
2232 2233
	cmpdi	r3, 0
	bgt	guest_exit_cont
2234

2235 2236 2237
	/* see if any other thread is already exiting */
	lwz	r0,VCORE_ENTRY_EXIT(r5)
	cmpwi	r0,0x100
2238
	bge	guest_exit_cont
2239

2240
	b	kvmppc_cede_reentry	/* if not go back to guest */
2241 2242

	/* cede when already previously prodded case */
2243 2244
kvm_cede_prodded:
	li	r0,0
2245 2246 2247 2248 2249 2250 2251
	stb	r0,VCPU_PRODDED(r3)
	sync			/* order testing prodded vs. clearing ceded */
	stb	r0,VCPU_CEDED(r3)
	li	r3,H_SUCCESS
	blr

	/* we've ceded but we want to give control to the host */
2252
kvm_cede_exit:
2253
	b	hcall_real_fallback
2254

2255 2256 2257
	/* Try to handle a machine check in real mode */
machine_check_realmode:
	mr	r3, r9		/* get vcpu pointer */
2258
	bl	kvmppc_realmode_machine_check
2259
	nop
2260
	cmpdi	r3, 0		/* Did we handle MCE ? */
2261 2262
	ld	r9, HSTATE_KVM_VCPU(r13)
	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
	/*
	 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
	 * machine check interrupt (set HSRR0 to 0x200). And for handled
	 * errors (no-fatal), just go back to guest execution with current
	 * HSRR0 instead of exiting guest. This new approach will inject
	 * machine check to guest for fatal error causing guest to crash.
	 *
	 * The old code used to return to host for unhandled errors which
	 * was causing guest to hang with soft lockups inside guest and
	 * makes it difficult to recover guest instance.
	 */
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	bne	2f	/* Continue guest execution. */
2277 2278
	/* If not, deliver a machine check.  SRR0/1 are already set */
	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2279
	ld	r11, VCPU_MSR(r9)
2280
	bl	kvmppc_msr_interrupt
2281
2:	b	fast_interrupt_c_return
2282

2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
/*
 * Check the reason we woke from nap, and take appropriate action.
 * Returns:
 *	0 if nothing needs to be done
 *	1 if something happened that needs to be handled by the host
 *	-1 if there was a guest wakeup (IPI)
 *
 * Also sets r12 to the interrupt vector for any interrupt that needs
 * to be handled now by the host (0x500 for external interrupt), or zero.
 */
kvmppc_check_wake_reason:
	mfspr	r6, SPRN_SRR1
2295 2296 2297 2298 2299 2300
BEGIN_FTR_SECTION
	rlwinm	r6, r6, 45-31, 0xf	/* extract wake reason field (P8) */
FTR_SECTION_ELSE
	rlwinm	r6, r6, 45-31, 0xe	/* P7 wake reason field is 3 bits */
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
	cmpwi	r6, 8			/* was it an external interrupt? */
2301 2302 2303 2304 2305 2306
	li	r12, BOOK3S_INTERRUPT_EXTERNAL
	beq	kvmppc_read_intr	/* if so, see what it was */
	li	r3, 0
	li	r12, 0
	cmpwi	r6, 6			/* was it the decrementer? */
	beq	0f
2307 2308 2309
BEGIN_FTR_SECTION
	cmpwi	r6, 5			/* privileged doorbell? */
	beq	0f
2310 2311
	cmpwi	r6, 3			/* hypervisor doorbell? */
	beq	3f
2312
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2313 2314 2315
	li	r3, 1			/* anything else, return 1 */
0:	blr

2316 2317 2318 2319 2320
	/* hypervisor doorbell */
3:	li	r12, BOOK3S_INTERRUPT_H_DOORBELL
	li	r3, 1
	blr

2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
/*
 * Determine what sort of external interrupt is pending (if any).
 * Returns:
 *	0 if no interrupt is pending
 *	1 if an interrupt is pending that needs to be handled by the host
 *	-1 if there was a guest wakeup IPI (which has now been cleared)
 */
kvmppc_read_intr:
	/* see if a host IPI is pending */
	li	r3, 1
	lbz	r0, HSTATE_HOST_IPI(r13)
	cmpwi	r0, 0
	bne	1f
2334

2335 2336
	/* Now read the interrupt from the ICP */
	ld	r6, HSTATE_XICS_PHYS(r13)
2337
	li	r7, XICS_XIRR
2338 2339 2340 2341
	cmpdi	r6, 0
	beq-	1f
	lwzcix	r0, r6, r7
	rlwinm.	r3, r0, 0, 0xffffff
2342
	sync
2343
	beq	1f			/* if nothing pending in the ICP */
2344

2345 2346 2347 2348 2349 2350 2351 2352
	/* We found something in the ICP...
	 *
	 * If it's not an IPI, stash it in the PACA and return to
	 * the host, we don't (yet) handle directing real external
	 * interrupts directly to the guest
	 */
	cmpwi	r3, XICS_IPI		/* if there is, is it an IPI? */
	bne	42f
2353

2354 2355 2356 2357 2358 2359
	/* It's an IPI, clear the MFRR and EOI it */
	li	r3, 0xff
	li	r8, XICS_MFRR
	stbcix	r3, r6, r8		/* clear the IPI */
	stwcix	r0, r6, r7		/* EOI it */
	sync
2360

2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376
	/* We need to re-check host IPI now in case it got set in the
	 * meantime. If it's clear, we bounce the interrupt to the
	 * guest
	 */
	lbz	r0, HSTATE_HOST_IPI(r13)
	cmpwi	r0, 0
	bne-	43f

	/* OK, it's an IPI for us */
	li	r3, -1
1:	blr

42:	/* It's not an IPI and it's for the host, stash it in the PACA
	 * before exit, it will be picked up by the host ICP driver
	 */
	stw	r0, HSTATE_SAVED_XIRR(r13)
2377
	li	r3, 1
2378 2379 2380 2381 2382 2383
	b	1b

43:	/* We raced with the host, we need to resend that IPI, bummer */
	li	r0, IPI_PRIORITY
	stbcix	r0, r6, r8		/* set the IPI */
	sync
2384
	li	r3, 1
2385
	b	1b
2386

2387 2388 2389
/*
 * Save away FP, VMX and VSX registers.
 * r3 = vcpu pointer
2390 2391
 * N.B. r30 and r31 are volatile across this function,
 * thus it is not callable from C.
2392
 */
2393 2394 2395
kvmppc_save_fp:
	mflr	r30
	mr	r31,r3
2396 2397
	mfmsr	r5
	ori	r8,r5,MSR_FP
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
	isync
2410 2411
	addi	r3,r3,VCPU_FPRS
	bl	.store_fp_state
2412 2413
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
2414 2415
	addi	r3,r31,VCPU_VRS
	bl	.store_vr_state
2416 2417 2418
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
	mfspr	r6,SPRN_VRSAVE
2419
	stw	r6,VCPU_VRSAVE(r31)
2420
	mtlr	r30
2421 2422 2423 2424 2425
	blr

/*
 * Load up FP, VMX and VSX registers
 * r4 = vcpu pointer
2426 2427
 * N.B. r30 and r31 are volatile across this function,
 * thus it is not callable from C.
2428 2429
 */
kvmppc_load_fp:
2430 2431
	mflr	r30
	mr	r31,r4
2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445
	mfmsr	r9
	ori	r8,r9,MSR_FP
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
	isync
2446 2447
	addi	r3,r4,VCPU_FPRS
	bl	.load_fp_state
2448 2449
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
2450 2451
	addi	r3,r31,VCPU_VRS
	bl	.load_vr_state
2452 2453
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
2454
	lwz	r7,VCPU_VRSAVE(r31)
2455
	mtspr	SPRN_VRSAVE,r7
2456 2457
	mtlr	r30
	mr	r4,r31
2458
	blr
2459 2460 2461 2462 2463 2464 2465 2466

/*
 * We come here if we get any exception or interrupt while we are
 * executing host real mode code while in guest MMU context.
 * For now just spin, but we should do something better.
 */
kvmppc_bad_host_intr:
	b	.
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483

/*
 * This mimics the MSR transition on IRQ delivery.  The new guest MSR is taken
 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
 *   r11 has the guest MSR value (in/out)
 *   r9 has a vcpu pointer (in)
 *   r0 is used as a scratch register
 */
kvmppc_msr_interrupt:
	rldicl	r0, r11, 64 - MSR_TS_S_LG, 62
	cmpwi	r0, 2 /* Check if we are in transactional state..  */
	ld	r11, VCPU_INTR_MSR(r9)
	bne	1f
	/* ... if transactional, change to suspended */
	li	r0, 1
1:	rldimi	r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
	blr
2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501

/*
 * This works around a hardware bug on POWER8E processors, where
 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
 * performance monitor interrupt.  Instead, when we need to have
 * an interrupt pending, we have to arrange for a counter to overflow.
 */
kvmppc_fix_pmao:
	li	r3, 0
	mtspr	SPRN_MMCR2, r3
	lis	r3, (MMCR0_PMXE | MMCR0_FCECE)@h
	ori	r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
	mtspr	SPRN_MMCR0, r3
	lis	r3, 0x7fff
	ori	r3, r3, 0xffff
	mtspr	SPRN_PMC6, r3
	isync
	blr