exceptions-64s.S 42.3 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file contains the 64-bit "server" PowerPC variant
 * of the low level exception handling including exception
 * vectors, exception return, part of the slb and stab
 * handling and other fixed offset specific things.
 *
 * This file is meant to be #included from head_64.S due to
L
Lucas De Marchi 已提交
8
 * position dependent assembly.
9 10 11 12 13 14
 *
 * Most of this originates from head_64.S and thus has the same
 * copyright history.
 *
 */

15
#include <asm/hw_irq.h>
16
#include <asm/exception-64s.h>
17
#include <asm/ptrace.h>
18
#include <asm/cpuidle.h>
19

20 21 22
/*
 * We layout physical memory as follows:
 * 0x0000 - 0x00ff : Secondary processor spin code
23 24 25 26
 * 0x0100 - 0x17ff : pSeries Interrupt prologs
 * 0x1800 - 0x4000 : interrupt support common interrupt prologs
 * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
 * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
27
 * 0x7000 - 0x7fff : FWNMI data area
28 29
 * 0x8000 - 0x8fff : Initial (CPU0) segment table
 * 0x9000 -        : Early init and support code
30
 */
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
	/* Syscall routine is used twice, in reloc-off and reloc-on paths */
#define SYSCALL_PSERIES_1 					\
BEGIN_FTR_SECTION						\
	cmpdi	r0,0x1ebe ; 					\
	beq-	1f ;						\
END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\
	mr	r9,r13 ;					\
	GET_PACA(r13) ;						\
	mfspr	r11,SPRN_SRR0 ;					\
0:

#define SYSCALL_PSERIES_2_RFID 					\
	mfspr	r12,SPRN_SRR1 ;					\
	ld	r10,PACAKBASE(r13) ; 				\
	LOAD_HANDLER(r10, system_call_entry) ; 			\
	mtspr	SPRN_SRR0,r10 ; 				\
	ld	r10,PACAKMSR(r13) ;				\
	mtspr	SPRN_SRR1,r10 ; 				\
	rfid ; 							\
	b	. ;	/* prevent speculative execution */

#define SYSCALL_PSERIES_3					\
	/* Fast LE/BE switch system call */			\
1:	mfspr	r12,SPRN_SRR1 ;					\
	xori	r12,r12,MSR_LE ;				\
	mtspr	SPRN_SRR1,r12 ;					\
	rfid ;		/* return to userspace */		\
	b	. ;	/* prevent speculative execution */

60 61
#if defined(CONFIG_RELOCATABLE)
	/*
62 63
	 * We can't branch directly so we do it via the CTR which
	 * is volatile across system calls.
64 65 66 67
	 */
#define SYSCALL_PSERIES_2_DIRECT				\
	mflr	r10 ;						\
	ld	r12,PACAKBASE(r13) ; 				\
68
	LOAD_HANDLER(r12, system_call_entry) ;			\
69
	mtctr	r12 ;						\
70 71 72 73 74
	mfspr	r12,SPRN_SRR1 ;					\
	/* Re-use of r13... No spare regs to do this */	\
	li	r13,MSR_RI ;					\
	mtmsrd 	r13,1 ;						\
	GET_PACA(r13) ;	/* get r13 back */			\
75
	bctr ;
76 77 78 79 80 81
#else
	/* We can branch directly */
#define SYSCALL_PSERIES_2_DIRECT				\
	mfspr	r12,SPRN_SRR1 ;					\
	li	r10,MSR_RI ;					\
	mtmsrd 	r10,1 ;			/* Set RI (EE=0) */	\
82
	b	system_call_common ;
83
#endif
84 85 86 87 88 89 90 91 92 93 94 95 96

/*
 * This is the start of the interrupt handlers for pSeries
 * This code runs with relocation off.
 * Code from here to __end_interrupts gets copied down to real
 * address 0x100 when we are running a relocatable kernel.
 * Therefore any relative branches in this section must only
 * branch to labels in this section.
 */
	. = 0x100
	.globl __start_interrupts
__start_interrupts:

97 98 99 100 101 102
	.globl system_reset_pSeries;
system_reset_pSeries:
	SET_SCRATCH0(r13)
#ifdef CONFIG_PPC_P7_NAP
BEGIN_FTR_SECTION
	/* Running native on arch 2.06 or later, check if we are
103
	 * waking up from nap/sleep/winkle.
104 105
	 */
	mfspr	r13,SPRN_SRR1
106 107 108
	rlwinm.	r13,r13,47-31,30,31
	beq	9f

109 110
	cmpwi	cr3,r13,2

111 112 113 114
	/*
	 * Check if last bit of HSPGR0 is set. This indicates whether we are
	 * waking up from winkle.
	 */
115
	GET_PACA(r13)
116 117 118 119 120
	clrldi	r5,r13,63
	clrrdi	r13,r13,1
	cmpwi	cr4,r5,1
	mtspr	SPRN_HSPRG0,r13

121 122 123 124 125 126 127 128 129 130
	lbz	r0,PACA_THREAD_IDLE_STATE(r13)
	cmpwi   cr2,r0,PNV_THREAD_NAP
	bgt     cr2,8f				/* Either sleep or Winkle */

	/* Waking up from nap should not cause hypervisor state loss */
	bgt	cr3,.

	/* Waking up from nap */
	li	r0,PNV_THREAD_RUNNING
	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */
131

132
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
133 134 135 136 137 138 139
	li	r0,KVM_HWTHREAD_IN_KERNEL
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
	/* Order setting hwthread_state vs. testing hwthread_req */
	sync
	lbz	r0,HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r0,0
	beq	1f
140 141 142 143
	b	kvm_start_guest
1:
#endif

144 145
	/* Return SRR1 from power7_nap() */
	mfspr	r3,SPRN_SRR1
146
	beq	cr3,2f
147 148
	b	power7_wakeup_noloss
2:	b	power7_wakeup_loss
149 150 151

	/* Fast Sleep wakeup on PowerNV */
8:	GET_PACA(r13)
152
	b 	power7_wakeup_tb_loss
153

154
9:
155
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
156
#endif /* CONFIG_PPC_P7_NAP */
157 158
	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
				 NOTEST, 0x100)
159 160

	. = 0x200
161 162 163 164 165
machine_check_pSeries_1:
	/* This is moved out of line as it can be patched by FW, but
	 * some code path might still want to branch into the original
	 * vector
	 */
166
	SET_SCRATCH0(r13)		/* save r13 */
167 168 169 170 171 172 173 174 175
#ifdef CONFIG_PPC_P7_NAP
BEGIN_FTR_SECTION
	/* Running native on arch 2.06 or later, check if we are
	 * waking up from nap. We only handle no state loss and
	 * supervisor state loss. We do -not- handle hypervisor
	 * state loss at this time.
	 */
	mfspr	r13,SPRN_SRR1
	rlwinm.	r13,r13,47-31,30,31
176
	OPT_GET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
177 178
	beq	9f

179 180
	mfspr	r13,SPRN_SRR1
	rlwinm.	r13,r13,47-31,30,31
181 182 183
	/* waking up from powersave (nap) state */
	cmpwi	cr1,r13,2
	/* Total loss of HV state is fatal. let's just stay stuck here */
184
	OPT_GET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
185 186
	bgt	cr1,.
9:
187
	OPT_SET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
188 189
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
#endif /* CONFIG_PPC_P7_NAP */
190
	EXCEPTION_PROLOG_0(PACA_EXMC)
191 192 193
BEGIN_FTR_SECTION
	b	machine_check_pSeries_early
FTR_SECTION_ELSE
194
	b	machine_check_pSeries_0
195
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
196 197 198 199

	. = 0x300
	.globl data_access_pSeries
data_access_pSeries:
200
	SET_SCRATCH0(r13)
201
	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
202
				 KVMTEST, 0x300)
203 204 205 206

	. = 0x380
	.globl data_access_slb_pSeries
data_access_slb_pSeries:
207
	SET_SCRATCH0(r13)
208
	EXCEPTION_PROLOG_0(PACA_EXSLB)
209
	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
210 211
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_DAR
212
	mfspr	r12,SPRN_SRR1
213
#ifndef CONFIG_RELOCATABLE
214
	b	slb_miss_realmode
215 216
#else
	/*
217
	 * We can't just use a direct branch to slb_miss_realmode
218 219 220 221 222
	 * because the distance from here to there depends on where
	 * the kernel ends up being put.
	 */
	mfctr	r11
	ld	r10,PACAKBASE(r13)
223
	LOAD_HANDLER(r10, slb_miss_realmode)
224 225 226 227
	mtctr	r10
	bctr
#endif

228
	STD_EXCEPTION_PSERIES(0x400, instruction_access)
229 230 231 232

	. = 0x480
	.globl instruction_access_slb_pSeries
instruction_access_slb_pSeries:
233
	SET_SCRATCH0(r13)
234
	EXCEPTION_PROLOG_0(PACA_EXSLB)
235
	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x480)
236 237
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
238
	mfspr	r12,SPRN_SRR1
239
#ifndef CONFIG_RELOCATABLE
240
	b	slb_miss_realmode
241 242 243
#else
	mfctr	r11
	ld	r10,PACAKBASE(r13)
244
	LOAD_HANDLER(r10, slb_miss_realmode)
245 246 247 248
	mtctr	r10
	bctr
#endif

249 250 251
	/* We open code these as we can't have a ". = x" (even with
	 * x = "." within a feature section
	 */
252
	. = 0x500;
253 254
	.globl hardware_interrupt_pSeries;
	.globl hardware_interrupt_hv;
255
hardware_interrupt_pSeries:
256
hardware_interrupt_hv:
257
	BEGIN_FTR_SECTION
258 259 260
		_MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
					    EXC_HV, SOFTEN_TEST_HV)
		KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
261 262
	FTR_SECTION_ELSE
		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
263
					    EXC_STD, SOFTEN_TEST_PR)
264
		KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
265
	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
266

267
	STD_EXCEPTION_PSERIES(0x600, alignment)
268
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x600)
269

270
	STD_EXCEPTION_PSERIES(0x700, program_check)
271
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x700)
272

273
	STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
274
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x800)
275

276 277 278 279 280
	. = 0x900
	.globl decrementer_pSeries
decrementer_pSeries:
	_MASKABLE_EXCEPTION_PSERIES(0x900, decrementer, EXC_STD, SOFTEN_TEST_PR)

281
	STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
282

283
	MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
284
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xa00)
285

286
	STD_EXCEPTION_PSERIES(0xb00, trap_0b)
287
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xb00)
288 289 290 291

	. = 0xc00
	.globl	system_call_pSeries
system_call_pSeries:
292 293 294 295 296 297 298
	 /*
	  * If CONFIG_KVM_BOOK3S_64_HANDLER is set, save the PPR (on systems
	  * that support it) before changing to HMT_MEDIUM. That allows the KVM
	  * code to save that value into the guest state (it is the guest's PPR
	  * value). Otherwise just change to HMT_MEDIUM as userspace has
	  * already saved the PPR.
	  */
299 300 301 302
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
	SET_SCRATCH0(r13)
	GET_PACA(r13)
	std	r9,PACA_EXGEN+EX_R9(r13)
303 304
	OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);
	HMT_MEDIUM;
305
	std	r10,PACA_EXGEN+EX_R10(r13)
306
	OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r9, CPU_FTR_HAS_PPR);
307 308 309
	mfcr	r9
	KVMTEST(0xc00)
	GET_SCRATCH0(r13)
310 311
#else
	HMT_MEDIUM;
312
#endif
313 314 315
	SYSCALL_PSERIES_1
	SYSCALL_PSERIES_2_RFID
	SYSCALL_PSERIES_3
316 317
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)

318
	STD_EXCEPTION_PSERIES(0xd00, single_step)
319
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xd00)
320 321 322 323 324

	/* At 0xe??? we have a bunch of hypervisor exceptions, we branch
	 * out of line to handle them
	 */
	. = 0xe00
325
hv_data_storage_trampoline:
326 327
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
328
	b	h_data_storage_hv
329

330
	. = 0xe20
331
hv_instr_storage_trampoline:
332 333
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
334
	b	h_instr_storage_hv
335

336
	. = 0xe40
337
emulation_assist_trampoline:
338 339
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
340
	b	emulation_assist_hv
341

342
	. = 0xe60
343
hv_exception_trampoline:
344 345
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
346
	b	hmi_exception_early
347

348
	. = 0xe80
349
hv_doorbell_trampoline:
350 351
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
352
	b	h_doorbell_hv
353 354 355 356 357 358 359

	/* We need to deal with the Altivec unavailable exception
	 * here which is at 0xf20, thus in the middle of the
	 * prolog code of the PerformanceMonitor one. A little
	 * trickery is thus necessary
	 */
	. = 0xf00
360
performance_monitor_pseries_trampoline:
361 362
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
363 364 365
	b	performance_monitor_pSeries

	. = 0xf20
366
altivec_unavailable_pseries_trampoline:
367 368
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
369 370 371
	b	altivec_unavailable_pSeries

	. = 0xf40
372
vsx_unavailable_pseries_trampoline:
373 374
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
375 376
	b	vsx_unavailable_pSeries

377
	. = 0xf60
378
facility_unavailable_trampoline:
379 380
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
381
	b	facility_unavailable_pSeries
382

383
	. = 0xf80
384
hv_facility_unavailable_trampoline:
385 386 387 388
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
	b	facility_unavailable_hv

389
#ifdef CONFIG_CBE_RAS
390
	STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
391
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
392
#endif /* CONFIG_CBE_RAS */
393

394
	STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
395
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
396

397
	. = 0x1500
M
Michael Neuling 已提交
398
	.global denorm_exception_hv
399 400
denorm_exception_hv:
	mtspr	SPRN_SPRG_HSCRATCH0,r13
401
	EXCEPTION_PROLOG_0(PACA_EXGEN)
402
	EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
403 404 405 406 407 408 409 410 411

#ifdef CONFIG_PPC_DENORMALISATION
	mfspr	r10,SPRN_HSRR1
	mfspr	r11,SPRN_HSRR0		/* save HSRR0 */
	andis.	r10,r10,(HSRR1_DENORM)@h /* denorm? */
	addi	r11,r11,-4		/* HSRR0 is next instruction */
	bne+	denorm_assist
#endif

412
	KVMTEST(0x1500)
413 414 415
	EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)

416
#ifdef CONFIG_CBE_RAS
417
	STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
418
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
419
#endif /* CONFIG_CBE_RAS */
420

421
	STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
422
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x1700)
423

424
#ifdef CONFIG_CBE_RAS
425
	STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
426
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
427 428
#else
	. = 0x1800
429 430 431
#endif /* CONFIG_CBE_RAS */


432 433
/*** Out of line interrupts support ***/

434
	.align	7
435
	/* moved from 0x200 */
436 437 438 439 440 441 442 443 444
machine_check_pSeries_early:
BEGIN_FTR_SECTION
	EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
	/*
	 * Register contents:
	 * R13		= PACA
	 * R9		= CR
	 * Original R9 to R13 is saved on PACA_EXMC
	 *
445 446 447
	 * Switch to mc_emergency stack and handle re-entrancy (we limit
	 * the nested MCE upto level 4 to avoid stack overflow).
	 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
	 *
	 * We use paca->in_mce to check whether this is the first entry or
	 * nested machine check. We increment paca->in_mce to track nested
	 * machine checks.
	 *
	 * If this is the first entry then set stack pointer to
	 * paca->mc_emergency_sp, otherwise r1 is already pointing to
	 * stack frame on mc_emergency stack.
	 *
	 * NOTE: We are here with MSR_ME=0 (off), which means we risk a
	 * checkstop if we get another machine check exception before we do
	 * rfid with MSR_ME=1.
	 */
	mr	r11,r1			/* Save r1 */
	lhz	r10,PACA_IN_MCE(r13)
	cmpwi	r10,0			/* Are we in nested machine check */
	bne	0f			/* Yes, we are. */
	/* First machine check entry */
	ld	r1,PACAMCEMERGSP(r13)	/* Use MC emergency stack */
0:	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame */
	addi	r10,r10,1		/* increment paca->in_mce */
	sth	r10,PACA_IN_MCE(r13)
470 471 472
	/* Limit nested MCE to level 4 to avoid stack overflow */
	cmpwi	r10,4
	bgt	2f			/* Check if we hit limit of 4 */
473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
	std	r11,GPR1(r1)		/* Save r1 on the stack. */
	std	r11,0(r1)		/* make stack chain pointer */
	mfspr	r11,SPRN_SRR0		/* Save SRR0 */
	std	r11,_NIP(r1)
	mfspr	r11,SPRN_SRR1		/* Save SRR1 */
	std	r11,_MSR(r1)
	mfspr	r11,SPRN_DAR		/* Save DAR */
	std	r11,_DAR(r1)
	mfspr	r11,SPRN_DSISR		/* Save DSISR */
	std	r11,_DSISR(r1)
	std	r9,_CCR(r1)		/* Save CR in stackframe */
	/* Save r9 through r13 from EXMC save area to stack frame. */
	EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
	mfmsr	r11			/* get MSR value */
	ori	r11,r11,MSR_ME		/* turn on ME bit */
	ori	r11,r11,MSR_RI		/* turn on RI bit */
	ld	r12,PACAKBASE(r13)	/* get high part of &label */
	LOAD_HANDLER(r12, machine_check_handle_early)
491
1:	mtspr	SPRN_SRR0,r12
492 493 494
	mtspr	SPRN_SRR1,r11
	rfid
	b	.	/* prevent speculative execution */
495 496 497 498 499 500 501 502 503 504 505 506 507
2:
	/* Stack overflow. Stay on emergency stack and panic.
	 * Keep the ME bit off while panic-ing, so that if we hit
	 * another machine check we checkstop.
	 */
	addi	r1,r1,INT_FRAME_SIZE	/* go back to previous stack frame */
	ld	r11,PACAKMSR(r13)
	ld	r12,PACAKBASE(r13)
	LOAD_HANDLER(r12, unrecover_mce)
	li	r10,MSR_ME
	andc	r11,r11,r10		/* Turn off MSR_ME */
	b	1b
	b	.	/* prevent speculative execution */
508 509
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)

510 511 512 513
machine_check_pSeries:
	.globl machine_check_fwnmi
machine_check_fwnmi:
	SET_SCRATCH0(r13)		/* save r13 */
514 515 516 517
	EXCEPTION_PROLOG_0(PACA_EXMC)
machine_check_pSeries_0:
	EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST, 0x200)
	EXCEPTION_PROLOG_PSERIES_1(machine_check_common, EXC_STD)
518
	KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
519 520
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
	KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
521 522 523
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x400)
	KVM_HANDLER(PACA_EXSLB, EXC_STD, 0x480)
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x900)
524 525
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)

526 527 528 529 530 531 532 533 534 535 536 537
#ifdef CONFIG_PPC_DENORMALISATION
denorm_assist:
BEGIN_FTR_SECTION
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER6 do that here for all FP regs.
 */
	mfmsr	r10
	ori	r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
	xori	r10,r10,(MSR_FE0|MSR_FE1)
	mtmsrd	r10
	sync
538 539 540 541 542 543 544 545

#define FMR2(n)  fmr (n), (n) ; fmr n+1, n+1
#define FMR4(n)  FMR2(n) ; FMR2(n+2)
#define FMR8(n)  FMR4(n) ; FMR4(n+4)
#define FMR16(n) FMR8(n) ; FMR8(n+8)
#define FMR32(n) FMR16(n) ; FMR16(n+16)
	FMR32(0)

546 547 548 549 550 551 552 553 554
FTR_SECTION_ELSE
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER7 do that here for the first 32 VSX registers only.
 */
	mfmsr	r10
	oris	r10,r10,MSR_VSX@h
	mtmsrd	r10
	sync
555 556 557 558 559 560 561 562

#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
	XVCPSGNDP32(0)

563
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
564 565 566 567 568 569 570 571 572 573

BEGIN_FTR_SECTION
	b	denorm_done
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER8 we need to do that for all 64 VSX registers
 */
	XVCPSGNDP32(32)
denorm_done:
574 575 576
	mtspr	SPRN_HSRR0,r11
	mtcrf	0x80,r9
	ld	r9,PACA_EXGEN+EX_R9(r13)
577
	RESTORE_PPR_PACA(PACA_EXGEN, r10)
578 579 580 581
BEGIN_FTR_SECTION
	ld	r10,PACA_EXGEN+EX_CFAR(r13)
	mtspr	SPRN_CFAR,r10
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
582 583 584 585 586 587 588 589
	ld	r10,PACA_EXGEN+EX_R10(r13)
	ld	r11,PACA_EXGEN+EX_R11(r13)
	ld	r12,PACA_EXGEN+EX_R12(r13)
	ld	r13,PACA_EXGEN+EX_R13(r13)
	HRFID
	b	.
#endif

590
	.align	7
591
	/* moved from 0xe00 */
592
	STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
593
	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
594
	STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
595
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
596
	STD_EXCEPTION_HV_OOL(0xe42, emulation_assist)
597
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
598
	MASKABLE_EXCEPTION_HV_OOL(0xe62, hmi_exception)
599
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
600

601
	MASKABLE_EXCEPTION_HV_OOL(0xe82, h_doorbell)
602
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
603 604

	/* moved from 0xf00 */
605
	STD_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
606
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf00)
607
	STD_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
608
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf20)
609
	STD_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
610
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf40)
611
	STD_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
612
	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf60)
613 614
	STD_EXCEPTION_HV_OOL(0xf82, facility_unavailable)
	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xf82)
615 616

/*
617 618 619 620
 * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
 * - If it was a decrementer interrupt, we bump the dec to max and and return.
 * - If it was a doorbell we return immediately since doorbells are edge
 *   triggered and won't automatically refire.
621 622
 * - If it was a HMI we return immediately since we handled it in realmode
 *   and it won't refire.
623 624
 * - else we hard disable and return.
 * This is called with r10 containing the value to OR to the paca field.
625
 */
626 627 628 629 630 631
#define MASKED_INTERRUPT(_H)				\
masked_##_H##interrupt:					\
	std	r11,PACA_EXGEN+EX_R11(r13);		\
	lbz	r11,PACAIRQHAPPENED(r13);		\
	or	r11,r11,r10;				\
	stb	r11,PACAIRQHAPPENED(r13);		\
632 633
	cmpwi	r10,PACA_IRQ_DEC;			\
	bne	1f;					\
634 635 636 637
	lis	r10,0x7fff;				\
	ori	r10,r10,0xffff;				\
	mtspr	SPRN_DEC,r10;				\
	b	2f;					\
638
1:	cmpwi	r10,PACA_IRQ_DBELL;			\
639 640
	beq	2f;					\
	cmpwi	r10,PACA_IRQ_HMI;			\
641 642
	beq	2f;					\
	mfspr	r10,SPRN_##_H##SRR1;			\
643 644 645 646 647 648 649 650 651
	rldicl	r10,r10,48,1; /* clear MSR_EE */	\
	rotldi	r10,r10,16;				\
	mtspr	SPRN_##_H##SRR1,r10;			\
2:	mtcrf	0x80,r9;				\
	ld	r9,PACA_EXGEN+EX_R9(r13);		\
	ld	r10,PACA_EXGEN+EX_R10(r13);		\
	ld	r11,PACA_EXGEN+EX_R11(r13);		\
	GET_SCRATCH0(r13);				\
	##_H##rfid;					\
652
	b	.
653 654 655
	
	MASKED_INTERRUPT()
	MASKED_INTERRUPT(H)
656

657 658
/*
 * Called from arch_local_irq_enable when an interrupt needs
659 660
 * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
 * which kind of interrupt. MSR:EE is already off. We generate a
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
 * stackframe like if a real interrupt had happened.
 *
 * Note: While MSR:EE is off, we need to make sure that _MSR
 * in the generated frame has EE set to 1 or the exception
 * handler will not properly re-enable them.
 */
_GLOBAL(__replay_interrupt)
	/* We are going to jump to the exception common code which
	 * will retrieve various register values from the PACA which
	 * we don't give a damn about, so we don't bother storing them.
	 */
	mfmsr	r12
	mflr	r11
	mfcr	r9
	ori	r12,r12,MSR_EE
676 677 678 679 680 681 682 683 684 685 686 687
	cmpwi	r3,0x900
	beq	decrementer_common
	cmpwi	r3,0x500
	beq	hardware_interrupt_common
BEGIN_FTR_SECTION
	cmpwi	r3,0xe80
	beq	h_doorbell_common
FTR_SECTION_ELSE
	cmpwi	r3,0xa00
	beq	doorbell_super_common
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
	blr
688

689 690 691 692 693 694 695
#ifdef CONFIG_PPC_PSERIES
/*
 * Vectors for the FWNMI option.  Share common code.
 */
	.globl system_reset_fwnmi
      .align 7
system_reset_fwnmi:
696
	SET_SCRATCH0(r13)		/* save r13 */
697 698
	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
				 NOTEST, 0x100)
699 700 701

#endif /* CONFIG_PPC_PSERIES */

702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
kvmppc_skip_interrupt:
	/*
	 * Here all GPRs are unchanged from when the interrupt happened
	 * except for r13, which is saved in SPRG_SCRATCH0.
	 */
	mfspr	r13, SPRN_SRR0
	addi	r13, r13, 4
	mtspr	SPRN_SRR0, r13
	GET_SCRATCH0(r13)
	rfid
	b	.

kvmppc_skip_Hinterrupt:
	/*
	 * Here all GPRs are unchanged from when the interrupt happened
	 * except for r13, which is saved in SPRG_SCRATCH0.
	 */
	mfspr	r13, SPRN_HSRR0
	addi	r13, r13, 4
	mtspr	SPRN_HSRR0, r13
	GET_SCRATCH0(r13)
	hrfid
	b	.
#endif

728 729 730 731
/*
 * Code from here down to __end_handlers is invoked from the
 * exception prologs above.  Because the prologs assemble the
 * addresses of these handlers using the LOAD_HANDLER macro,
732 733
 * which uses an ori instruction, these handlers must be in
 * the first 64k of the kernel image.
734 735 736 737
 */

/*** Common interrupt handlers ***/

738
	STD_EXCEPTION_COMMON(0x100, system_reset, system_reset_exception)
739

740
	STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
741 742
	STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, timer_interrupt)
	STD_EXCEPTION_COMMON(0x980, hdecrementer, hdec_interrupt)
743
#ifdef CONFIG_PPC_DOORBELL
744
	STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, doorbell_exception)
745
#else
746
	STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, unknown_exception)
747
#endif
748 749 750 751
	STD_EXCEPTION_COMMON(0xb00, trap_0b, unknown_exception)
	STD_EXCEPTION_COMMON(0xd00, single_step, single_step_exception)
	STD_EXCEPTION_COMMON(0xe00, trap_0e, unknown_exception)
	STD_EXCEPTION_COMMON(0xe40, emulation_assist, emulation_assist_interrupt)
752
	STD_EXCEPTION_COMMON_ASYNC(0xe60, hmi_exception, handle_hmi_exception)
753
#ifdef CONFIG_PPC_DOORBELL
754
	STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, doorbell_exception)
755
#else
756
	STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, unknown_exception)
757
#endif
758 759 760
	STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, performance_monitor_exception)
	STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, instruction_breakpoint_exception)
	STD_EXCEPTION_COMMON(0x1502, denorm, unknown_exception)
761
#ifdef CONFIG_ALTIVEC
762
	STD_EXCEPTION_COMMON(0x1700, altivec_assist, altivec_assist_exception)
763
#else
764
	STD_EXCEPTION_COMMON(0x1700, altivec_assist, unknown_exception)
765 766
#endif
#ifdef CONFIG_CBE_RAS
767 768 769
	STD_EXCEPTION_COMMON(0x1200, cbe_system_error, cbe_system_error_exception)
	STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, cbe_maintenance_exception)
	STD_EXCEPTION_COMMON(0x1800, cbe_thermal, cbe_thermal_exception)
770 771
#endif /* CONFIG_CBE_RAS */

772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
	/*
	 * Relocation-on interrupts: A subset of the interrupts can be delivered
	 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
	 * it.  Addresses are the same as the original interrupt addresses, but
	 * offset by 0xc000000000004000.
	 * It's impossible to receive interrupts below 0x300 via this mechanism.
	 * KVM: None of these traps are from the guest ; anything that escalated
	 * to HV=1 from HV=0 is delivered via real mode handlers.
	 */

	/*
	 * This uses the standard macro, since the original 0x300 vector
	 * only has extra guff for STAB-based processors -- which never
	 * come here.
	 */
	STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
	. = 0x4380
	.globl data_access_slb_relon_pSeries
data_access_slb_relon_pSeries:
	SET_SCRATCH0(r13)
792
	EXCEPTION_PROLOG_0(PACA_EXSLB)
793 794 795 796 797
	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_DAR
	mfspr	r12,SPRN_SRR1
#ifndef CONFIG_RELOCATABLE
798
	b	slb_miss_realmode
799 800
#else
	/*
801
	 * We can't just use a direct branch to slb_miss_realmode
802 803 804 805 806
	 * because the distance from here to there depends on where
	 * the kernel ends up being put.
	 */
	mfctr	r11
	ld	r10,PACAKBASE(r13)
807
	LOAD_HANDLER(r10, slb_miss_realmode)
808 809 810 811 812 813 814 815 816
	mtctr	r10
	bctr
#endif

	STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
	. = 0x4480
	.globl instruction_access_slb_relon_pSeries
instruction_access_slb_relon_pSeries:
	SET_SCRATCH0(r13)
817
	EXCEPTION_PROLOG_0(PACA_EXSLB)
818 819 820 821 822
	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
	mfspr	r12,SPRN_SRR1
#ifndef CONFIG_RELOCATABLE
823
	b	slb_miss_realmode
824 825 826
#else
	mfctr	r11
	ld	r10,PACAKBASE(r13)
827
	LOAD_HANDLER(r10, slb_miss_realmode)
828 829 830 831 832 833 834 835 836 837 838 839 840
	mtctr	r10
	bctr
#endif

	. = 0x4500
	.globl hardware_interrupt_relon_pSeries;
	.globl hardware_interrupt_relon_hv;
hardware_interrupt_relon_pSeries:
hardware_interrupt_relon_hv:
	BEGIN_FTR_SECTION
		_MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
	FTR_SECTION_ELSE
		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
841
	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
842 843 844 845 846
	STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
	STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
	STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
	MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
	STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
847
	MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
848 849 850 851 852 853 854 855 856 857 858 859 860
	STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)

	. = 0x4c00
	.globl system_call_relon_pSeries
system_call_relon_pSeries:
	HMT_MEDIUM
	SYSCALL_PSERIES_1
	SYSCALL_PSERIES_2_DIRECT
	SYSCALL_PSERIES_3

	STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)

	. = 0x4e00
861
	b	.	/* Can't happen, see v2.07 Book III-S section 6.5 */
862 863

	. = 0x4e20
864
	b	.	/* Can't happen, see v2.07 Book III-S section 6.5 */
865 866

	. = 0x4e40
867
emulation_assist_relon_trampoline:
868 869
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
870 871 872
	b	emulation_assist_relon_hv

	. = 0x4e60
873
	b	.	/* Can't happen, see v2.07 Book III-S section 6.5 */
874

875
	. = 0x4e80
876
h_doorbell_relon_trampoline:
877 878
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
879
	b	h_doorbell_relon_hv
880 881

	. = 0x4f00
882
performance_monitor_relon_pseries_trampoline:
883 884
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
885 886 887
	b	performance_monitor_relon_pSeries

	. = 0x4f20
888
altivec_unavailable_relon_pseries_trampoline:
889 890
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
891 892 893
	b	altivec_unavailable_relon_pSeries

	. = 0x4f40
894
vsx_unavailable_relon_pseries_trampoline:
895 896
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
897 898
	b	vsx_unavailable_relon_pSeries

899
	. = 0x4f60
900
facility_unavailable_relon_trampoline:
901 902
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
903
	b	facility_unavailable_relon_pSeries
904

905
	. = 0x4f80
906
hv_facility_unavailable_relon_trampoline:
907 908
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
909
	b	hv_facility_unavailable_relon_hv
910

911 912 913 914 915 916 917 918 919 920 921 922
	STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
#ifdef CONFIG_PPC_DENORMALISATION
	. = 0x5500
	b	denorm_exception_hv
#endif
	STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)

	/* Other future vectors */
	.align	7
	.globl	__end_interrupts
__end_interrupts:

923 924 925 926
	.align	7
system_call_entry:
	b	system_call_common

927
ppc64_runlatch_on_trampoline:
928
	b	__ppc64_runlatch_on
929

930 931 932 933 934 935 936 937 938 939 940 941 942
/*
 * Here r13 points to the paca, r9 contains the saved CR,
 * SRR0 and SRR1 are saved in r11 and r12,
 * r9 - r13 are saved in paca->exgen.
 */
	.align	7
	.globl data_access_common
data_access_common:
	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXGEN+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
943
	RECONCILE_IRQ_STATE(r10, r11)
944
	ld	r12,_MSR(r1)
945 946 947
	ld	r3,PACA_EXGEN+EX_DAR(r13)
	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
	li	r5,0x300
948
	b	do_hash_page		/* Try to handle as hpte fault */
949

950
	.align  7
951
	.globl  h_data_storage_common
952
h_data_storage_common:
953 954 955 956 957
	mfspr   r10,SPRN_HDAR
	std     r10,PACA_EXGEN+EX_DAR(r13)
	mfspr   r10,SPRN_HDSISR
	stw     r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
958
	bl      save_nvgprs
959
	RECONCILE_IRQ_STATE(r10, r11)
960
	addi    r3,r1,STACK_FRAME_OVERHEAD
961 962
	bl      unknown_exception
	b       ret_from_except
963

964 965 966 967
	.align	7
	.globl instruction_access_common
instruction_access_common:
	EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
968
	RECONCILE_IRQ_STATE(r10, r11)
969
	ld	r12,_MSR(r1)
970 971 972
	ld	r3,_NIP(r1)
	andis.	r4,r12,0x5820
	li	r5,0x400
973
	b	do_hash_page		/* Try to handle as hpte fault */
974

975
	STD_EXCEPTION_COMMON(0xe20, h_instr_storage, unknown_exception)
976

977 978 979 980 981 982 983 984 985 986 987 988 989 990
	/*
	 * Machine check is different because we use a different
	 * save area: PACA_EXMC instead of PACA_EXGEN.
	 */
	.align	7
	.globl machine_check_common
machine_check_common:

	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXGEN+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
	FINISH_NAP
991
	RECONCILE_IRQ_STATE(r10, r11)
992 993 994 995
	ld	r3,PACA_EXGEN+EX_DAR(r13)
	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
996
	bl	save_nvgprs
997
	addi	r3,r1,STACK_FRAME_OVERHEAD
998 999
	bl	machine_check_exception
	b	ret_from_except
1000

1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
	.align	7
	.globl alignment_common
alignment_common:
	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXGEN+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
	ld	r3,PACA_EXGEN+EX_DAR(r13)
	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
1013
	bl	save_nvgprs
1014
	RECONCILE_IRQ_STATE(r10, r11)
1015
	addi	r3,r1,STACK_FRAME_OVERHEAD
1016 1017
	bl	alignment_exception
	b	ret_from_except
1018 1019 1020 1021 1022

	.align	7
	.globl program_check_common
program_check_common:
	EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1023
	bl	save_nvgprs
1024
	RECONCILE_IRQ_STATE(r10, r11)
1025
	addi	r3,r1,STACK_FRAME_OVERHEAD
1026 1027
	bl	program_check_exception
	b	ret_from_except
1028 1029 1030 1031 1032 1033

	.align	7
	.globl fp_unavailable_common
fp_unavailable_common:
	EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
	bne	1f			/* if from user, just load it up */
1034
	bl	save_nvgprs
1035
	RECONCILE_IRQ_STATE(r10, r11)
1036
	addi	r3,r1,STACK_FRAME_OVERHEAD
1037
	bl	kernel_fp_unavailable_exception
1038
	BUG_OPCODE
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
1:
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif
1049
	bl	load_up_fpu
1050
	b	fast_exception_return
1051 1052
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
1053
	bl	save_nvgprs
1054
	RECONCILE_IRQ_STATE(r10, r11)
1055
	addi	r3,r1,STACK_FRAME_OVERHEAD
1056 1057
	bl	fp_unavailable_tm
	b	ret_from_except
1058
#endif
1059 1060 1061 1062 1063 1064 1065
	.align	7
	.globl altivec_unavailable_common
altivec_unavailable_common:
	EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	beq	1f
1066 1067 1068 1069 1070 1071 1072 1073 1074
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  BEGIN_FTR_SECTION_NESTED(69)
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
#endif
1075
	bl	load_up_altivec
1076
	b	fast_exception_return
1077 1078
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
1079
	bl	save_nvgprs
1080
	RECONCILE_IRQ_STATE(r10, r11)
1081
	addi	r3,r1,STACK_FRAME_OVERHEAD
1082 1083
	bl	altivec_unavailable_tm
	b	ret_from_except
1084
#endif
1085 1086 1087
1:
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
1088
	bl	save_nvgprs
1089
	RECONCILE_IRQ_STATE(r10, r11)
1090
	addi	r3,r1,STACK_FRAME_OVERHEAD
1091 1092
	bl	altivec_unavailable_exception
	b	ret_from_except
1093 1094 1095 1096 1097 1098 1099

	.align	7
	.globl vsx_unavailable_common
vsx_unavailable_common:
	EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
1100
	beq	1f
1101 1102 1103 1104 1105 1106 1107 1108 1109
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  BEGIN_FTR_SECTION_NESTED(69)
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
#endif
1110
	b	load_up_vsx
1111 1112
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
1113
	bl	save_nvgprs
1114
	RECONCILE_IRQ_STATE(r10, r11)
1115
	addi	r3,r1,STACK_FRAME_OVERHEAD
1116 1117
	bl	vsx_unavailable_tm
	b	ret_from_except
1118
#endif
1119 1120 1121
1:
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
1122
	bl	save_nvgprs
1123
	RECONCILE_IRQ_STATE(r10, r11)
1124
	addi	r3,r1,STACK_FRAME_OVERHEAD
1125 1126
	bl	vsx_unavailable_exception
	b	ret_from_except
1127

1128 1129
	STD_EXCEPTION_COMMON(0xf60, facility_unavailable, facility_unavailable_exception)
	STD_EXCEPTION_COMMON(0xf80, hv_facility_unavailable, facility_unavailable_exception)
1130

1131 1132 1133 1134
	.align	7
	.globl	__end_handlers
__end_handlers:

1135
	/* Equivalents to the above handlers for relocation-on interrupt vectors */
1136 1137
	STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
	MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
1138

1139 1140 1141
	STD_RELON_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
	STD_RELON_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
	STD_RELON_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
1142
	STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
1143
	STD_RELON_EXCEPTION_HV_OOL(0xf80, hv_facility_unavailable)
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159

#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
/*
 * Data area reserved for FWNMI option.
 * This address (0x7000) is fixed by the RPA.
 */
	.= 0x7000
	.globl fwnmi_data_area
fwnmi_data_area:

	/* pseries and powernv need to keep the whole page from
	 * 0x7000 to 0x8000 free for use by the firmware
	 */
	. = 0x8000
#endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */

1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
	.globl hmi_exception_early
hmi_exception_early:
	EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0xe60)
	mr	r10,r1			/* Save r1			*/
	ld	r1,PACAEMERGSP(r13)	/* Use emergency stack		*/
	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame		*/
	std	r9,_CCR(r1)		/* save CR in stackframe	*/
	mfspr	r11,SPRN_HSRR0		/* Save HSRR0 */
	std	r11,_NIP(r1)		/* save HSRR0 in stackframe	*/
	mfspr	r12,SPRN_HSRR1		/* Save SRR1 */
	std	r12,_MSR(r1)		/* save SRR1 in stackframe	*/
	std	r10,0(r1)		/* make stack chain pointer	*/
	std	r0,GPR0(r1)		/* save r0 in stackframe	*/
	std	r10,GPR1(r1)		/* save r1 in stackframe	*/
	EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
	EXCEPTION_PROLOG_COMMON_3(0xe60)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	hmi_exception_realmode
	/* Windup the stack. */
	/* Move original HSRR0 and HSRR1 into the respective regs */
	ld	r9,_MSR(r1)
	mtspr	SPRN_HSRR1,r9
	ld	r3,_NIP(r1)
	mtspr	SPRN_HSRR0,r3
	ld	r9,_CTR(r1)
	mtctr	r9
	ld	r9,_XER(r1)
	mtxer	r9
	ld	r9,_LINK(r1)
	mtlr	r9
	REST_GPR(0, r1)
	REST_8GPRS(2, r1)
	REST_GPR(10, r1)
	ld	r11,_CCR(r1)
	mtcr	r11
	REST_GPR(11, r1)
	REST_2GPRS(12, r1)
	/* restore original r1. */
	ld	r1,GPR1(r1)

	/*
	 * Go to virtual mode and pull the HMI event information from
	 * firmware.
	 */
	.globl hmi_exception_after_realmode
hmi_exception_after_realmode:
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
	b	hmi_exception_hv

1210

1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
#define MACHINE_CHECK_HANDLER_WINDUP			\
	/* Clear MSR_RI before setting SRR0 and SRR1. */\
	li	r0,MSR_RI;				\
	mfmsr	r9;		/* get MSR value */	\
	andc	r9,r9,r0;				\
	mtmsrd	r9,1;		/* Clear MSR_RI */	\
	/* Move original SRR0 and SRR1 into the respective regs */	\
	ld	r9,_MSR(r1);				\
	mtspr	SPRN_SRR1,r9;				\
	ld	r3,_NIP(r1);				\
	mtspr	SPRN_SRR0,r3;				\
	ld	r9,_CTR(r1);				\
	mtctr	r9;					\
	ld	r9,_XER(r1);				\
	mtxer	r9;					\
	ld	r9,_LINK(r1);				\
	mtlr	r9;					\
	REST_GPR(0, r1);				\
	REST_8GPRS(2, r1);				\
	REST_GPR(10, r1);				\
	ld	r11,_CCR(r1);				\
	mtcr	r11;					\
	/* Decrement paca->in_mce. */			\
	lhz	r12,PACA_IN_MCE(r13);			\
	subi	r12,r12,1;				\
	sth	r12,PACA_IN_MCE(r13);			\
	REST_GPR(11, r1);				\
	REST_2GPRS(12, r1);				\
	/* restore original r1. */			\
	ld	r1,GPR1(r1)

	/*
	 * Handle machine check early in real mode. We come here with
	 * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
	 */
	.align	7
	.globl machine_check_handle_early
machine_check_handle_early:
	std	r0,GPR0(r1)	/* Save r0 */
	EXCEPTION_PROLOG_COMMON_3(0x200)
1251
	bl	save_nvgprs
1252
	addi	r3,r1,STACK_FRAME_OVERHEAD
1253
	bl	machine_check_early
1254
	std	r3,RESULT(r1)	/* Save result */
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
	ld	r12,_MSR(r1)
#ifdef	CONFIG_PPC_P7_NAP
	/*
	 * Check if thread was in power saving mode. We come here when any
	 * of the following is true:
	 * a. thread wasn't in power saving mode
	 * b. thread was in power saving mode with no state loss or
	 *    supervisor state loss
	 *
	 * Go back to nap again if (b) is true.
	 */
	rlwinm.	r11,r12,47-31,30,31	/* Was it in power saving mode? */
	beq	4f			/* No, it wasn;t */
	/* Thread was in power saving mode. Go back to nap again. */
	cmpwi	r11,2
	bne	3f
	/* Supervisor state loss */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)
1274
3:	bl	machine_check_queue_event
1275 1276 1277
	MACHINE_CHECK_HANDLER_WINDUP
	GET_PACA(r13)
	ld	r1,PACAR1(r13)
1278
	li	r3,PNV_THREAD_NAP
1279
	b	power7_enter_nap_mode
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
4:
#endif
	/*
	 * Check if we are coming from hypervisor userspace. If yes then we
	 * continue in host kernel in V mode to deliver the MC event.
	 */
	rldicl.	r11,r12,4,63		/* See if MC hit while in HV mode. */
	beq	5f
	andi.	r11,r12,MSR_PR		/* See if coming from user. */
	bne	9f			/* continue in V mode if we are. */

5:
1292
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
	/*
	 * We are coming from kernel context. Check if we are coming from
	 * guest. if yes, then we can continue. We will fall through
	 * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
	 */
	lbz	r11,HSTATE_IN_GUEST(r13)
	cmpwi	r11,0			/* Check if coming from guest */
	bne	9f			/* continue if we are. */
#endif
	/*
	 * At this point we are not sure about what context we come from.
	 * Queue up the MCE event and return from the interrupt.
	 * But before that, check if this is an un-recoverable exception.
	 * If yes, then stay on emergency stack and panic.
	 */
	andi.	r11,r12,MSR_RI
	bne	2f
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
1:	mfspr	r11,SPRN_SRR0
	ld	r10,PACAKBASE(r13)
	LOAD_HANDLER(r10,unrecover_mce)
	mtspr	SPRN_SRR0,r10
	ld	r10,PACAKMSR(r13)
	/*
	 * We are going down. But there are chances that we might get hit by
	 * another MCE during panic path and we may run into unstable state
	 * with no way out. Hence, turn ME bit off while going down, so that
	 * when another MCE is hit during panic path, system will checkstop
	 * and hypervisor will get restarted cleanly by SP.
	 */
	li	r3,MSR_ME
	andc	r10,r10,r3		/* Turn off MSR_ME */
	mtspr	SPRN_SRR1,r10
	rfid
	b	.
1327
2:
1328 1329 1330 1331 1332 1333 1334 1335
	/*
	 * Check if we have successfully handled/recovered from error, if not
	 * then stay on emergency stack and panic.
	 */
	ld	r3,RESULT(r1)	/* Load result */
	cmpdi	r3,0		/* see if we handled MCE successfully */

	beq	1b		/* if !handled then panic */
1336 1337 1338 1339 1340
	/*
	 * Return from MC interrupt.
	 * Queue up the MCE event so that we can log it later, while
	 * returning from kernel or opal call.
	 */
1341
	bl	machine_check_queue_event
1342 1343 1344 1345 1346 1347 1348
	MACHINE_CHECK_HANDLER_WINDUP
	rfid
9:
	/* Deliver the machine check to host kernel in V mode. */
	MACHINE_CHECK_HANDLER_WINDUP
	b	machine_check_pSeries

1349 1350 1351
unrecover_mce:
	/* Invoke machine_check_exception to print MCE event and panic. */
	addi	r3,r1,STACK_FRAME_OVERHEAD
1352
	bl	machine_check_exception
1353 1354 1355 1356 1357
	/*
	 * We will not reach here. Even if we did, there is no way out. Call
	 * unrecoverable_exception and die.
	 */
1:	addi	r3,r1,STACK_FRAME_OVERHEAD
1358
	bl	unrecoverable_exception
1359
	b	1b
1360 1361 1362 1363 1364 1365 1366 1367
/*
 * r13 points to the PACA, r9 contains the saved CR,
 * r12 contain the saved SRR1, SRR0 is still ready for return
 * r3 has the faulting address
 * r9 - r13 are saved in paca->exslb.
 * r3 is saved in paca->slb_r3
 * We assume we aren't going to take any exceptions during this procedure.
 */
1368
slb_miss_realmode:
1369 1370 1371 1372 1373 1374 1375 1376
	mflr	r10
#ifdef CONFIG_RELOCATABLE
	mtctr	r11
#endif

	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
	std	r10,PACA_EXSLB+EX_LR(r13)	/* save LR */

1377
	bl	slb_allocate_realmode
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415

	/* All done -- return from exception. */

	ld	r10,PACA_EXSLB+EX_LR(r13)
	ld	r3,PACA_EXSLB+EX_R3(r13)
	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */

	mtlr	r10

	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
	beq-	2f

.machine	push
.machine	"power4"
	mtcrf	0x80,r9
	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
.machine	pop

	RESTORE_PPR_PACA(PACA_EXSLB, r9)
	ld	r9,PACA_EXSLB+EX_R9(r13)
	ld	r10,PACA_EXSLB+EX_R10(r13)
	ld	r11,PACA_EXSLB+EX_R11(r13)
	ld	r12,PACA_EXSLB+EX_R12(r13)
	ld	r13,PACA_EXSLB+EX_R13(r13)
	rfid
	b	.	/* prevent speculative execution */

2:	mfspr	r11,SPRN_SRR0
	ld	r10,PACAKBASE(r13)
	LOAD_HANDLER(r10,unrecov_slb)
	mtspr	SPRN_SRR0,r10
	ld	r10,PACAKMSR(r13)
	mtspr	SPRN_SRR1,r10
	rfid
	b	.

unrecov_slb:
	EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1416
	RECONCILE_IRQ_STATE(r10, r11)
1417
	bl	save_nvgprs
1418
1:	addi	r3,r1,STACK_FRAME_OVERHEAD
1419
	bl	unrecoverable_exception
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
	b	1b


#ifdef CONFIG_PPC_970_NAP
power4_fixup_nap:
	andc	r9,r9,r10
	std	r9,TI_LOCAL_FLAGS(r11)
	ld	r10,_LINK(r1)		/* make idle task do the */
	std	r10,_NIP(r1)		/* equivalent of a blr */
	blr
#endif

1432 1433 1434 1435
/*
 * Hash table stuff
 */
	.align	7
1436
do_hash_page:
1437 1438 1439
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)

1440
	andis.	r0,r4,0xa410		/* weird error? */
1441
	bne-	handle_page_fault	/* if not, try to insert a HPTE */
1442 1443
	andis.  r0,r4,DSISR_DABRMATCH@h
	bne-    handle_dabr_fault
1444
	CURRENT_THREAD_INFO(r11, r1)
1445 1446 1447
	lwz	r0,TI_PREEMPT(r11)	/* If we're in an "NMI" */
	andis.	r0,r0,NMI_MASK@h	/* (i.e. an irq when soft-disabled) */
	bne	77f			/* then don't call hash_page now */
1448 1449 1450

	/*
	 * r3 contains the faulting address
1451
	 * r4 msr
1452
	 * r5 contains the trap number
1453
	 * r6 contains dsisr
1454
	 *
1455
	 * at return r3 = 0 for success, 1 for page fault, negative for error
1456
	 */
1457
        mr 	r4,r12
1458
	ld      r6,_DSISR(r1)
1459 1460
	bl	__hash_page		/* build HPTE if possible */
        cmpdi	r3,0			/* see if __hash_page succeeded */
1461

1462
	/* Success */
1463 1464
	beq	fast_exc_return_irq	/* Return from exception on success */

1465 1466
	/* Error */
	blt-	13f
1467

1468 1469 1470 1471 1472
/* Here we have a page fault that hash_page can't handle. */
handle_page_fault:
11:	ld	r4,_DAR(r1)
	ld	r5,_DSISR(r1)
	addi	r3,r1,STACK_FRAME_OVERHEAD
1473
	bl	do_page_fault
1474
	cmpdi	r3,0
1475
	beq+	12f
1476
	bl	save_nvgprs
1477 1478 1479
	mr	r5,r3
	addi	r3,r1,STACK_FRAME_OVERHEAD
	lwz	r4,_DAR(r1)
1480 1481
	bl	bad_page_fault
	b	ret_from_except
1482

1483 1484
/* We have a data breakpoint exception - handle it */
handle_dabr_fault:
1485
	bl	save_nvgprs
1486 1487 1488
	ld      r4,_DAR(r1)
	ld      r5,_DSISR(r1)
	addi    r3,r1,STACK_FRAME_OVERHEAD
1489 1490
	bl      do_break
12:	b       ret_from_except_lite
1491

1492 1493 1494 1495

/* We have a page fault that hash_page could handle but HV refused
 * the PTE insertion
 */
1496
13:	bl	save_nvgprs
1497 1498 1499
	mr	r5,r3
	addi	r3,r1,STACK_FRAME_OVERHEAD
	ld	r4,_DAR(r1)
1500 1501
	bl	low_hash_fault
	b	ret_from_except
1502

1503 1504 1505 1506 1507 1508 1509
/*
 * We come here as a result of a DSI at a point where we don't want
 * to call hash_page, such as when we are accessing memory (possibly
 * user memory) inside a PMU interrupt that occurred while interrupts
 * were soft-disabled.  We want to invoke the exception handler for
 * the access, or panic if there isn't a handler.
 */
1510
77:	bl	save_nvgprs
1511 1512 1513
	mr	r4,r3
	addi	r3,r1,STACK_FRAME_OVERHEAD
	li	r5,SIGSEGV
1514 1515
	bl	bad_page_fault
	b	ret_from_except
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575

/*
 * Here we have detected that the kernel stack pointer is bad.
 * R9 contains the saved CR, r13 points to the paca,
 * r10 contains the (bad) kernel stack pointer,
 * r11 and r12 contain the saved SRR0 and SRR1.
 * We switch to using an emergency stack, save the registers there,
 * and call kernel_bad_stack(), which panics.
 */
bad_stack:
	ld	r1,PACAEMERGSP(r13)
	subi	r1,r1,64+INT_FRAME_SIZE
	std	r9,_CCR(r1)
	std	r10,GPR1(r1)
	std	r11,_NIP(r1)
	std	r12,_MSR(r1)
	mfspr	r11,SPRN_DAR
	mfspr	r12,SPRN_DSISR
	std	r11,_DAR(r1)
	std	r12,_DSISR(r1)
	mflr	r10
	mfctr	r11
	mfxer	r12
	std	r10,_LINK(r1)
	std	r11,_CTR(r1)
	std	r12,_XER(r1)
	SAVE_GPR(0,r1)
	SAVE_GPR(2,r1)
	ld	r10,EX_R3(r3)
	std	r10,GPR3(r1)
	SAVE_GPR(4,r1)
	SAVE_4GPRS(5,r1)
	ld	r9,EX_R9(r3)
	ld	r10,EX_R10(r3)
	SAVE_2GPRS(9,r1)
	ld	r9,EX_R11(r3)
	ld	r10,EX_R12(r3)
	ld	r11,EX_R13(r3)
	std	r9,GPR11(r1)
	std	r10,GPR12(r1)
	std	r11,GPR13(r1)
BEGIN_FTR_SECTION
	ld	r10,EX_CFAR(r3)
	std	r10,ORIG_GPR3(r1)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
	SAVE_8GPRS(14,r1)
	SAVE_10GPRS(22,r1)
	lhz	r12,PACA_TRAP_SAVE(r13)
	std	r12,_TRAP(r1)
	addi	r11,r1,INT_FRAME_SIZE
	std	r11,0(r1)
	li	r12,0
	std	r12,0(r11)
	ld	r2,PACATOC(r13)
	ld	r11,exception_marker@toc(r2)
	std	r12,RESULT(r1)
	std	r11,STACK_FRAME_OVERHEAD-16(r1)
1:	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	kernel_bad_stack
	b	1b