entry.S 25.1 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative.h>
C
Catalin Marinas 已提交
25 26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
C
Catalin Marinas 已提交
28
#include <asm/errno.h>
29
#include <asm/esr.h>
30
#include <asm/irq.h>
31 32
#include <asm/memory.h>
#include <asm/mmu.h>
33
#include <asm/processor.h>
34
#include <asm/ptrace.h>
C
Catalin Marinas 已提交
35
#include <asm/thread_info.h>
A
Al Viro 已提交
36
#include <asm/asm-uaccess.h>
C
Catalin Marinas 已提交
37 38
#include <asm/unistd.h>

L
Larry Bassel 已提交
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

C
Catalin Marinas 已提交
65 66 67 68 69 70 71 72 73
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

74
	.macro kernel_ventry, el, label, regsize = 64
75
	.align 7
76 77 78 79 80 81 82 83 84 85 86
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	.if	\el == 0
	.if	\regsize == 64
	mrs	x30, tpidrro_el0
	msr	tpidrro_el0, xzr
	.else
	mov	x30, xzr
	.endif
	.endif
#endif

87
	sub	sp, sp, #S_FRAME_SIZE
88 89 90 91 92 93 94 95 96 97
#ifdef CONFIG_VMAP_STACK
	/*
	 * Test whether the SP has overflowed, without corrupting a GPR.
	 * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
	 */
	add	sp, sp, x0			// sp' = sp + x0
	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
	tbnz	x0, #THREAD_SHIFT, 0f
	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
98
	b	el\()\el\()_\label
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129

0:
	/*
	 * Either we've just detected an overflow, or we've taken an exception
	 * while on the overflow stack. Either way, we won't return to
	 * userspace, and can clobber EL0 registers to free up GPRs.
	 */

	/* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
	msr	tpidr_el0, x0

	/* Recover the original x0 value and stash it in tpidrro_el0 */
	sub	x0, sp, x0
	msr	tpidrro_el0, x0

	/* Switch to the overflow stack */
	adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0

	/*
	 * Check whether we were already on the overflow stack. This may happen
	 * after panic() re-enables interrupts.
	 */
	mrs	x0, tpidr_el0			// sp of interrupted context
	sub	x0, sp, x0			// delta with top of overflow stack
	tst	x0, #~(OVERFLOW_STACK_SIZE - 1)	// within range?
	b.ne	__bad_stack			// no? -> bad stack pointer

	/* We were already on the overflow stack. Restore sp/x0 and carry on. */
	sub	sp, sp, x0
	mrs	x0, tpidrro_el0
#endif
130
	b	el\()\el\()_\label
131 132
	.endm

133 134 135 136 137
	.macro tramp_alias, dst, sym
	mov_q	\dst, TRAMP_VALIAS
	add	\dst, \dst, #(\sym - .entry.tramp.text)
	.endm

138
	.macro	kernel_entry, el, regsize = 64
C
Catalin Marinas 已提交
139 140 141
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

C
Catalin Marinas 已提交
158 159
	.if	\el == 0
	mrs	x21, sp_el0
160 161
	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
162
	disable_step_tsk x19, x20		// exceptions when scheduling.
163 164

	mov	x29, xzr			// fp pointed to user-space
C
Catalin Marinas 已提交
165 166
	.else
	add	x21, sp, #S_FRAME_SIZE
167 168
	get_thread_info tsk
	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
169
	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
170 171
	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
	mov	x20, #TASK_SIZE_64
172
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
173
	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
174
	.endif /* \el == 0 */
C
Catalin Marinas 已提交
175 176 177
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
178

179 180 181 182 183 184 185 186 187 188 189 190
	/*
	 * In order to be able to dump the contents of struct pt_regs at the
	 * time the exception was taken (in case we attempt to walk the call
	 * stack later), chain it together with the stack frames.
	 */
	.if \el == 0
	stp	xzr, xzr, [sp, #S_STACKFRAME]
	.else
	stp	x29, x22, [sp, #S_STACKFRAME]
	.endif
	add	x29, sp, #S_STACKFRAME

191 192 193 194 195 196 197 198 199 200 201 202 203 204
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
	 * EL0, there is no need to check the state of TTBR0_EL1 since
	 * accesses are always enabled.
	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
	 * user mappings.
	 */
alternative_if ARM64_HAS_PAN
	b	1f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
205
	mrs	x21, ttbr1_el1
206 207 208 209 210 211 212 213 214 215
	tst	x21, #0xffff << 48		// Check for the reserved ASID
	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
	b.eq	1f				// TTBR0 access already disabled
	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
	.endif

	__uaccess_ttbr0_disable x21
1:
#endif

C
Catalin Marinas 已提交
216 217
	stp	x22, x23, [sp, #S_PC]

218
	/* Not in a syscall by default (el0_svc overwrites for real syscall) */
C
Catalin Marinas 已提交
219
	.if	\el == 0
220
	mov	w21, #NO_SYSCALL
221
	str	w21, [sp, #S_SYSCALLNO]
C
Catalin Marinas 已提交
222 223
	.endif

224 225 226 227 228 229 230
	/*
	 * Set sp_el0 to current thread_info.
	 */
	.if	\el == 0
	msr	sp_el0, tsk
	.endif

C
Catalin Marinas 已提交
231 232 233 234 235 236 237 238 239
	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

240
	.macro	kernel_exit, el
241
	.if	\el != 0
242 243
	disable_daif

244 245
	/* Restore the task's original addr_limit. */
	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
246
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
247 248 249 250

	/* No need to restore UAO, it will be restored from SPSR_EL1 */
	.endif

C
Catalin Marinas 已提交
251 252
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
L
Larry Bassel 已提交
253
	ct_user_enter
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
	.endif

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
	 * PAN bit checking.
	 */
alternative_if ARM64_HAS_PAN
	b	2f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
	.endif

269
	__uaccess_ttbr0_enable x0, x1
270 271 272 273 274 275 276 277

	.if	\el == 0
	/*
	 * Enable errata workarounds only if returning to user. The only
	 * workaround currently required for TTBR0_EL1 changes are for the
	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
	 * corruption).
	 */
278
	post_ttbr_update_workaround
279 280 281 282 283 284 285 286 287
	.endif
1:
	.if	\el != 0
	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
	.endif
2:
#endif

	.if	\el == 0
C
Catalin Marinas 已提交
288
	ldr	x23, [sp, #S_SP]		// load return stack pointer
289
	msr	sp_el0, x23
290 291 292
	tst	x22, #PSR_MODE32_BIT		// native task?
	b.eq	3f

293
#ifdef CONFIG_ARM64_ERRATUM_845719
M
Mark Rutland 已提交
294
alternative_if ARM64_WORKAROUND_845719
295 296 297
#ifdef CONFIG_PID_IN_CONTEXTIDR
	mrs	x29, contextidr_el1
	msr	contextidr_el1, x29
298
#else
299
	msr contextidr_el1, xzr
300
#endif
M
Mark Rutland 已提交
301
alternative_else_nop_endif
302
#endif
303
3:
C
Catalin Marinas 已提交
304
	.endif
305

306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
	ldp	x0, x1, [sp, #16 * 0]
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340

#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
	eret
#else
	.if	\el == 0
	bne	4f
	msr	far_el1, x30
	tramp_alias	x30, tramp_exit_native
	br	x30
4:
	tramp_alias	x30, tramp_exit_compat
	br	x30
	.else
	eret
	.endif
#endif
C
Catalin Marinas 已提交
341 342
	.endm

343
	.macro	irq_stack_entry
344 345 346
	mov	x19, sp			// preserve the original sp

	/*
347 348 349
	 * Compare sp with the base of the task stack.
	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
	 * and should switch to the irq stack.
350
	 */
351 352 353 354
	ldr	x25, [tsk, TSK_STACK]
	eor	x25, x25, x19
	and	x25, x25, #~(THREAD_SIZE - 1)
	cbnz	x25, 9998f
355

M
Mark Rutland 已提交
356
	ldr_this_cpu x25, irq_stack_ptr, x26
357
	mov	x26, #IRQ_STACK_SIZE
358
	add	x26, x25, x26
359 360

	/* switch to the irq stack */
361 362 363 364 365 366 367 368 369 370 371 372
	mov	sp, x26
9998:
	.endm

	/*
	 * x19 should be preserved between irq_stack_entry and
	 * irq_stack_exit.
	 */
	.macro	irq_stack_exit
	mov	sp, x19
	.endm

C
Catalin Marinas 已提交
373 374 375 376 377 378
/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
379 380 381
wsc_nr	.req	w25		// number of system calls
wscno	.req	w26		// syscall number
xscno	.req	x26		// syscall number (zero-extended)
C
Catalin Marinas 已提交
382 383 384 385 386 387 388
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
389
	ldr_l	x1, handle_arch_irq
C
Catalin Marinas 已提交
390
	mov	x0, sp
391
	irq_stack_entry
C
Catalin Marinas 已提交
392
	blr	x1
393
	irq_stack_exit
C
Catalin Marinas 已提交
394 395 396 397 398 399 400
	.endm

	.text

/*
 * Exception vectors.
 */
401
	.pushsection ".entry.text", "ax"
C
Catalin Marinas 已提交
402 403 404

	.align	11
ENTRY(vectors)
405 406 407 408
	kernel_ventry	1, sync_invalid			// Synchronous EL1t
	kernel_ventry	1, irq_invalid			// IRQ EL1t
	kernel_ventry	1, fiq_invalid			// FIQ EL1t
	kernel_ventry	1, error_invalid		// Error EL1t
C
Catalin Marinas 已提交
409

410 411 412 413
	kernel_ventry	1, sync				// Synchronous EL1h
	kernel_ventry	1, irq				// IRQ EL1h
	kernel_ventry	1, fiq_invalid			// FIQ EL1h
	kernel_ventry	1, error			// Error EL1h
C
Catalin Marinas 已提交
414

415 416 417 418
	kernel_ventry	0, sync				// Synchronous 64-bit EL0
	kernel_ventry	0, irq				// IRQ 64-bit EL0
	kernel_ventry	0, fiq_invalid			// FIQ 64-bit EL0
	kernel_ventry	0, error			// Error 64-bit EL0
C
Catalin Marinas 已提交
419 420

#ifdef CONFIG_COMPAT
421 422 423 424
	kernel_ventry	0, sync_compat, 32		// Synchronous 32-bit EL0
	kernel_ventry	0, irq_compat, 32		// IRQ 32-bit EL0
	kernel_ventry	0, fiq_invalid_compat, 32	// FIQ 32-bit EL0
	kernel_ventry	0, error_compat, 32		// Error 32-bit EL0
C
Catalin Marinas 已提交
425
#else
426 427 428 429
	kernel_ventry	0, sync_invalid, 32		// Synchronous 32-bit EL0
	kernel_ventry	0, irq_invalid, 32		// IRQ 32-bit EL0
	kernel_ventry	0, fiq_invalid, 32		// FIQ 32-bit EL0
	kernel_ventry	0, error_invalid, 32		// Error 32-bit EL0
C
Catalin Marinas 已提交
430 431 432
#endif
END(vectors)

433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
#ifdef CONFIG_VMAP_STACK
	/*
	 * We detected an overflow in kernel_ventry, which switched to the
	 * overflow stack. Stash the exception regs, and head to our overflow
	 * handler.
	 */
__bad_stack:
	/* Restore the original x0 value */
	mrs	x0, tpidrro_el0

	/*
	 * Store the original GPRs to the new stack. The orginal SP (minus
	 * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
	 */
	sub	sp, sp, #S_FRAME_SIZE
	kernel_entry 1
	mrs	x0, tpidr_el0
	add	x0, x0, #S_FRAME_SIZE
	str	x0, [sp, #S_SP]

	/* Stash the regs for handle_bad_stack */
	mov	x0, sp

	/* Time to die */
	bl	handle_bad_stack
	ASM_BUG()
#endif /* CONFIG_VMAP_STACK */

C
Catalin Marinas 已提交
461 462 463 464
/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
465
	kernel_entry \el, \regsize
C
Catalin Marinas 已提交
466 467 468
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
469 470
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
518 519
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
C
Catalin Marinas 已提交
520
	b.eq	el1_da
521 522
	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
	b.eq	el1_ia
M
Mark Rutland 已提交
523
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
C
Catalin Marinas 已提交
524
	b.eq	el1_undef
M
Mark Rutland 已提交
525
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
526
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
527
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
528
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
529
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
C
Catalin Marinas 已提交
530
	b.eq	el1_undef
M
Mark Rutland 已提交
531
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
C
Catalin Marinas 已提交
532 533
	b.ge	el1_dbg
	b	el1_inv
534 535 536 537 538

el1_ia:
	/*
	 * Fall through to the Data abort case
	 */
C
Catalin Marinas 已提交
539 540 541 542
el1_da:
	/*
	 * Data abort handling
	 */
543
	mrs	x3, far_el1
J
James Morse 已提交
544
	inherit_daif	pstate=x23, tmp=x2
545
	clear_address_tag x0, x3
C
Catalin Marinas 已提交
546 547 548 549 550 551 552 553 554
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
J
James Morse 已提交
555
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
556
	mov	x2, sp
557 558
	bl	do_sp_pc_abort
	ASM_BUG()
C
Catalin Marinas 已提交
559 560 561 562
el1_undef:
	/*
	 * Undefined instruction
	 */
J
James Morse 已提交
563
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
564
	mov	x0, sp
565 566
	bl	do_undefinstr
	ASM_BUG()
C
Catalin Marinas 已提交
567 568 569 570
el1_dbg:
	/*
	 * Debug exception handling
	 */
M
Mark Rutland 已提交
571
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
572
	cinc	x24, x24, eq			// set bit '0'
C
Catalin Marinas 已提交
573 574 575 576 577 578 579
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
J
James Morse 已提交
580
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
581
	mov	x0, sp
582
	mov	x2, x1
C
Catalin Marinas 已提交
583
	mov	x1, #BAD_SYNC
584 585
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
586 587 588 589 590
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
J
James Morse 已提交
591
	enable_da_f
C
Catalin Marinas 已提交
592 593 594
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
595

C
Catalin Marinas 已提交
596
	irq_handler
597

C
Catalin Marinas 已提交
598
#ifdef CONFIG_PREEMPT
599
	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
600
	cbnz	w24, 1f				// preempt count != 0
601
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
C
Catalin Marinas 已提交
602 603 604 605 606 607 608 609 610 611 612 613 614
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
615
1:	bl	preempt_schedule_irq		// irq en/disable is done inside
616
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
C
Catalin Marinas 已提交
617 618 619 620 621 622 623 624 625 626 627
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
628 629
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
C
Catalin Marinas 已提交
630
	b.eq	el0_svc
M
Mark Rutland 已提交
631
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
632
	b.eq	el0_da
M
Mark Rutland 已提交
633
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
634
	b.eq	el0_ia
M
Mark Rutland 已提交
635
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
636
	b.eq	el0_fpsimd_acc
637 638
	cmp	x24, #ESR_ELx_EC_SVE		// SVE access
	b.eq	el0_sve_acc
M
Mark Rutland 已提交
639
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
C
Catalin Marinas 已提交
640
	b.eq	el0_fpsimd_exc
M
Mark Rutland 已提交
641
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
642
	b.eq	el0_sys
M
Mark Rutland 已提交
643
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
644
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
645
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
646
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
647
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
648
	b.eq	el0_undef
M
Mark Rutland 已提交
649
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
650 651 652 653 654 655 656 657
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
658 659
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
C
Catalin Marinas 已提交
660
	b.eq	el0_svc_compat
M
Mark Rutland 已提交
661
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
662
	b.eq	el0_da
M
Mark Rutland 已提交
663
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
664
	b.eq	el0_ia
M
Mark Rutland 已提交
665
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
666
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
667
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
C
Catalin Marinas 已提交
668
	b.eq	el0_fpsimd_exc
669 670
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
671
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
672
	b.eq	el0_undef
M
Mark Rutland 已提交
673
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
674
	b.eq	el0_undef
M
Mark Rutland 已提交
675
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
676
	b.eq	el0_undef
M
Mark Rutland 已提交
677
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
678
	b.eq	el0_undef
M
Mark Rutland 已提交
679
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
680
	b.eq	el0_undef
M
Mark Rutland 已提交
681
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
682
	b.eq	el0_undef
M
Mark Rutland 已提交
683
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
684 685 686 687 688 689
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
690
	ldr	x16, [tsk, #TSK_TI_FLAGS]	// load thread flags
691
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
692 693
	mov	wscno, w7			// syscall number in w7 (r7)
	mov     wsc_nr, #__NR_compat_syscalls
C
Catalin Marinas 已提交
694 695 696 697 698 699
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
700 701 702 703

el0_error_compat:
	kernel_entry 0, 32
	b	el0_error_naked
C
Catalin Marinas 已提交
704 705 706 707 708 709
#endif

el0_da:
	/*
	 * Data abort handling
	 */
710
	mrs	x26, far_el1
J
James Morse 已提交
711
	enable_daif
L
Larry Bassel 已提交
712
	ct_user_exit
713
	clear_address_tag x0, x26
C
Catalin Marinas 已提交
714 715
	mov	x1, x25
	mov	x2, sp
716 717
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
718 719 720 721
el0_ia:
	/*
	 * Instruction abort handling
	 */
722
	mrs	x26, far_el1
J
James Morse 已提交
723
	enable_daif
L
Larry Bassel 已提交
724
	ct_user_exit
725
	mov	x0, x26
M
Mark Rutland 已提交
726
	mov	x1, x25
C
Catalin Marinas 已提交
727
	mov	x2, sp
728 729
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
730 731 732 733
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
J
James Morse 已提交
734
	enable_daif
L
Larry Bassel 已提交
735
	ct_user_exit
C
Catalin Marinas 已提交
736 737
	mov	x0, x25
	mov	x1, sp
738 739
	bl	do_fpsimd_acc
	b	ret_to_user
740 741 742 743 744 745 746 747 748 749
el0_sve_acc:
	/*
	 * Scalable Vector Extension access
	 */
	enable_daif
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sve_acc
	b	ret_to_user
C
Catalin Marinas 已提交
750 751
el0_fpsimd_exc:
	/*
752
	 * Floating Point, Advanced SIMD or SVE exception
C
Catalin Marinas 已提交
753
	 */
J
James Morse 已提交
754
	enable_daif
L
Larry Bassel 已提交
755
	ct_user_exit
C
Catalin Marinas 已提交
756 757
	mov	x0, x25
	mov	x1, sp
758 759
	bl	do_fpsimd_exc
	b	ret_to_user
C
Catalin Marinas 已提交
760 761 762 763
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
764
	mrs	x26, far_el1
J
James Morse 已提交
765
	enable_daif
766
	ct_user_exit
767
	mov	x0, x26
C
Catalin Marinas 已提交
768 769
	mov	x1, x25
	mov	x2, sp
770 771
	bl	do_sp_pc_abort
	b	ret_to_user
C
Catalin Marinas 已提交
772 773 774 775
el0_undef:
	/*
	 * Undefined instruction
	 */
J
James Morse 已提交
776
	enable_daif
L
Larry Bassel 已提交
777
	ct_user_exit
778
	mov	x0, sp
779 780
	bl	do_undefinstr
	b	ret_to_user
781 782 783 784
el0_sys:
	/*
	 * System instructions, for trapped cache maintenance instructions
	 */
J
James Morse 已提交
785
	enable_daif
786 787 788 789 790
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sysinstr
	b	ret_to_user
C
Catalin Marinas 已提交
791 792 793 794 795 796 797 798
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
799
	bl	do_debug_exception
J
James Morse 已提交
800
	enable_daif
L
Larry Bassel 已提交
801
	ct_user_exit
802
	b	ret_to_user
C
Catalin Marinas 已提交
803
el0_inv:
J
James Morse 已提交
804
	enable_daif
L
Larry Bassel 已提交
805
	ct_user_exit
C
Catalin Marinas 已提交
806 807
	mov	x0, sp
	mov	x1, #BAD_SYNC
808
	mov	x2, x25
809
	bl	bad_el0_sync
810
	b	ret_to_user
C
Catalin Marinas 已提交
811 812 813 814 815 816
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
J
James Morse 已提交
817
	enable_da_f
C
Catalin Marinas 已提交
818 819 820
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
821

L
Larry Bassel 已提交
822
	ct_user_exit
C
Catalin Marinas 已提交
823
	irq_handler
824

C
Catalin Marinas 已提交
825 826 827 828 829 830
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
el1_error:
	kernel_entry 1
	mrs	x1, esr_el1
	enable_dbg
	mov	x0, sp
	bl	do_serror
	kernel_exit 1
ENDPROC(el1_error)

el0_error:
	kernel_entry 0
el0_error_naked:
	mrs	x1, esr_el1
	enable_dbg
	mov	x0, sp
	bl	do_serror
	enable_daif
	ct_user_exit
	b	ret_to_user
ENDPROC(el0_error)


C
Catalin Marinas 已提交
853 854 855 856 857
/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
858
	disable_daif
859
	str	x0, [sp, #S_X0]			// returned x0
860
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
861 862
	and	x2, x1, #_TIF_SYSCALL_WORK
	cbnz	x2, ret_fast_syscall_trace
C
Catalin Marinas 已提交
863
	and	x2, x1, #_TIF_WORK_MASK
864
	cbnz	x2, work_pending
865
	enable_step_tsk x1, x2
866
	kernel_exit 0
867
ret_fast_syscall_trace:
868
	enable_daif
869
	b	__sys_trace_return_skipped	// we already saved x0
C
Catalin Marinas 已提交
870 871 872 873 874 875 876

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
work_pending:
	mov	x0, sp				// 'regs'
	bl	do_notify_resume
877
#ifdef CONFIG_TRACE_IRQFLAGS
878
	bl	trace_hardirqs_on		// enabled while in userspace
879
#endif
880
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
881
	b	finish_ret_to_user
C
Catalin Marinas 已提交
882 883 884
/*
 * "slow" syscall return path.
 */
885
ret_to_user:
886
	disable_daif
887
	ldr	x1, [tsk, #TSK_TI_FLAGS]
C
Catalin Marinas 已提交
888 889
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
890
finish_ret_to_user:
891
	enable_step_tsk x1, x2
892
	kernel_exit 0
C
Catalin Marinas 已提交
893 894 895 896 897 898 899
ENDPROC(ret_to_user)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
900
	ldr	x16, [tsk, #TSK_TI_FLAGS]	// load thread flags
C
Catalin Marinas 已提交
901
	adrp	stbl, sys_call_table		// load syscall table pointer
902 903
	mov	wscno, w8			// syscall number in w8
	mov	wsc_nr, #__NR_syscalls
904

905 906
#ifdef CONFIG_ARM64_SVE
alternative_if_not ARM64_SVE
907
	b	el0_svc_naked
908
alternative_else_nop_endif
909 910 911 912 913 914 915 916 917 918 919 920 921 922
	tbz	x16, #TIF_SVE, el0_svc_naked	// Skip unless TIF_SVE set:
	bic	x16, x16, #_TIF_SVE		// discard SVE state
	str	x16, [tsk, #TSK_TI_FLAGS]

	/*
	 * task_fpsimd_load() won't be called to update CPACR_EL1 in
	 * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
	 * happens if a context switch or kernel_neon_begin() or context
	 * modification (sigreturn, ptrace) intervenes.
	 * So, ensure that CPACR_EL1 is already correct for the fast-path case:
	 */
	mrs	x9, cpacr_el1
	bic	x9, x9, #CPACR_EL1_ZEN_EL0EN	// disable SVE for el0
	msr	cpacr_el1, x9			// synchronised by eret to el0
923
#endif
924

C
Catalin Marinas 已提交
925
el0_svc_naked:					// compat entry point
926
	stp	x0, xscno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
J
James Morse 已提交
927
	enable_daif
L
Larry Bassel 已提交
928
	ct_user_exit 1
C
Catalin Marinas 已提交
929

930
	tst	x16, #_TIF_SYSCALL_WORK		// check for syscall hooks
931
	b.ne	__sys_trace
932
	cmp     wscno, wsc_nr			// check upper syscall limit
C
Catalin Marinas 已提交
933
	b.hs	ni_sys
934
	ldr	x16, [stbl, xscno, lsl #3]	// address in the syscall table
935 936
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
C
Catalin Marinas 已提交
937 938
ni_sys:
	mov	x0, sp
939 940
	bl	do_ni_syscall
	b	ret_fast_syscall
C
Catalin Marinas 已提交
941 942 943 944 945 946 947
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
948
	cmp     wscno, #NO_SYSCALL		// user-issued syscall(-1)?
949
	b.ne	1f
950
	mov	x0, #-ENOSYS			// set default errno if so
951 952
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
953
	bl	syscall_trace_enter
954
	cmp	w0, #NO_SYSCALL			// skip the syscall?
955
	b.eq	__sys_trace_return_skipped
956
	mov	wscno, w0			// syscall number (possibly new)
C
Catalin Marinas 已提交
957
	mov	x1, sp				// pointer to regs
958
	cmp	wscno, wsc_nr			// check upper syscall limit
959
	b.hs	__ni_sys_trace
C
Catalin Marinas 已提交
960 961 962 963
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
964
	ldr	x16, [stbl, xscno, lsl #3]	// address in the syscall table
965
	blr	x16				// call sys_* routine
C
Catalin Marinas 已提交
966 967

__sys_trace_return:
968 969
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
970 971
	mov	x0, sp
	bl	syscall_trace_exit
C
Catalin Marinas 已提交
972 973
	b	ret_to_user

974 975 976 977 978
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

979 980
	.popsection				// .entry.text

981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
/*
 * Exception vectors trampoline.
 */
	.pushsection ".entry.tramp.text", "ax"

	.macro tramp_map_kernel, tmp
	mrs	\tmp, ttbr1_el1
	sub	\tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
	bic	\tmp, \tmp, #USER_ASID_FLAG
	msr	ttbr1_el1, \tmp
	.endm

	.macro tramp_unmap_kernel, tmp
	mrs	\tmp, ttbr1_el1
	add	\tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
	orr	\tmp, \tmp, #USER_ASID_FLAG
	msr	ttbr1_el1, \tmp
	/*
	 * We avoid running the post_ttbr_update_workaround here because the
	 * user and kernel ASIDs don't have conflicting mappings, so any
	 * "blessing" as described in:
	 *
	 *   http://lkml.kernel.org/r/56BB848A.6060603@caviumnetworks.com
	 *
	 * will not hurt correctness. Whilst this may partially defeat the
	 * point of using split ASIDs in the first place, it avoids
	 * the hit of invalidating the entire I-cache on every return to
	 * userspace.
	 */
	.endm

	.macro tramp_ventry, regsize = 64
	.align	7
1:
	.if	\regsize == 64
	msr	tpidrro_el0, x30	// Restored in kernel_ventry
	.endif
	tramp_map_kernel	x30
	ldr	x30, =vectors
	prfm	plil1strm, [x30, #(1b - tramp_vectors)]
	msr	vbar_el1, x30
	add	x30, x30, #(1b - tramp_vectors)
	isb
	br	x30
	.endm

	.macro tramp_exit, regsize = 64
	adr	x30, tramp_vectors
	msr	vbar_el1, x30
	tramp_unmap_kernel	x30
	.if	\regsize == 64
	mrs	x30, far_el1
	.endif
	eret
	.endm

	.align	11
ENTRY(tramp_vectors)
	.space	0x400

	tramp_ventry
	tramp_ventry
	tramp_ventry
	tramp_ventry

	tramp_ventry	32
	tramp_ventry	32
	tramp_ventry	32
	tramp_ventry	32
END(tramp_vectors)

ENTRY(tramp_exit_native)
	tramp_exit
END(tramp_exit_native)

ENTRY(tramp_exit_compat)
	tramp_exit	32
END(tramp_exit_compat)

	.ltorg
	.popsection				// .entry.tramp.text
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

C
Catalin Marinas 已提交
1065 1066 1067 1068 1069 1070 1071
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
	mov	x10, #THREAD_CPU_CONTEXT
	add	x8, x0, x10
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
	add	x8, x1, x10
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
	msr	sp_el0, x1
	ret
ENDPROC(cpu_switch_to)
NOKPROBE(cpu_switch_to)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
	b	ret_to_user
ENDPROC(ret_from_fork)
NOKPROBE(ret_from_fork)