entry.S 21.8 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative.h>
C
Catalin Marinas 已提交
25 26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
C
Catalin Marinas 已提交
28
#include <asm/errno.h>
29
#include <asm/esr.h>
30
#include <asm/irq.h>
31
#include <asm/processor.h>
32
#include <asm/ptrace.h>
C
Catalin Marinas 已提交
33
#include <asm/thread_info.h>
A
Al Viro 已提交
34
#include <asm/asm-uaccess.h>
C
Catalin Marinas 已提交
35 36
#include <asm/unistd.h>

L
Larry Bassel 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

C
Catalin Marinas 已提交
63 64 65 66 67 68 69 70 71
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

72 73
	.macro kernel_ventry	label
	.align 7
74
	sub	sp, sp, #S_FRAME_SIZE
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
#ifdef CONFIG_VMAP_STACK
	/*
	 * Test whether the SP has overflowed, without corrupting a GPR.
	 * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
	 */
	add	sp, sp, x0			// sp' = sp + x0
	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
	tbnz	x0, #THREAD_SHIFT, 0f
	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
	b	\label

0:
	/*
	 * Either we've just detected an overflow, or we've taken an exception
	 * while on the overflow stack. Either way, we won't return to
	 * userspace, and can clobber EL0 registers to free up GPRs.
	 */

	/* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
	msr	tpidr_el0, x0

	/* Recover the original x0 value and stash it in tpidrro_el0 */
	sub	x0, sp, x0
	msr	tpidrro_el0, x0

	/* Switch to the overflow stack */
	adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0

	/*
	 * Check whether we were already on the overflow stack. This may happen
	 * after panic() re-enables interrupts.
	 */
	mrs	x0, tpidr_el0			// sp of interrupted context
	sub	x0, sp, x0			// delta with top of overflow stack
	tst	x0, #~(OVERFLOW_STACK_SIZE - 1)	// within range?
	b.ne	__bad_stack			// no? -> bad stack pointer

	/* We were already on the overflow stack. Restore sp/x0 and carry on. */
	sub	sp, sp, x0
	mrs	x0, tpidrro_el0
#endif
117 118 119 120
	b	\label
	.endm

	.macro	kernel_entry, el, regsize = 64
C
Catalin Marinas 已提交
121 122 123
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

C
Catalin Marinas 已提交
140 141
	.if	\el == 0
	mrs	x21, sp_el0
142 143
	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
144
	disable_step_tsk x19, x20		// exceptions when scheduling.
145 146

	mov	x29, xzr			// fp pointed to user-space
C
Catalin Marinas 已提交
147 148
	.else
	add	x21, sp, #S_FRAME_SIZE
149 150
	get_thread_info tsk
	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
151
	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
152 153
	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
	mov	x20, #TASK_SIZE_64
154
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
155
	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
156
	.endif /* \el == 0 */
C
Catalin Marinas 已提交
157 158 159
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
160

161 162 163 164 165 166 167 168 169 170 171 172
	/*
	 * In order to be able to dump the contents of struct pt_regs at the
	 * time the exception was taken (in case we attempt to walk the call
	 * stack later), chain it together with the stack frames.
	 */
	.if \el == 0
	stp	xzr, xzr, [sp, #S_STACKFRAME]
	.else
	stp	x29, x22, [sp, #S_STACKFRAME]
	.endif
	add	x29, sp, #S_STACKFRAME

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
	 * EL0, there is no need to check the state of TTBR0_EL1 since
	 * accesses are always enabled.
	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
	 * user mappings.
	 */
alternative_if ARM64_HAS_PAN
	b	1f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	mrs	x21, ttbr0_el1
	tst	x21, #0xffff << 48		// Check for the reserved ASID
	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
	b.eq	1f				// TTBR0 access already disabled
	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
	.endif

	__uaccess_ttbr0_disable x21
1:
#endif

C
Catalin Marinas 已提交
198 199
	stp	x22, x23, [sp, #S_PC]

200
	/* Not in a syscall by default (el0_svc overwrites for real syscall) */
C
Catalin Marinas 已提交
201
	.if	\el == 0
202
	mov	w21, #NO_SYSCALL
203
	str	w21, [sp, #S_SYSCALLNO]
C
Catalin Marinas 已提交
204 205
	.endif

206 207 208 209 210 211 212
	/*
	 * Set sp_el0 to current thread_info.
	 */
	.if	\el == 0
	msr	sp_el0, tsk
	.endif

C
Catalin Marinas 已提交
213 214 215 216 217 218 219 220 221
	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

222
	.macro	kernel_exit, el
223
	.if	\el != 0
224 225
	disable_daif

226 227
	/* Restore the task's original addr_limit. */
	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
228
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
229 230 231 232

	/* No need to restore UAO, it will be restored from SPSR_EL1 */
	.endif

C
Catalin Marinas 已提交
233 234
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
L
Larry Bassel 已提交
235
	ct_user_enter
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
	.endif

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
	 * PAN bit checking.
	 */
alternative_if ARM64_HAS_PAN
	b	2f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
	.endif

	__uaccess_ttbr0_enable x0

	.if	\el == 0
	/*
	 * Enable errata workarounds only if returning to user. The only
	 * workaround currently required for TTBR0_EL1 changes are for the
	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
	 * corruption).
	 */
	post_ttbr0_update_workaround
	.endif
1:
	.if	\el != 0
	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
	.endif
2:
#endif

	.if	\el == 0
C
Catalin Marinas 已提交
270
	ldr	x23, [sp, #S_SP]		// load return stack pointer
271
	msr	sp_el0, x23
272
#ifdef CONFIG_ARM64_ERRATUM_845719
M
Mark Rutland 已提交
273
alternative_if ARM64_WORKAROUND_845719
274 275 276 277
	tbz	x22, #4, 1f
#ifdef CONFIG_PID_IN_CONTEXTIDR
	mrs	x29, contextidr_el1
	msr	contextidr_el1, x29
278
#else
279
	msr contextidr_el1, xzr
280
#endif
281
1:
M
Mark Rutland 已提交
282
alternative_else_nop_endif
283
#endif
C
Catalin Marinas 已提交
284
	.endif
285

286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
	ldp	x0, x1, [sp, #16 * 0]
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
C
Catalin Marinas 已提交
305 306 307
	eret					// return to kernel
	.endm

308
	.macro	irq_stack_entry
309 310 311
	mov	x19, sp			// preserve the original sp

	/*
312 313 314
	 * Compare sp with the base of the task stack.
	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
	 * and should switch to the irq stack.
315
	 */
316 317 318 319
	ldr	x25, [tsk, TSK_STACK]
	eor	x25, x25, x19
	and	x25, x25, #~(THREAD_SIZE - 1)
	cbnz	x25, 9998f
320

M
Mark Rutland 已提交
321
	ldr_this_cpu x25, irq_stack_ptr, x26
322
	mov	x26, #IRQ_STACK_SIZE
323
	add	x26, x25, x26
324 325

	/* switch to the irq stack */
326 327 328 329 330 331 332 333 334 335 336 337
	mov	sp, x26
9998:
	.endm

	/*
	 * x19 should be preserved between irq_stack_entry and
	 * irq_stack_exit.
	 */
	.macro	irq_stack_exit
	mov	sp, x19
	.endm

C
Catalin Marinas 已提交
338 339 340 341 342 343
/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
344 345 346
wsc_nr	.req	w25		// number of system calls
wscno	.req	w26		// syscall number
xscno	.req	x26		// syscall number (zero-extended)
C
Catalin Marinas 已提交
347 348 349 350 351 352 353
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
354
	ldr_l	x1, handle_arch_irq
C
Catalin Marinas 已提交
355
	mov	x0, sp
356
	irq_stack_entry
C
Catalin Marinas 已提交
357
	blr	x1
358
	irq_stack_exit
C
Catalin Marinas 已提交
359 360 361 362 363 364 365
	.endm

	.text

/*
 * Exception vectors.
 */
366
	.pushsection ".entry.text", "ax"
C
Catalin Marinas 已提交
367 368 369

	.align	11
ENTRY(vectors)
370 371 372 373
	kernel_ventry	el1_sync_invalid		// Synchronous EL1t
	kernel_ventry	el1_irq_invalid			// IRQ EL1t
	kernel_ventry	el1_fiq_invalid			// FIQ EL1t
	kernel_ventry	el1_error_invalid		// Error EL1t
C
Catalin Marinas 已提交
374

375 376 377 378
	kernel_ventry	el1_sync			// Synchronous EL1h
	kernel_ventry	el1_irq				// IRQ EL1h
	kernel_ventry	el1_fiq_invalid			// FIQ EL1h
	kernel_ventry	el1_error_invalid		// Error EL1h
C
Catalin Marinas 已提交
379

380 381 382 383
	kernel_ventry	el0_sync			// Synchronous 64-bit EL0
	kernel_ventry	el0_irq				// IRQ 64-bit EL0
	kernel_ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	kernel_ventry	el0_error_invalid		// Error 64-bit EL0
C
Catalin Marinas 已提交
384 385

#ifdef CONFIG_COMPAT
386 387 388 389
	kernel_ventry	el0_sync_compat			// Synchronous 32-bit EL0
	kernel_ventry	el0_irq_compat			// IRQ 32-bit EL0
	kernel_ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	kernel_ventry	el0_error_invalid_compat	// Error 32-bit EL0
C
Catalin Marinas 已提交
390
#else
391 392 393 394
	kernel_ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	kernel_ventry	el0_irq_invalid			// IRQ 32-bit EL0
	kernel_ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	kernel_ventry	el0_error_invalid		// Error 32-bit EL0
C
Catalin Marinas 已提交
395 396 397
#endif
END(vectors)

398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
#ifdef CONFIG_VMAP_STACK
	/*
	 * We detected an overflow in kernel_ventry, which switched to the
	 * overflow stack. Stash the exception regs, and head to our overflow
	 * handler.
	 */
__bad_stack:
	/* Restore the original x0 value */
	mrs	x0, tpidrro_el0

	/*
	 * Store the original GPRs to the new stack. The orginal SP (minus
	 * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
	 */
	sub	sp, sp, #S_FRAME_SIZE
	kernel_entry 1
	mrs	x0, tpidr_el0
	add	x0, x0, #S_FRAME_SIZE
	str	x0, [sp, #S_SP]

	/* Stash the regs for handle_bad_stack */
	mov	x0, sp

	/* Time to die */
	bl	handle_bad_stack
	ASM_BUG()
#endif /* CONFIG_VMAP_STACK */

C
Catalin Marinas 已提交
426 427 428 429
/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
430
	kernel_entry \el, \regsize
C
Catalin Marinas 已提交
431 432 433
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
434 435
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
487 488
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
C
Catalin Marinas 已提交
489
	b.eq	el1_da
490 491
	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
	b.eq	el1_ia
M
Mark Rutland 已提交
492
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
C
Catalin Marinas 已提交
493
	b.eq	el1_undef
M
Mark Rutland 已提交
494
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
495
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
496
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
497
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
498
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
C
Catalin Marinas 已提交
499
	b.eq	el1_undef
M
Mark Rutland 已提交
500
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
C
Catalin Marinas 已提交
501 502
	b.ge	el1_dbg
	b	el1_inv
503 504 505 506 507

el1_ia:
	/*
	 * Fall through to the Data abort case
	 */
C
Catalin Marinas 已提交
508 509 510 511
el1_da:
	/*
	 * Data abort handling
	 */
512
	mrs	x3, far_el1
J
James Morse 已提交
513
	inherit_daif	pstate=x23, tmp=x2
514
	clear_address_tag x0, x3
C
Catalin Marinas 已提交
515 516 517 518 519 520 521 522 523
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
J
James Morse 已提交
524
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
525
	mov	x2, sp
526 527
	bl	do_sp_pc_abort
	ASM_BUG()
C
Catalin Marinas 已提交
528 529 530 531
el1_undef:
	/*
	 * Undefined instruction
	 */
J
James Morse 已提交
532
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
533
	mov	x0, sp
534 535
	bl	do_undefinstr
	ASM_BUG()
C
Catalin Marinas 已提交
536 537 538 539
el1_dbg:
	/*
	 * Debug exception handling
	 */
M
Mark Rutland 已提交
540
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
541
	cinc	x24, x24, eq			// set bit '0'
C
Catalin Marinas 已提交
542 543 544 545 546 547 548
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
J
James Morse 已提交
549
	inherit_daif	pstate=x23, tmp=x2
C
Catalin Marinas 已提交
550
	mov	x0, sp
551
	mov	x2, x1
C
Catalin Marinas 已提交
552
	mov	x1, #BAD_SYNC
553 554
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
555 556 557 558 559
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
560
	enable_dbg
C
Catalin Marinas 已提交
561 562 563
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
564

C
Catalin Marinas 已提交
565
	irq_handler
566

C
Catalin Marinas 已提交
567
#ifdef CONFIG_PREEMPT
568
	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
569
	cbnz	w24, 1f				// preempt count != 0
570
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
C
Catalin Marinas 已提交
571 572 573 574 575 576 577 578 579 580 581 582 583
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
584
1:	bl	preempt_schedule_irq		// irq en/disable is done inside
585
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
C
Catalin Marinas 已提交
586 587 588 589 590 591 592 593 594 595 596
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
597 598
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
C
Catalin Marinas 已提交
599
	b.eq	el0_svc
M
Mark Rutland 已提交
600
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
601
	b.eq	el0_da
M
Mark Rutland 已提交
602
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
603
	b.eq	el0_ia
M
Mark Rutland 已提交
604
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
605
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
606
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
C
Catalin Marinas 已提交
607
	b.eq	el0_fpsimd_exc
M
Mark Rutland 已提交
608
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
609
	b.eq	el0_sys
M
Mark Rutland 已提交
610
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
611
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
612
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
613
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
614
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
615
	b.eq	el0_undef
M
Mark Rutland 已提交
616
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
617 618 619 620 621 622 623 624
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
625 626
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
C
Catalin Marinas 已提交
627
	b.eq	el0_svc_compat
M
Mark Rutland 已提交
628
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
629
	b.eq	el0_da
M
Mark Rutland 已提交
630
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
631
	b.eq	el0_ia
M
Mark Rutland 已提交
632
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
633
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
634
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
C
Catalin Marinas 已提交
635
	b.eq	el0_fpsimd_exc
636 637
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
638
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
639
	b.eq	el0_undef
M
Mark Rutland 已提交
640
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
641
	b.eq	el0_undef
M
Mark Rutland 已提交
642
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
643
	b.eq	el0_undef
M
Mark Rutland 已提交
644
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
645
	b.eq	el0_undef
M
Mark Rutland 已提交
646
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
647
	b.eq	el0_undef
M
Mark Rutland 已提交
648
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
649
	b.eq	el0_undef
M
Mark Rutland 已提交
650
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
651 652 653 654 655 656
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
657
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
658 659
	mov	wscno, w7			// syscall number in w7 (r7)
	mov     wsc_nr, #__NR_compat_syscalls
C
Catalin Marinas 已提交
660 661 662 663 664 665 666 667 668 669 670 671
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
672
	mrs	x26, far_el1
C
Catalin Marinas 已提交
673
	// enable interrupts before calling the main handler
674
	enable_dbg_and_irq
L
Larry Bassel 已提交
675
	ct_user_exit
676
	clear_address_tag x0, x26
C
Catalin Marinas 已提交
677 678
	mov	x1, x25
	mov	x2, sp
679 680
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
681 682 683 684
el0_ia:
	/*
	 * Instruction abort handling
	 */
685
	mrs	x26, far_el1
C
Catalin Marinas 已提交
686
	// enable interrupts before calling the main handler
687
	enable_dbg_and_irq
L
Larry Bassel 已提交
688
	ct_user_exit
689
	mov	x0, x26
M
Mark Rutland 已提交
690
	mov	x1, x25
C
Catalin Marinas 已提交
691
	mov	x2, sp
692 693
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
694 695 696 697
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
698
	enable_dbg
L
Larry Bassel 已提交
699
	ct_user_exit
C
Catalin Marinas 已提交
700 701
	mov	x0, x25
	mov	x1, sp
702 703
	bl	do_fpsimd_acc
	b	ret_to_user
C
Catalin Marinas 已提交
704 705 706 707
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
708
	enable_dbg
L
Larry Bassel 已提交
709
	ct_user_exit
C
Catalin Marinas 已提交
710 711
	mov	x0, x25
	mov	x1, sp
712 713
	bl	do_fpsimd_exc
	b	ret_to_user
C
Catalin Marinas 已提交
714 715 716 717
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
718
	mrs	x26, far_el1
C
Catalin Marinas 已提交
719
	// enable interrupts before calling the main handler
720
	enable_dbg_and_irq
721
	ct_user_exit
722
	mov	x0, x26
C
Catalin Marinas 已提交
723 724
	mov	x1, x25
	mov	x2, sp
725 726
	bl	do_sp_pc_abort
	b	ret_to_user
C
Catalin Marinas 已提交
727 728 729 730
el0_undef:
	/*
	 * Undefined instruction
	 */
731
	// enable interrupts before calling the main handler
732
	enable_dbg_and_irq
L
Larry Bassel 已提交
733
	ct_user_exit
734
	mov	x0, sp
735 736
	bl	do_undefinstr
	b	ret_to_user
737 738 739 740 741 742 743 744 745 746
el0_sys:
	/*
	 * System instructions, for trapped cache maintenance instructions
	 */
	enable_dbg_and_irq
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sysinstr
	b	ret_to_user
C
Catalin Marinas 已提交
747 748 749 750 751 752 753 754
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
755 756
	bl	do_debug_exception
	enable_dbg
L
Larry Bassel 已提交
757
	ct_user_exit
758
	b	ret_to_user
C
Catalin Marinas 已提交
759
el0_inv:
760
	enable_dbg
L
Larry Bassel 已提交
761
	ct_user_exit
C
Catalin Marinas 已提交
762 763
	mov	x0, sp
	mov	x1, #BAD_SYNC
764
	mov	x2, x25
765
	bl	bad_el0_sync
766
	b	ret_to_user
C
Catalin Marinas 已提交
767 768 769 770 771 772 773 774 775 776
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
777

L
Larry Bassel 已提交
778
	ct_user_exit
C
Catalin Marinas 已提交
779
	irq_handler
780

C
Catalin Marinas 已提交
781 782 783 784 785 786 787 788 789 790 791
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
792
	disable_daif
793
	str	x0, [sp, #S_X0]			// returned x0
794
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
795 796
	and	x2, x1, #_TIF_SYSCALL_WORK
	cbnz	x2, ret_fast_syscall_trace
C
Catalin Marinas 已提交
797
	and	x2, x1, #_TIF_WORK_MASK
798
	cbnz	x2, work_pending
799
	enable_step_tsk x1, x2
800
	kernel_exit 0
801
ret_fast_syscall_trace:
802
	enable_daif
803
	b	__sys_trace_return_skipped	// we already saved x0
C
Catalin Marinas 已提交
804 805 806 807 808 809 810

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
work_pending:
	mov	x0, sp				// 'regs'
	bl	do_notify_resume
811
#ifdef CONFIG_TRACE_IRQFLAGS
812
	bl	trace_hardirqs_on		// enabled while in userspace
813
#endif
814
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
815
	b	finish_ret_to_user
C
Catalin Marinas 已提交
816 817 818
/*
 * "slow" syscall return path.
 */
819
ret_to_user:
820
	disable_daif
821
	ldr	x1, [tsk, #TSK_TI_FLAGS]
C
Catalin Marinas 已提交
822 823
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
824
finish_ret_to_user:
825
	enable_step_tsk x1, x2
826
	kernel_exit 0
C
Catalin Marinas 已提交
827 828 829 830 831 832 833 834
ENDPROC(ret_to_user)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
835 836
	mov	wscno, w8			// syscall number in w8
	mov	wsc_nr, #__NR_syscalls
C
Catalin Marinas 已提交
837
el0_svc_naked:					// compat entry point
838
	stp	x0, xscno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
839
	enable_dbg_and_irq
L
Larry Bassel 已提交
840
	ct_user_exit 1
C
Catalin Marinas 已提交
841

842
	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
843 844
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
845
	cmp     wscno, wsc_nr			// check upper syscall limit
C
Catalin Marinas 已提交
846
	b.hs	ni_sys
847
	ldr	x16, [stbl, xscno, lsl #3]	// address in the syscall table
848 849
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
C
Catalin Marinas 已提交
850 851
ni_sys:
	mov	x0, sp
852 853
	bl	do_ni_syscall
	b	ret_fast_syscall
C
Catalin Marinas 已提交
854 855 856 857 858 859 860
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
861
	cmp     wscno, #NO_SYSCALL		// user-issued syscall(-1)?
862
	b.ne	1f
863
	mov	x0, #-ENOSYS			// set default errno if so
864 865
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
866
	bl	syscall_trace_enter
867
	cmp	w0, #NO_SYSCALL			// skip the syscall?
868
	b.eq	__sys_trace_return_skipped
869
	mov	wscno, w0			// syscall number (possibly new)
C
Catalin Marinas 已提交
870
	mov	x1, sp				// pointer to regs
871
	cmp	wscno, wsc_nr			// check upper syscall limit
872
	b.hs	__ni_sys_trace
C
Catalin Marinas 已提交
873 874 875 876
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
877
	ldr	x16, [stbl, xscno, lsl #3]	// address in the syscall table
878
	blr	x16				// call sys_* routine
C
Catalin Marinas 已提交
879 880

__sys_trace_return:
881 882
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
883 884
	mov	x0, sp
	bl	syscall_trace_exit
C
Catalin Marinas 已提交
885 886
	b	ret_to_user

887 888 889 890 891
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

892 893
	.popsection				// .entry.text

C
Catalin Marinas 已提交
894 895 896 897 898 899 900
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
	mov	x10, #THREAD_CPU_CONTEXT
	add	x8, x0, x10
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
	add	x8, x1, x10
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
	msr	sp_el0, x1
	ret
ENDPROC(cpu_switch_to)
NOKPROBE(cpu_switch_to)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
	b	ret_to_user
ENDPROC(ret_from_fork)
NOKPROBE(ret_from_fork)