entry.S 19.5 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative.h>
C
Catalin Marinas 已提交
25 26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
C
Catalin Marinas 已提交
28
#include <asm/errno.h>
29
#include <asm/esr.h>
30
#include <asm/irq.h>
31
#include <asm/memory.h>
32
#include <asm/ptrace.h>
C
Catalin Marinas 已提交
33
#include <asm/thread_info.h>
A
Al Viro 已提交
34
#include <asm/asm-uaccess.h>
C
Catalin Marinas 已提交
35 36
#include <asm/unistd.h>

L
Larry Bassel 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

C
Catalin Marinas 已提交
63 64 65 66 67 68 69 70 71 72
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

	.macro	kernel_entry, el, regsize = 64
73
	sub	sp, sp, #S_FRAME_SIZE
C
Catalin Marinas 已提交
74 75 76
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

C
Catalin Marinas 已提交
93 94
	.if	\el == 0
	mrs	x21, sp_el0
95 96
	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
97
	disable_step_tsk x19, x20		// exceptions when scheduling.
98 99

	mov	x29, xzr			// fp pointed to user-space
C
Catalin Marinas 已提交
100 101
	.else
	add	x21, sp, #S_FRAME_SIZE
102 103
	get_thread_info tsk
	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
104
	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
105 106
	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
	mov	x20, #TASK_SIZE_64
107
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
108
	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
109
	.endif /* \el == 0 */
C
Catalin Marinas 已提交
110 111 112
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
	 * EL0, there is no need to check the state of TTBR0_EL1 since
	 * accesses are always enabled.
	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
	 * user mappings.
	 */
alternative_if ARM64_HAS_PAN
	b	1f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	mrs	x21, ttbr0_el1
	tst	x21, #0xffff << 48		// Check for the reserved ASID
	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
	b.eq	1f				// TTBR0 access already disabled
	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
	.endif

	__uaccess_ttbr0_disable x21
1:
#endif

C
Catalin Marinas 已提交
139 140 141 142 143 144 145 146 147 148
	stp	x22, x23, [sp, #S_PC]

	/*
	 * Set syscallno to -1 by default (overridden later if real syscall).
	 */
	.if	\el == 0
	mvn	x21, xzr
	str	x21, [sp, #S_SYSCALLNO]
	.endif

149 150 151 152 153 154 155
	/*
	 * Set sp_el0 to current thread_info.
	 */
	.if	\el == 0
	msr	sp_el0, tsk
	.endif

C
Catalin Marinas 已提交
156 157 158 159 160 161 162 163 164
	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

165
	.macro	kernel_exit, el
166 167 168
	.if	\el != 0
	/* Restore the task's original addr_limit. */
	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
169
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
170 171 172 173

	/* No need to restore UAO, it will be restored from SPSR_EL1 */
	.endif

C
Catalin Marinas 已提交
174 175
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
L
Larry Bassel 已提交
176
	ct_user_enter
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	.endif

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
	 * PAN bit checking.
	 */
alternative_if ARM64_HAS_PAN
	b	2f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
	.endif

	__uaccess_ttbr0_enable x0

	.if	\el == 0
	/*
	 * Enable errata workarounds only if returning to user. The only
	 * workaround currently required for TTBR0_EL1 changes are for the
	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
	 * corruption).
	 */
	post_ttbr0_update_workaround
	.endif
1:
	.if	\el != 0
	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
	.endif
2:
#endif

	.if	\el == 0
C
Catalin Marinas 已提交
211
	ldr	x23, [sp, #S_SP]		// load return stack pointer
212
	msr	sp_el0, x23
213
#ifdef CONFIG_ARM64_ERRATUM_845719
M
Mark Rutland 已提交
214
alternative_if ARM64_WORKAROUND_845719
215 216 217 218
	tbz	x22, #4, 1f
#ifdef CONFIG_PID_IN_CONTEXTIDR
	mrs	x29, contextidr_el1
	msr	contextidr_el1, x29
219
#else
220
	msr contextidr_el1, xzr
221
#endif
222
1:
M
Mark Rutland 已提交
223
alternative_else_nop_endif
224
#endif
C
Catalin Marinas 已提交
225
	.endif
226

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
	ldp	x0, x1, [sp, #16 * 0]
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
C
Catalin Marinas 已提交
246 247 248
	eret					// return to kernel
	.endm

249
	.macro	irq_stack_entry
250 251 252
	mov	x19, sp			// preserve the original sp

	/*
253 254 255
	 * Compare sp with the base of the task stack.
	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
	 * and should switch to the irq stack.
256
	 */
257 258 259 260
	ldr	x25, [tsk, TSK_STACK]
	eor	x25, x25, x19
	and	x25, x25, #~(THREAD_SIZE - 1)
	cbnz	x25, 9998f
261

262
	adr_this_cpu x25, irq_stack, x26
263 264
	mov	x26, #IRQ_STACK_START_SP
	add	x26, x25, x26
265 266

	/* switch to the irq stack */
267 268
	mov	sp, x26

269 270 271 272 273
	/*
	 * Add a dummy stack frame, this non-standard format is fixed up
	 * by unwind_frame()
	 */
	stp     x29, x19, [sp, #-16]!
274 275 276 277 278 279 280 281 282 283 284 285 286
	mov	x29, sp

9998:
	.endm

	/*
	 * x19 should be preserved between irq_stack_entry and
	 * irq_stack_exit.
	 */
	.macro	irq_stack_exit
	mov	sp, x19
	.endm

C
Catalin Marinas 已提交
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
sc_nr	.req	x25		// number of system calls
scno	.req	x26		// syscall number
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
302
	ldr_l	x1, handle_arch_irq
C
Catalin Marinas 已提交
303
	mov	x0, sp
304
	irq_stack_entry
C
Catalin Marinas 已提交
305
	blr	x1
306
	irq_stack_exit
C
Catalin Marinas 已提交
307 308 309 310 311 312 313
	.endm

	.text

/*
 * Exception vectors.
 */
314
	.pushsection ".entry.text", "ax"
C
Catalin Marinas 已提交
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	.align	11
ENTRY(vectors)
	ventry	el1_sync_invalid		// Synchronous EL1t
	ventry	el1_irq_invalid			// IRQ EL1t
	ventry	el1_fiq_invalid			// FIQ EL1t
	ventry	el1_error_invalid		// Error EL1t

	ventry	el1_sync			// Synchronous EL1h
	ventry	el1_irq				// IRQ EL1h
	ventry	el1_fiq_invalid			// FIQ EL1h
	ventry	el1_error_invalid		// Error EL1h

	ventry	el0_sync			// Synchronous 64-bit EL0
	ventry	el0_irq				// IRQ 64-bit EL0
	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	ventry	el0_error_invalid		// Error 64-bit EL0

#ifdef CONFIG_COMPAT
	ventry	el0_sync_compat			// Synchronous 32-bit EL0
	ventry	el0_irq_compat			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	ventry	el0_error_invalid_compat	// Error 32-bit EL0
#else
	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	ventry	el0_irq_invalid			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	ventry	el0_error_invalid		// Error 32-bit EL0
#endif
END(vectors)

/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
350
	kernel_entry \el, \regsize
C
Catalin Marinas 已提交
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
	b	bad_mode
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
406 407
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
C
Catalin Marinas 已提交
408
	b.eq	el1_da
409 410
	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
	b.eq	el1_ia
M
Mark Rutland 已提交
411
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
C
Catalin Marinas 已提交
412
	b.eq	el1_undef
M
Mark Rutland 已提交
413
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
414
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
415
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
416
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
417
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
C
Catalin Marinas 已提交
418
	b.eq	el1_undef
M
Mark Rutland 已提交
419
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
C
Catalin Marinas 已提交
420 421
	b.ge	el1_dbg
	b	el1_inv
422 423 424 425 426

el1_ia:
	/*
	 * Fall through to the Data abort case
	 */
C
Catalin Marinas 已提交
427 428 429 430
el1_da:
	/*
	 * Data abort handling
	 */
431
	mrs	x3, far_el1
432
	enable_dbg
C
Catalin Marinas 已提交
433 434 435 436
	// re-enable interrupts if they were enabled in the aborted context
	tbnz	x23, #7, 1f			// PSR_I_BIT
	enable_irq
1:
437
	clear_address_tag x0, x3
C
Catalin Marinas 已提交
438 439 440 441 442 443 444 445 446 447 448
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	// disable interrupts before pulling preserved data off the stack
	disable_irq
	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
449
	enable_dbg
C
Catalin Marinas 已提交
450 451 452 453 454 455
	mov	x2, sp
	b	do_sp_pc_abort
el1_undef:
	/*
	 * Undefined instruction
	 */
456
	enable_dbg
C
Catalin Marinas 已提交
457 458 459 460 461 462
	mov	x0, sp
	b	do_undefinstr
el1_dbg:
	/*
	 * Debug exception handling
	 */
M
Mark Rutland 已提交
463
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
464
	cinc	x24, x24, eq			// set bit '0'
C
Catalin Marinas 已提交
465 466 467 468 469 470 471
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
472
	enable_dbg
C
Catalin Marinas 已提交
473
	mov	x0, sp
474
	mov	x2, x1
C
Catalin Marinas 已提交
475 476 477 478 479 480 481
	mov	x1, #BAD_SYNC
	b	bad_mode
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
482
	enable_dbg
C
Catalin Marinas 已提交
483 484 485
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
486

C
Catalin Marinas 已提交
487
	irq_handler
488

C
Catalin Marinas 已提交
489
#ifdef CONFIG_PREEMPT
490
	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
491
	cbnz	w24, 1f				// preempt count != 0
492
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
C
Catalin Marinas 已提交
493 494 495 496 497 498 499 500 501 502 503 504 505
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
506
1:	bl	preempt_schedule_irq		// irq en/disable is done inside
507
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
C
Catalin Marinas 已提交
508 509 510 511 512 513 514 515 516 517 518
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
519 520
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
C
Catalin Marinas 已提交
521
	b.eq	el0_svc
M
Mark Rutland 已提交
522
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
523
	b.eq	el0_da
M
Mark Rutland 已提交
524
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
525
	b.eq	el0_ia
M
Mark Rutland 已提交
526
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
527
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
528
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
C
Catalin Marinas 已提交
529
	b.eq	el0_fpsimd_exc
M
Mark Rutland 已提交
530
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
531
	b.eq	el0_sys
M
Mark Rutland 已提交
532
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
533
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
534
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
535
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
536
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
537
	b.eq	el0_undef
M
Mark Rutland 已提交
538
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
539 540 541 542 543 544 545 546
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
547 548
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
C
Catalin Marinas 已提交
549
	b.eq	el0_svc_compat
M
Mark Rutland 已提交
550
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
551
	b.eq	el0_da
M
Mark Rutland 已提交
552
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
553
	b.eq	el0_ia
M
Mark Rutland 已提交
554
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
555
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
556
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
C
Catalin Marinas 已提交
557
	b.eq	el0_fpsimd_exc
558 559
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
560
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
561
	b.eq	el0_undef
M
Mark Rutland 已提交
562
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
563
	b.eq	el0_undef
M
Mark Rutland 已提交
564
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
565
	b.eq	el0_undef
M
Mark Rutland 已提交
566
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
567
	b.eq	el0_undef
M
Mark Rutland 已提交
568
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
569
	b.eq	el0_undef
M
Mark Rutland 已提交
570
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
571
	b.eq	el0_undef
M
Mark Rutland 已提交
572
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
573 574 575 576 577 578
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
579
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
C
Catalin Marinas 已提交
580 581 582 583 584 585 586 587 588 589 590 591 592 593
	uxtw	scno, w7			// syscall number in w7 (r7)
	mov     sc_nr, #__NR_compat_syscalls
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
594
	mrs	x26, far_el1
C
Catalin Marinas 已提交
595
	// enable interrupts before calling the main handler
596
	enable_dbg_and_irq
L
Larry Bassel 已提交
597
	ct_user_exit
598
	clear_address_tag x0, x26
C
Catalin Marinas 已提交
599 600
	mov	x1, x25
	mov	x2, sp
601 602
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
603 604 605 606
el0_ia:
	/*
	 * Instruction abort handling
	 */
607
	mrs	x26, far_el1
C
Catalin Marinas 已提交
608
	// enable interrupts before calling the main handler
609
	enable_dbg_and_irq
L
Larry Bassel 已提交
610
	ct_user_exit
611
	mov	x0, x26
M
Mark Rutland 已提交
612
	mov	x1, x25
C
Catalin Marinas 已提交
613
	mov	x2, sp
614 615
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
616 617 618 619
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
620
	enable_dbg
L
Larry Bassel 已提交
621
	ct_user_exit
C
Catalin Marinas 已提交
622 623
	mov	x0, x25
	mov	x1, sp
624 625
	bl	do_fpsimd_acc
	b	ret_to_user
C
Catalin Marinas 已提交
626 627 628 629
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
630
	enable_dbg
L
Larry Bassel 已提交
631
	ct_user_exit
C
Catalin Marinas 已提交
632 633
	mov	x0, x25
	mov	x1, sp
634 635
	bl	do_fpsimd_exc
	b	ret_to_user
C
Catalin Marinas 已提交
636 637 638 639
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
640
	mrs	x26, far_el1
C
Catalin Marinas 已提交
641
	// enable interrupts before calling the main handler
642
	enable_dbg_and_irq
643
	ct_user_exit
644
	mov	x0, x26
C
Catalin Marinas 已提交
645 646
	mov	x1, x25
	mov	x2, sp
647 648
	bl	do_sp_pc_abort
	b	ret_to_user
C
Catalin Marinas 已提交
649 650 651 652
el0_undef:
	/*
	 * Undefined instruction
	 */
653
	// enable interrupts before calling the main handler
654
	enable_dbg_and_irq
L
Larry Bassel 已提交
655
	ct_user_exit
656
	mov	x0, sp
657 658
	bl	do_undefinstr
	b	ret_to_user
659 660 661 662 663 664 665 666 667 668
el0_sys:
	/*
	 * System instructions, for trapped cache maintenance instructions
	 */
	enable_dbg_and_irq
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sysinstr
	b	ret_to_user
C
Catalin Marinas 已提交
669 670 671 672 673 674 675 676
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
677 678
	bl	do_debug_exception
	enable_dbg
L
Larry Bassel 已提交
679
	ct_user_exit
680
	b	ret_to_user
C
Catalin Marinas 已提交
681
el0_inv:
682
	enable_dbg
L
Larry Bassel 已提交
683
	ct_user_exit
C
Catalin Marinas 已提交
684 685
	mov	x0, sp
	mov	x1, #BAD_SYNC
686
	mov	x2, x25
687
	bl	bad_el0_sync
688
	b	ret_to_user
C
Catalin Marinas 已提交
689 690 691 692 693 694 695 696 697 698
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
699

L
Larry Bassel 已提交
700
	ct_user_exit
C
Catalin Marinas 已提交
701
	irq_handler
702

C
Catalin Marinas 已提交
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
718 719
	mov	x10, #THREAD_CPU_CONTEXT
	add	x8, x0, x10
C
Catalin Marinas 已提交
720 721 722 723 724 725 726 727
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
728
	add	x8, x1, x10
C
Catalin Marinas 已提交
729 730 731 732 733 734 735 736
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
737
	msr	sp_el0, x1
C
Catalin Marinas 已提交
738 739 740 741 742 743 744 745 746
	ret
ENDPROC(cpu_switch_to)

/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
	disable_irq				// disable interrupts
747
	str	x0, [sp, #S_X0]			// returned x0
748
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
749 750
	and	x2, x1, #_TIF_SYSCALL_WORK
	cbnz	x2, ret_fast_syscall_trace
C
Catalin Marinas 已提交
751
	and	x2, x1, #_TIF_WORK_MASK
752
	cbnz	x2, work_pending
753
	enable_step_tsk x1, x2
754
	kernel_exit 0
755 756
ret_fast_syscall_trace:
	enable_irq				// enable interrupts
757
	b	__sys_trace_return_skipped	// we already saved x0
C
Catalin Marinas 已提交
758 759 760 761 762 763 764

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
work_pending:
	mov	x0, sp				// 'regs'
	bl	do_notify_resume
765
#ifdef CONFIG_TRACE_IRQFLAGS
766
	bl	trace_hardirqs_on		// enabled while in userspace
767
#endif
768
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
769
	b	finish_ret_to_user
C
Catalin Marinas 已提交
770 771 772
/*
 * "slow" syscall return path.
 */
773
ret_to_user:
C
Catalin Marinas 已提交
774
	disable_irq				// disable interrupts
775
	ldr	x1, [tsk, #TSK_TI_FLAGS]
C
Catalin Marinas 已提交
776 777
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
778
finish_ret_to_user:
779
	enable_step_tsk x1, x2
780
	kernel_exit 0
C
Catalin Marinas 已提交
781 782 783 784 785 786 787
ENDPROC(ret_to_user)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
788 789 790 791
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
C
Catalin Marinas 已提交
792 793 794 795 796 797 798 799 800 801 802 803 804
	b	ret_to_user
ENDPROC(ret_from_fork)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
	uxtw	scno, w8			// syscall number in w8
	mov	sc_nr, #__NR_syscalls
el0_svc_naked:					// compat entry point
	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
805
	enable_dbg_and_irq
L
Larry Bassel 已提交
806
	ct_user_exit 1
C
Catalin Marinas 已提交
807

808
	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
809 810
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
C
Catalin Marinas 已提交
811 812 813
	cmp     scno, sc_nr                     // check upper syscall limit
	b.hs	ni_sys
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
814 815
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
C
Catalin Marinas 已提交
816 817
ni_sys:
	mov	x0, sp
818 819
	bl	do_ni_syscall
	b	ret_fast_syscall
C
Catalin Marinas 已提交
820 821 822 823 824 825 826
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
827 828 829 830 831 832
	mov	w0, #-1				// set default errno for
	cmp     scno, x0			// user-issued syscall(-1)
	b.ne	1f
	mov	x0, #-ENOSYS
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
833
	bl	syscall_trace_enter
834 835
	cmp	w0, #-1				// skip the syscall?
	b.eq	__sys_trace_return_skipped
C
Catalin Marinas 已提交
836 837 838
	uxtw	scno, w0			// syscall number (possibly new)
	mov	x1, sp				// pointer to regs
	cmp	scno, sc_nr			// check upper syscall limit
839
	b.hs	__ni_sys_trace
C
Catalin Marinas 已提交
840 841 842 843 844
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
845
	blr	x16				// call sys_* routine
C
Catalin Marinas 已提交
846 847

__sys_trace_return:
848 849
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
850 851
	mov	x0, sp
	bl	syscall_trace_exit
C
Catalin Marinas 已提交
852 853
	b	ret_to_user

854 855 856 857 858
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

859 860
	.popsection				// .entry.text

C
Catalin Marinas 已提交
861 862 863 864 865 866 867
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)