entry.S 19.6 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative.h>
C
Catalin Marinas 已提交
25 26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
C
Catalin Marinas 已提交
28
#include <asm/errno.h>
29
#include <asm/esr.h>
30
#include <asm/irq.h>
31
#include <asm/memory.h>
32
#include <asm/ptrace.h>
C
Catalin Marinas 已提交
33
#include <asm/thread_info.h>
A
Al Viro 已提交
34
#include <asm/asm-uaccess.h>
C
Catalin Marinas 已提交
35 36
#include <asm/unistd.h>

L
Larry Bassel 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

C
Catalin Marinas 已提交
63 64 65 66 67 68 69 70 71 72
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

	.macro	kernel_entry, el, regsize = 64
73
	sub	sp, sp, #S_FRAME_SIZE
C
Catalin Marinas 已提交
74 75 76
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

C
Catalin Marinas 已提交
93 94
	.if	\el == 0
	mrs	x21, sp_el0
95 96
	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
97
	disable_step_tsk x19, x20		// exceptions when scheduling.
98 99

	mov	x29, xzr			// fp pointed to user-space
C
Catalin Marinas 已提交
100 101
	.else
	add	x21, sp, #S_FRAME_SIZE
102 103
	get_thread_info tsk
	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
104
	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
105 106
	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
	mov	x20, #TASK_SIZE_64
107
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
108
	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
109
	.endif /* \el == 0 */
C
Catalin Marinas 已提交
110 111 112
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
	 * EL0, there is no need to check the state of TTBR0_EL1 since
	 * accesses are always enabled.
	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
	 * user mappings.
	 */
alternative_if ARM64_HAS_PAN
	b	1f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	mrs	x21, ttbr0_el1
	tst	x21, #0xffff << 48		// Check for the reserved ASID
	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
	b.eq	1f				// TTBR0 access already disabled
	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
	.endif

	__uaccess_ttbr0_disable x21
1:
#endif

C
Catalin Marinas 已提交
139 140 141 142 143 144 145 146 147 148
	stp	x22, x23, [sp, #S_PC]

	/*
	 * Set syscallno to -1 by default (overridden later if real syscall).
	 */
	.if	\el == 0
	mvn	x21, xzr
	str	x21, [sp, #S_SYSCALLNO]
	.endif

149 150 151 152 153 154 155
	/*
	 * Set sp_el0 to current thread_info.
	 */
	.if	\el == 0
	msr	sp_el0, tsk
	.endif

C
Catalin Marinas 已提交
156 157 158 159 160 161 162 163 164
	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

165
	.macro	kernel_exit, el
166 167 168
	.if	\el != 0
	/* Restore the task's original addr_limit. */
	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
169
	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
170 171 172 173

	/* No need to restore UAO, it will be restored from SPSR_EL1 */
	.endif

C
Catalin Marinas 已提交
174 175
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
L
Larry Bassel 已提交
176
	ct_user_enter
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	.endif

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
	/*
	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
	 * PAN bit checking.
	 */
alternative_if ARM64_HAS_PAN
	b	2f				// skip TTBR0 PAN
alternative_else_nop_endif

	.if	\el != 0
	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
	.endif

	__uaccess_ttbr0_enable x0

	.if	\el == 0
	/*
	 * Enable errata workarounds only if returning to user. The only
	 * workaround currently required for TTBR0_EL1 changes are for the
	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
	 * corruption).
	 */
	post_ttbr0_update_workaround
	.endif
1:
	.if	\el != 0
	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
	.endif
2:
#endif

	.if	\el == 0
C
Catalin Marinas 已提交
211
	ldr	x23, [sp, #S_SP]		// load return stack pointer
212
	msr	sp_el0, x23
213
#ifdef CONFIG_ARM64_ERRATUM_845719
M
Mark Rutland 已提交
214
alternative_if ARM64_WORKAROUND_845719
215 216 217 218
	tbz	x22, #4, 1f
#ifdef CONFIG_PID_IN_CONTEXTIDR
	mrs	x29, contextidr_el1
	msr	contextidr_el1, x29
219
#else
220
	msr contextidr_el1, xzr
221
#endif
222
1:
M
Mark Rutland 已提交
223
alternative_else_nop_endif
224
#endif
C
Catalin Marinas 已提交
225
	.endif
226

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
	ldp	x0, x1, [sp, #16 * 0]
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
C
Catalin Marinas 已提交
246 247 248
	eret					// return to kernel
	.endm

249
	.macro	irq_stack_entry
250 251 252
	mov	x19, sp			// preserve the original sp

	/*
253 254 255
	 * Compare sp with the base of the task stack.
	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
	 * and should switch to the irq stack.
256
	 */
257 258 259 260
	ldr	x25, [tsk, TSK_STACK]
	eor	x25, x25, x19
	and	x25, x25, #~(THREAD_SIZE - 1)
	cbnz	x25, 9998f
261

262
	adr_this_cpu x25, irq_stack, x26
263 264
	mov	x26, #IRQ_STACK_START_SP
	add	x26, x25, x26
265 266

	/* switch to the irq stack */
267 268
	mov	sp, x26

269 270 271 272 273
	/*
	 * Add a dummy stack frame, this non-standard format is fixed up
	 * by unwind_frame()
	 */
	stp     x29, x19, [sp, #-16]!
274 275 276 277 278 279 280 281 282 283 284 285 286
	mov	x29, sp

9998:
	.endm

	/*
	 * x19 should be preserved between irq_stack_entry and
	 * irq_stack_exit.
	 */
	.macro	irq_stack_exit
	mov	sp, x19
	.endm

C
Catalin Marinas 已提交
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
sc_nr	.req	x25		// number of system calls
scno	.req	x26		// syscall number
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
302
	ldr_l	x1, handle_arch_irq
C
Catalin Marinas 已提交
303
	mov	x0, sp
304
	irq_stack_entry
C
Catalin Marinas 已提交
305
	blr	x1
306
	irq_stack_exit
C
Catalin Marinas 已提交
307 308 309 310 311 312 313
	.endm

	.text

/*
 * Exception vectors.
 */
314
	.pushsection ".entry.text", "ax"
C
Catalin Marinas 已提交
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	.align	11
ENTRY(vectors)
	ventry	el1_sync_invalid		// Synchronous EL1t
	ventry	el1_irq_invalid			// IRQ EL1t
	ventry	el1_fiq_invalid			// FIQ EL1t
	ventry	el1_error_invalid		// Error EL1t

	ventry	el1_sync			// Synchronous EL1h
	ventry	el1_irq				// IRQ EL1h
	ventry	el1_fiq_invalid			// FIQ EL1h
	ventry	el1_error_invalid		// Error EL1h

	ventry	el0_sync			// Synchronous 64-bit EL0
	ventry	el0_irq				// IRQ 64-bit EL0
	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	ventry	el0_error_invalid		// Error 64-bit EL0

#ifdef CONFIG_COMPAT
	ventry	el0_sync_compat			// Synchronous 32-bit EL0
	ventry	el0_irq_compat			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	ventry	el0_error_invalid_compat	// Error 32-bit EL0
#else
	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	ventry	el0_irq_invalid			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	ventry	el0_error_invalid		// Error 32-bit EL0
#endif
END(vectors)

/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
350
	kernel_entry \el, \regsize
C
Catalin Marinas 已提交
351 352 353
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
354 355
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
407 408
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
C
Catalin Marinas 已提交
409
	b.eq	el1_da
410 411
	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
	b.eq	el1_ia
M
Mark Rutland 已提交
412
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
C
Catalin Marinas 已提交
413
	b.eq	el1_undef
M
Mark Rutland 已提交
414
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
415
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
416
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
417
	b.eq	el1_sp_pc
M
Mark Rutland 已提交
418
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
C
Catalin Marinas 已提交
419
	b.eq	el1_undef
M
Mark Rutland 已提交
420
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
C
Catalin Marinas 已提交
421 422
	b.ge	el1_dbg
	b	el1_inv
423 424 425 426 427

el1_ia:
	/*
	 * Fall through to the Data abort case
	 */
C
Catalin Marinas 已提交
428 429 430 431
el1_da:
	/*
	 * Data abort handling
	 */
432
	mrs	x3, far_el1
433
	enable_dbg
C
Catalin Marinas 已提交
434 435 436 437
	// re-enable interrupts if they were enabled in the aborted context
	tbnz	x23, #7, 1f			// PSR_I_BIT
	enable_irq
1:
438
	clear_address_tag x0, x3
C
Catalin Marinas 已提交
439 440 441 442 443 444 445 446 447 448 449
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	// disable interrupts before pulling preserved data off the stack
	disable_irq
	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
450
	enable_dbg
C
Catalin Marinas 已提交
451
	mov	x2, sp
452 453
	bl	do_sp_pc_abort
	ASM_BUG()
C
Catalin Marinas 已提交
454 455 456 457
el1_undef:
	/*
	 * Undefined instruction
	 */
458
	enable_dbg
C
Catalin Marinas 已提交
459
	mov	x0, sp
460 461
	bl	do_undefinstr
	ASM_BUG()
C
Catalin Marinas 已提交
462 463 464 465
el1_dbg:
	/*
	 * Debug exception handling
	 */
M
Mark Rutland 已提交
466
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
467
	cinc	x24, x24, eq			// set bit '0'
C
Catalin Marinas 已提交
468 469 470 471 472 473 474
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
475
	enable_dbg
C
Catalin Marinas 已提交
476
	mov	x0, sp
477
	mov	x2, x1
C
Catalin Marinas 已提交
478
	mov	x1, #BAD_SYNC
479 480
	bl	bad_mode
	ASM_BUG()
C
Catalin Marinas 已提交
481 482 483 484 485
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
486
	enable_dbg
C
Catalin Marinas 已提交
487 488 489
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
490

C
Catalin Marinas 已提交
491
	irq_handler
492

C
Catalin Marinas 已提交
493
#ifdef CONFIG_PREEMPT
494
	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
495
	cbnz	w24, 1f				// preempt count != 0
496
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
C
Catalin Marinas 已提交
497 498 499 500 501 502 503 504 505 506 507 508 509
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
510
1:	bl	preempt_schedule_irq		// irq en/disable is done inside
511
	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
C
Catalin Marinas 已提交
512 513 514 515 516 517 518 519 520 521 522
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
523 524
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
C
Catalin Marinas 已提交
525
	b.eq	el0_svc
M
Mark Rutland 已提交
526
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
527
	b.eq	el0_da
M
Mark Rutland 已提交
528
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
529
	b.eq	el0_ia
M
Mark Rutland 已提交
530
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
531
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
532
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
C
Catalin Marinas 已提交
533
	b.eq	el0_fpsimd_exc
M
Mark Rutland 已提交
534
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
535
	b.eq	el0_sys
M
Mark Rutland 已提交
536
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
C
Catalin Marinas 已提交
537
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
538
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
C
Catalin Marinas 已提交
539
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
540
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
541
	b.eq	el0_undef
M
Mark Rutland 已提交
542
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
543 544 545 546 547 548 549 550
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
M
Mark Rutland 已提交
551 552
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
C
Catalin Marinas 已提交
553
	b.eq	el0_svc_compat
M
Mark Rutland 已提交
554
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
C
Catalin Marinas 已提交
555
	b.eq	el0_da
M
Mark Rutland 已提交
556
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
C
Catalin Marinas 已提交
557
	b.eq	el0_ia
M
Mark Rutland 已提交
558
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
C
Catalin Marinas 已提交
559
	b.eq	el0_fpsimd_acc
M
Mark Rutland 已提交
560
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
C
Catalin Marinas 已提交
561
	b.eq	el0_fpsimd_exc
562 563
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
	b.eq	el0_sp_pc
M
Mark Rutland 已提交
564
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
C
Catalin Marinas 已提交
565
	b.eq	el0_undef
M
Mark Rutland 已提交
566
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
567
	b.eq	el0_undef
M
Mark Rutland 已提交
568
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
569
	b.eq	el0_undef
M
Mark Rutland 已提交
570
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
571
	b.eq	el0_undef
M
Mark Rutland 已提交
572
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
573
	b.eq	el0_undef
M
Mark Rutland 已提交
574
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
575
	b.eq	el0_undef
M
Mark Rutland 已提交
576
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
C
Catalin Marinas 已提交
577 578 579 580 581 582
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
583
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
C
Catalin Marinas 已提交
584 585 586 587 588 589 590 591 592 593 594 595 596 597
	uxtw	scno, w7			// syscall number in w7 (r7)
	mov     sc_nr, #__NR_compat_syscalls
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
598
	mrs	x26, far_el1
C
Catalin Marinas 已提交
599
	// enable interrupts before calling the main handler
600
	enable_dbg_and_irq
L
Larry Bassel 已提交
601
	ct_user_exit
602
	clear_address_tag x0, x26
C
Catalin Marinas 已提交
603 604
	mov	x1, x25
	mov	x2, sp
605 606
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
607 608 609 610
el0_ia:
	/*
	 * Instruction abort handling
	 */
611
	mrs	x26, far_el1
C
Catalin Marinas 已提交
612
	// enable interrupts before calling the main handler
613
	enable_dbg_and_irq
L
Larry Bassel 已提交
614
	ct_user_exit
615
	mov	x0, x26
M
Mark Rutland 已提交
616
	mov	x1, x25
C
Catalin Marinas 已提交
617
	mov	x2, sp
618 619
	bl	do_mem_abort
	b	ret_to_user
C
Catalin Marinas 已提交
620 621 622 623
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
624
	enable_dbg
L
Larry Bassel 已提交
625
	ct_user_exit
C
Catalin Marinas 已提交
626 627
	mov	x0, x25
	mov	x1, sp
628 629
	bl	do_fpsimd_acc
	b	ret_to_user
C
Catalin Marinas 已提交
630 631 632 633
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
634
	enable_dbg
L
Larry Bassel 已提交
635
	ct_user_exit
C
Catalin Marinas 已提交
636 637
	mov	x0, x25
	mov	x1, sp
638 639
	bl	do_fpsimd_exc
	b	ret_to_user
C
Catalin Marinas 已提交
640 641 642 643
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
644
	mrs	x26, far_el1
C
Catalin Marinas 已提交
645
	// enable interrupts before calling the main handler
646
	enable_dbg_and_irq
647
	ct_user_exit
648
	mov	x0, x26
C
Catalin Marinas 已提交
649 650
	mov	x1, x25
	mov	x2, sp
651 652
	bl	do_sp_pc_abort
	b	ret_to_user
C
Catalin Marinas 已提交
653 654 655 656
el0_undef:
	/*
	 * Undefined instruction
	 */
657
	// enable interrupts before calling the main handler
658
	enable_dbg_and_irq
L
Larry Bassel 已提交
659
	ct_user_exit
660
	mov	x0, sp
661 662
	bl	do_undefinstr
	b	ret_to_user
663 664 665 666 667 668 669 670 671 672
el0_sys:
	/*
	 * System instructions, for trapped cache maintenance instructions
	 */
	enable_dbg_and_irq
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sysinstr
	b	ret_to_user
C
Catalin Marinas 已提交
673 674 675 676 677 678 679 680
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
681 682
	bl	do_debug_exception
	enable_dbg
L
Larry Bassel 已提交
683
	ct_user_exit
684
	b	ret_to_user
C
Catalin Marinas 已提交
685
el0_inv:
686
	enable_dbg
L
Larry Bassel 已提交
687
	ct_user_exit
C
Catalin Marinas 已提交
688 689
	mov	x0, sp
	mov	x1, #BAD_SYNC
690
	mov	x2, x25
691
	bl	bad_el0_sync
692
	b	ret_to_user
C
Catalin Marinas 已提交
693 694 695 696 697 698 699 700 701 702
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
703

L
Larry Bassel 已提交
704
	ct_user_exit
C
Catalin Marinas 已提交
705
	irq_handler
706

C
Catalin Marinas 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
722 723
	mov	x10, #THREAD_CPU_CONTEXT
	add	x8, x0, x10
C
Catalin Marinas 已提交
724 725 726 727 728 729 730 731
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
732
	add	x8, x1, x10
C
Catalin Marinas 已提交
733 734 735 736 737 738 739 740
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
741
	msr	sp_el0, x1
C
Catalin Marinas 已提交
742 743 744 745 746 747 748 749 750
	ret
ENDPROC(cpu_switch_to)

/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
	disable_irq				// disable interrupts
751
	str	x0, [sp, #S_X0]			// returned x0
752
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
753 754
	and	x2, x1, #_TIF_SYSCALL_WORK
	cbnz	x2, ret_fast_syscall_trace
C
Catalin Marinas 已提交
755
	and	x2, x1, #_TIF_WORK_MASK
756
	cbnz	x2, work_pending
757
	enable_step_tsk x1, x2
758
	kernel_exit 0
759 760
ret_fast_syscall_trace:
	enable_irq				// enable interrupts
761
	b	__sys_trace_return_skipped	// we already saved x0
C
Catalin Marinas 已提交
762 763 764 765 766 767 768

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
work_pending:
	mov	x0, sp				// 'regs'
	bl	do_notify_resume
769
#ifdef CONFIG_TRACE_IRQFLAGS
770
	bl	trace_hardirqs_on		// enabled while in userspace
771
#endif
772
	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
773
	b	finish_ret_to_user
C
Catalin Marinas 已提交
774 775 776
/*
 * "slow" syscall return path.
 */
777
ret_to_user:
C
Catalin Marinas 已提交
778
	disable_irq				// disable interrupts
779
	ldr	x1, [tsk, #TSK_TI_FLAGS]
C
Catalin Marinas 已提交
780 781
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
782
finish_ret_to_user:
783
	enable_step_tsk x1, x2
784
	kernel_exit 0
C
Catalin Marinas 已提交
785 786 787 788 789 790 791
ENDPROC(ret_to_user)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
792 793 794 795
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
C
Catalin Marinas 已提交
796 797 798 799 800 801 802 803 804 805 806 807 808
	b	ret_to_user
ENDPROC(ret_from_fork)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
	uxtw	scno, w8			// syscall number in w8
	mov	sc_nr, #__NR_syscalls
el0_svc_naked:					// compat entry point
	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
809
	enable_dbg_and_irq
L
Larry Bassel 已提交
810
	ct_user_exit 1
C
Catalin Marinas 已提交
811

812
	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
813 814
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
C
Catalin Marinas 已提交
815 816 817
	cmp     scno, sc_nr                     // check upper syscall limit
	b.hs	ni_sys
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
818 819
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
C
Catalin Marinas 已提交
820 821
ni_sys:
	mov	x0, sp
822 823
	bl	do_ni_syscall
	b	ret_fast_syscall
C
Catalin Marinas 已提交
824 825 826 827 828 829 830
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
831 832 833 834 835 836
	mov	w0, #-1				// set default errno for
	cmp     scno, x0			// user-issued syscall(-1)
	b.ne	1f
	mov	x0, #-ENOSYS
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
837
	bl	syscall_trace_enter
838 839
	cmp	w0, #-1				// skip the syscall?
	b.eq	__sys_trace_return_skipped
C
Catalin Marinas 已提交
840 841 842
	uxtw	scno, w0			// syscall number (possibly new)
	mov	x1, sp				// pointer to regs
	cmp	scno, sc_nr			// check upper syscall limit
843
	b.hs	__ni_sys_trace
C
Catalin Marinas 已提交
844 845 846 847 848
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
849
	blr	x16				// call sys_* routine
C
Catalin Marinas 已提交
850 851

__sys_trace_return:
852 853
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
854 855
	mov	x0, sp
	bl	syscall_trace_exit
C
Catalin Marinas 已提交
856 857
	b	ret_to_user

858 859 860 861 862
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

863 864
	.popsection				// .entry.text

C
Catalin Marinas 已提交
865 866 867 868 869 870 871
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)