entry_64.S 38.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 *
12 13
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
L
Linus Torvalds 已提交
14 15
 * NOTE: This code handles signal-recognition, which happens every time
 * after an interrupt and after each system call.
16 17
 *
 * A note on terminology:
18
 * - iret frame: Architecture defined interrupt frame from SS to RIP
19
 * at the top of the kernel process stack.
20 21 22 23
 *
 * Some macro usage:
 * - ENTRY/END Define functions in the symbol table.
 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
24
 * - idtentry - Define exception entry points.
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
31
#include "calling.h"
32
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
33 34 35 36
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
37
#include <asm/page_types.h>
38
#include <asm/irqflags.h>
39
#include <asm/paravirt.h>
40
#include <asm/percpu.h>
41
#include <asm/asm.h>
42
#include <asm/context_tracking.h>
43
#include <asm/smap.h>
44
#include <asm/pgtable_types.h>
45
#include <linux/err.h>
L
Linus Torvalds 已提交
46

R
Roland McGrath 已提交
47 48 49 50 51 52
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
#define AUDIT_ARCH_X86_64	(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT 0x80000000
#define __AUDIT_ARCH_LE	   0x40000000

L
Linus Torvalds 已提交
53
	.code64
J
Jiri Olsa 已提交
54 55
	.section .entry.text, "ax"

56

57
#ifdef CONFIG_PARAVIRT
58
ENTRY(native_usergs_sysret64)
59 60
	swapgs
	sysretq
61
ENDPROC(native_usergs_sysret64)
62 63
#endif /* CONFIG_PARAVIRT */

64

65
.macro TRACE_IRQS_IRETQ
66
#ifdef CONFIG_TRACE_IRQFLAGS
67
	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
68 69 70 71 72 73
	jnc  1f
	TRACE_IRQS_ON
1:
#endif
.endm

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
	call debug_stack_set_zero
	TRACE_IRQS_OFF
	call debug_stack_reset
.endm

.macro TRACE_IRQS_ON_DEBUG
	call debug_stack_set_zero
	TRACE_IRQS_ON
	call debug_stack_reset
.endm

99 100
.macro TRACE_IRQS_IRETQ_DEBUG
	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
101 102 103 104 105 106 107 108 109 110 111
	jnc  1f
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
# define TRACE_IRQS_OFF_DEBUG		TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG		TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG		TRACE_IRQS_IRETQ
#endif

L
Linus Torvalds 已提交
112
/*
113
 * 64bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
114
 *
115 116 117 118 119 120 121
 * 64bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
122
 * rax  system call number
123 124
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
125 126
 * rdi  arg0
 * rsi  arg1
127
 * rdx  arg2
128
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
129 130
 * r8   arg4
 * r9   arg5
131
 * (note: r12-r15,rbp,rbx are callee-preserved in C ABI)
132
 *
L
Linus Torvalds 已提交
133 134
 * Only called from user space.
 *
135
 * When user can change pt_regs->foo always force IRET. That is because
136 137
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
138
 */
L
Linus Torvalds 已提交
139 140

ENTRY(system_call)
141 142 143 144 145
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
146 147 148 149 150 151
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
152
GLOBAL(system_call_after_swapgs)
153

154
	movq	%rsp,PER_CPU_VAR(rsp_scratch)
155
	movq	PER_CPU_VAR(cpu_current_top_of_stack),%rsp
156 157

	/* Construct struct pt_regs on stack */
158 159
	pushq $__USER_DS			/* pt_regs->ss */
	pushq PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
160
	/*
161 162 163 164 165
	 * Re-enable interrupts.
	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
	 * must execute atomically in the face of possible interrupt-driven
	 * task preemption. We must enable interrupts only after we're done
	 * with using rsp_scratch:
166 167
	 */
	ENABLE_INTERRUPTS(CLBR_NONE)
168 169 170 171 172 173 174 175 176 177 178 179 180
	pushq	%r11			/* pt_regs->flags */
	pushq	$__USER_CS		/* pt_regs->cs */
	pushq	%rcx			/* pt_regs->ip */
	pushq	%rax			/* pt_regs->orig_ax */
	pushq	%rdi			/* pt_regs->di */
	pushq	%rsi			/* pt_regs->si */
	pushq	%rdx			/* pt_regs->dx */
	pushq	%rcx			/* pt_regs->cx */
	pushq	$-ENOSYS		/* pt_regs->ax */
	pushq	%r8			/* pt_regs->r8 */
	pushq	%r9			/* pt_regs->r9 */
	pushq	%r10			/* pt_regs->r10 */
	pushq	%r11			/* pt_regs->r11 */
181
	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
182

183
	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
L
Linus Torvalds 已提交
184
	jnz tracesys
R
Roland McGrath 已提交
185
system_call_fastpath:
186
#if __SYSCALL_MASK == ~0
L
Linus Torvalds 已提交
187
	cmpq $__NR_syscall_max,%rax
188 189 190 191
#else
	andl $__SYSCALL_MASK,%eax
	cmpl $__NR_syscall_max,%eax
#endif
192
	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
L
Linus Torvalds 已提交
193
	movq %r10,%rcx
194
	call *sys_call_table(,%rax,8)
195
	movq %rax,RAX(%rsp)
196
1:
L
Linus Torvalds 已提交
197
/*
198 199
 * Syscall return path ending with SYSRET (fast path).
 * Has incompletely filled pt_regs.
200
 */
201
	LOCKDEP_SYS_EXIT
202 203 204 205
	/*
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
206
	DISABLE_INTERRUPTS(CLBR_NONE)
207 208 209 210 211 212 213 214 215

	/*
	 * We must check ti flags with interrupts (or at least preemption)
	 * off because we must *never* return to userspace without
	 * processing exit work that is enqueued if we're preempted here.
	 * In particular, returning to userspace with any of the one-shot
	 * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
	 * very bad.
	 */
216 217
	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz int_ret_from_sys_call_irqs_off	/* Go to the slow path */
218

219 220 221
	RESTORE_C_REGS_EXCEPT_RCX_R11
	movq	RIP(%rsp),%rcx
	movq	EFLAGS(%rsp),%r11
222
	movq	RSP(%rsp),%rsp
223 224 225 226
	/*
	 * 64bit SYSRET restores rip from rcx,
	 * rflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
227
	 * Restoration of rflags re-enables interrupts.
228 229 230 231 232 233 234 235 236
	 *
	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
	 * descriptor is not reinitialized.  This means that we should
	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
	 * exit the kernel, and re-enter using an interrupt vector.  (All
	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
	 * from happening by reloading SS in __switch_to.  (Actually
	 * detecting the failure in 64-bit userspace is tricky but can be
	 * done.)
237
	 */
238
	USERGS_SYSRET64
L
Linus Torvalds 已提交
239

240
	/* Do syscall entry tracing */
241
tracesys:
242
	movq %rsp, %rdi
243
	movl $AUDIT_ARCH_X86_64, %esi
244 245 246
	call syscall_trace_enter_phase1
	test %rax, %rax
	jnz tracesys_phase2		/* if needed, run the slow path */
247
	RESTORE_C_REGS_EXCEPT_RAX	/* else restore clobbered regs */
248
	movq ORIG_RAX(%rsp), %rax
249 250 251
	jmp system_call_fastpath	/*      and return to the fast path */

tracesys_phase2:
252
	SAVE_EXTRA_REGS
253
	movq %rsp, %rdi
254
	movl $AUDIT_ARCH_X86_64, %esi
255 256 257
	movq %rax,%rdx
	call syscall_trace_enter_phase2

258
	/*
D
Denys Vlasenko 已提交
259
	 * Reload registers from stack in case ptrace changed them.
260
	 * We don't reload %rax because syscall_trace_entry_phase2() returned
261 262
	 * the value it wants us to use in the table lookup.
	 */
263 264
	RESTORE_C_REGS_EXCEPT_RAX
	RESTORE_EXTRA_REGS
265
#if __SYSCALL_MASK == ~0
L
Linus Torvalds 已提交
266
	cmpq $__NR_syscall_max,%rax
267 268 269 270
#else
	andl $__SYSCALL_MASK,%eax
	cmpl $__NR_syscall_max,%eax
#endif
271
	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
L
Linus Torvalds 已提交
272 273
	movq %r10,%rcx	/* fixup for C */
	call *sys_call_table(,%rax,8)
274
	movq %rax,RAX(%rsp)
275
1:
276
	/* Use IRET because user could have changed pt_regs->foo */
277 278

/*
L
Linus Torvalds 已提交
279
 * Syscall return path ending with IRET.
280
 * Has correct iret frame.
281
 */
282
GLOBAL(int_ret_from_sys_call)
283
	DISABLE_INTERRUPTS(CLBR_NONE)
284
int_ret_from_sys_call_irqs_off: /* jumps come here from the irqs-off SYSRET path */
285
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
286 287
	movl $_TIF_ALLWORK_MASK,%edi
	/* edi:	mask to check */
288
GLOBAL(int_with_check)
289
	LOCKDEP_SYS_EXIT_IRQ
L
Linus Torvalds 已提交
290
	GET_THREAD_INFO(%rcx)
G
Glauber Costa 已提交
291
	movl TI_flags(%rcx),%edx
L
Linus Torvalds 已提交
292 293
	andl %edi,%edx
	jnz   int_careful
294 295
	andl	$~TS_COMPAT,TI_status(%rcx)
	jmp	syscall_return
L
Linus Torvalds 已提交
296 297 298 299 300 301 302

	/* Either reschedule or signal or syscall exit tracking needed. */
	/* First do a reschedule test. */
	/* edx:	work, edi: workmask */
int_careful:
	bt $TIF_NEED_RESCHED,%edx
	jnc  int_very_careful
303
	TRACE_IRQS_ON
304
	ENABLE_INTERRUPTS(CLBR_NONE)
305
	pushq %rdi
306
	SCHEDULE_USER
307
	popq %rdi
308
	DISABLE_INTERRUPTS(CLBR_NONE)
309
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
310 311
	jmp int_with_check

312
	/* handle signals and tracing -- both require a full pt_regs */
L
Linus Torvalds 已提交
313
int_very_careful:
314
	TRACE_IRQS_ON
315
	ENABLE_INTERRUPTS(CLBR_NONE)
316
	SAVE_EXTRA_REGS
317
	/* Check for syscall exit trace */
318
	testl $_TIF_WORK_SYSCALL_EXIT,%edx
L
Linus Torvalds 已提交
319
	jz int_signal
320
	pushq %rdi
321
	leaq 8(%rsp),%rdi	# &ptregs -> arg1
L
Linus Torvalds 已提交
322
	call syscall_trace_leave
323
	popq %rdi
324
	andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
L
Linus Torvalds 已提交
325
	jmp int_restore_rest
326

L
Linus Torvalds 已提交
327
int_signal:
P
Peter Zijlstra 已提交
328
	testl $_TIF_DO_NOTIFY_MASK,%edx
L
Linus Torvalds 已提交
329 330 331 332
	jz 1f
	movq %rsp,%rdi		# &ptregs -> arg1
	xorl %esi,%esi		# oldset -> arg2
	call do_notify_resume
R
Roland McGrath 已提交
333
1:	movl $_TIF_WORK_MASK,%edi
L
Linus Torvalds 已提交
334
int_restore_rest:
335
	RESTORE_EXTRA_REGS
336
	DISABLE_INTERRUPTS(CLBR_NONE)
337
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
338
	jmp int_with_check
339 340 341 342 343 344 345 346 347 348 349

syscall_return:
	/* The IRETQ could re-enable interrupts: */
	DISABLE_INTERRUPTS(CLBR_ANY)
	TRACE_IRQS_IRETQ

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
	movq RCX(%rsp),%rcx
350 351
	movq RIP(%rsp),%r11
	cmpq %rcx,%r11			/* RCX == RIP */
352 353 354 355 356
	jne opportunistic_sysret_failed

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
357
	 * the kernel, since userspace controls RSP.
358
	 *
359
	 * If width of "canonical tail" ever becomes variable, this will need
360 361 362 363 364
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
365 366 367 368 369 370
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405

	cmpq $__USER_CS,CS(%rsp)	/* CS must match SYSRET */
	jne opportunistic_sysret_failed

	movq R11(%rsp),%r11
	cmpq %r11,EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne opportunistic_sysret_failed

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
	 *           movq $stuck_here,%rcx
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
	testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz opportunistic_sysret_failed

	/* nothing to check for RSP */

	cmpq $__USER_DS,SS(%rsp)	/* SS must match SYSRET */
	jne opportunistic_sysret_failed

	/*
	 * We win!  This label is here just for ease of understanding
	 * perf profiles.  Nothing jumps here.
	 */
syscall_return_via_sysret:
406 407
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
408 409 410 411 412 413
	movq RSP(%rsp),%rsp
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
414
END(system_call)
415

416

417 418
	.macro FORK_LIKE func
ENTRY(stub_\func)
419
	SAVE_EXTRA_REGS 8
420
	jmp sys_\func
421 422 423 424 425 426
END(stub_\func)
	.endm

	FORK_LIKE  clone
	FORK_LIKE  fork
	FORK_LIKE  vfork
L
Linus Torvalds 已提交
427 428

ENTRY(stub_execve)
429 430 431 432 433 434 435 436 437 438 439 440
	call	sys_execve
return_from_execve:
	testl	%eax, %eax
	jz	1f
	/* exec failed, can use fast SYSRET code path in this case */
	ret
1:
	/* must use IRET code path (pt_regs->cs may have changed) */
	addq	$8, %rsp
	ZERO_EXTRA_REGS
	movq	%rax,RAX(%rsp)
	jmp	int_ret_from_sys_call
441
END(stub_execve)
442 443 444 445 446 447
/*
 * Remaining execve stubs are only 7 bytes long.
 * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
 */
	.align	8
GLOBAL(stub_execveat)
448 449
	call	sys_execveat
	jmp	return_from_execve
D
David Drysdale 已提交
450 451
END(stub_execveat)

452
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
453 454
	.align	8
GLOBAL(stub_x32_execve)
455
GLOBAL(stub32_execve)
456 457
	call	compat_sys_execve
	jmp	return_from_execve
458
END(stub32_execve)
459
END(stub_x32_execve)
460 461 462
	.align	8
GLOBAL(stub_x32_execveat)
GLOBAL(stub32_execveat)
463 464 465
	call	compat_sys_execveat
	jmp	return_from_execve
END(stub32_execveat)
466
END(stub_x32_execveat)
467 468
#endif

L
Linus Torvalds 已提交
469 470 471
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
472
 */
L
Linus Torvalds 已提交
473
ENTRY(stub_rt_sigreturn)
474 475 476 477 478 479 480 481
	/*
	 * SAVE_EXTRA_REGS result is not normally needed:
	 * sigreturn overwrites all pt_regs->GPREGS.
	 * But sigreturn can fail (!), and there is no easy way to detect that.
	 * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
	 * we SAVE_EXTRA_REGS here.
	 */
	SAVE_EXTRA_REGS 8
L
Linus Torvalds 已提交
482
	call sys_rt_sigreturn
483 484
return_from_stub:
	addq	$8, %rsp
485
	RESTORE_EXTRA_REGS
486
	movq %rax,RAX(%rsp)
L
Linus Torvalds 已提交
487
	jmp int_ret_from_sys_call
488
END(stub_rt_sigreturn)
L
Linus Torvalds 已提交
489

490 491
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
492
	SAVE_EXTRA_REGS 8
493
	call sys32_x32_rt_sigreturn
494
	jmp  return_from_stub
495 496 497
END(stub_x32_rt_sigreturn)
#endif

498 499 500 501 502 503 504 505 506
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)

	LOCK ; btr $TIF_FORK,TI_flags(%r8)

507 508
	pushq $0x0002
	popfq				# reset kernel eflags
509 510 511 512 513

	call schedule_tail			# rdi: 'prev' task parameter

	RESTORE_EXTRA_REGS

514
	testb	$3, CS(%rsp)			# from kernel_thread?
515

516 517 518 519
	/*
	 * By the time we get here, we have no idea whether our pt_regs,
	 * ti flags, and ti status came from the 64-bit SYSCALL fast path,
	 * the slow path, or one of the ia32entry paths.
520
	 * Use IRET code path to return, since it can safely handle
521 522
	 * all of the above.
	 */
523
	jnz	int_ret_from_sys_call
524

525 526
	/* We came from kernel_thread */
	/* nb: we depend on RESTORE_EXTRA_REGS above */
527 528 529 530 531 532 533
	movq %rbp, %rdi
	call *%rbx
	movl $0, RAX(%rsp)
	RESTORE_EXTRA_REGS
	jmp int_ret_from_sys_call
END(ret_from_fork)

534
/*
535 536
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
537
 */
538
	.align 8
539
ENTRY(irq_entries_start)
540 541
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
542
	pushq $(~vector+0x80)	/* Note: always in signed byte range */
543 544 545 546
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
547 548
END(irq_entries_start)

549
/*
L
Linus Torvalds 已提交
550 551 552
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
553 554 555
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
556

557
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
558
	.macro interrupt func
559
	cld
D
Denys Vlasenko 已提交
560 561 562 563 564 565 566
	/*
	 * Since nothing in interrupt handling code touches r12...r15 members
	 * of "struct pt_regs", and since interrupts can nest, we can save
	 * four stack slots and simultaneously provide
	 * an unwind-friendly stack layout by saving "truncated" pt_regs
	 * exactly up to rbp slot, without these members.
	 */
567 568 569 570 571 572
	ALLOC_PT_GPREGS_ON_STACK -RBP
	SAVE_C_REGS -RBP
	/* this goes to 0(%rsp) for unwinder, not for saving the value: */
	SAVE_EXTRA_REGS_RBP -RBP

	leaq -RBP(%rsp),%rdi	/* arg1 for \func (pointer to pt_regs) */
573

574
	testb	$3, CS-RBP(%rsp)
575
	jz	1f
576
	SWAPGS
577
1:
578
	/*
D
Denys Vlasenko 已提交
579
	 * Save previous stack pointer, optionally switch to interrupt stack.
580 581 582 583 584
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
585 586
	movq %rsp, %rsi
	incl PER_CPU_VAR(irq_count)
587 588 589 590 591
	cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
	pushq %rsi
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

L
Linus Torvalds 已提交
592 593 594
	call \func
	.endm

595 596 597 598
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
599 600
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
601
	ASM_CLAC
602
	addq $-0x80,(%rsp)		/* Adjust vector to [-256,-1] range */
L
Linus Torvalds 已提交
603
	interrupt do_IRQ
604
	/* 0(%rsp): old RSP */
605
ret_from_intr:
606
	DISABLE_INTERRUPTS(CLBR_NONE)
607
	TRACE_IRQS_OFF
608
	decl PER_CPU_VAR(irq_count)
609

610 611
	/* Restore saved previous stack */
	popq %rsi
D
Denys Vlasenko 已提交
612
	/* return code expects complete pt_regs - adjust rsp accordingly: */
613
	leaq -RBP(%rsi),%rsp
614

615
	testb	$3, CS(%rsp)
616
	jz	retint_kernel
L
Linus Torvalds 已提交
617
	/* Interrupt came from user space */
618
retint_user:
619
	GET_THREAD_INFO(%rcx)
L
Linus Torvalds 已提交
620 621
	/*
	 * %rcx: thread info. Interrupts off.
622
	 */
L
Linus Torvalds 已提交
623 624
retint_with_reschedule:
	movl $_TIF_WORK_MASK,%edi
625
retint_check:
626
	LOCKDEP_SYS_EXIT_IRQ
G
Glauber Costa 已提交
627
	movl TI_flags(%rcx),%edx
L
Linus Torvalds 已提交
628 629
	andl %edi,%edx
	jnz  retint_careful
630 631

retint_swapgs:		/* return to user-space */
632 633 634
	/*
	 * The iretq could re-enable interrupts:
	 */
635
	DISABLE_INTERRUPTS(CLBR_ANY)
636
	TRACE_IRQS_IRETQ
637

638
	SWAPGS
639
	jmp	restore_c_regs_and_iret
640

641
/* Returning to kernel space */
642
retint_kernel:
643 644 645 646
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
647
	jnc	1f
648 649
0:	cmpl	$0,PER_CPU_VAR(__preempt_count)
	jnz	1f
650
	call	preempt_schedule_irq
651
	jmp	0b
652
1:
653
#endif
654 655 656 657
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
658 659 660 661 662 663

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
restore_c_regs_and_iret:
664 665
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
I
Ingo Molnar 已提交
666

A
Adrian Bunk 已提交
667
irq_return:
668 669 670
	INTERRUPT_RETURN

ENTRY(native_iret)
671 672 673 674
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
675
#ifdef CONFIG_X86_ESPFIX64
676
	testb $4,(SS-RIP)(%rsp)
677
	jnz native_irq_return_ldt
678
#endif
679

680
.global native_irq_return_iret
681
native_irq_return_iret:
A
Andy Lutomirski 已提交
682 683 684 685 686 687
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
688
	iretq
I
Ingo Molnar 已提交
689

690
#ifdef CONFIG_X86_ESPFIX64
691
native_irq_return_ldt:
692 693
	pushq %rax
	pushq %rdi
694 695 696 697 698 699 700 701 702 703 704 705 706 707
	SWAPGS
	movq PER_CPU_VAR(espfix_waddr),%rdi
	movq %rax,(0*8)(%rdi)	/* RAX */
	movq (2*8)(%rsp),%rax	/* RIP */
	movq %rax,(1*8)(%rdi)
	movq (3*8)(%rsp),%rax	/* CS */
	movq %rax,(2*8)(%rdi)
	movq (4*8)(%rsp),%rax	/* RFLAGS */
	movq %rax,(3*8)(%rdi)
	movq (6*8)(%rsp),%rax	/* SS */
	movq %rax,(5*8)(%rdi)
	movq (5*8)(%rsp),%rax	/* RSP */
	movq %rax,(4*8)(%rdi)
	andl $0xffff0000,%eax
708
	popq %rdi
709 710 711
	orq PER_CPU_VAR(espfix_stack),%rax
	SWAPGS
	movq %rax,%rsp
712
	popq %rax
713
	jmp native_irq_return_iret
714
#endif
715

716
	/* edi: workmask, edx: work */
L
Linus Torvalds 已提交
717 718 719
retint_careful:
	bt    $TIF_NEED_RESCHED,%edx
	jnc   retint_signal
720
	TRACE_IRQS_ON
721
	ENABLE_INTERRUPTS(CLBR_NONE)
722
	pushq %rdi
723
	SCHEDULE_USER
724
	popq %rdi
L
Linus Torvalds 已提交
725
	GET_THREAD_INFO(%rcx)
726
	DISABLE_INTERRUPTS(CLBR_NONE)
727
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
728
	jmp retint_check
729

L
Linus Torvalds 已提交
730
retint_signal:
P
Peter Zijlstra 已提交
731
	testl $_TIF_DO_NOTIFY_MASK,%edx
732
	jz    retint_swapgs
733
	TRACE_IRQS_ON
734
	ENABLE_INTERRUPTS(CLBR_NONE)
735
	SAVE_EXTRA_REGS
736
	movq $-1,ORIG_RAX(%rsp)
737
	xorl %esi,%esi		# oldset
L
Linus Torvalds 已提交
738 739
	movq %rsp,%rdi		# &pt_regs
	call do_notify_resume
740
	RESTORE_EXTRA_REGS
741
	DISABLE_INTERRUPTS(CLBR_NONE)
742
	TRACE_IRQS_OFF
743
	GET_THREAD_INFO(%rcx)
R
Roland McGrath 已提交
744
	jmp retint_with_reschedule
L
Linus Torvalds 已提交
745

746
END(common_interrupt)
747

L
Linus Torvalds 已提交
748 749
/*
 * APIC interrupts.
750
 */
751
.macro apicinterrupt3 num sym do_sym
752
ENTRY(\sym)
753
	ASM_CLAC
754
	pushq $~(\num)
755
.Lcommon_\sym:
756
	interrupt \do_sym
L
Linus Torvalds 已提交
757
	jmp ret_from_intr
758 759
END(\sym)
.endm
L
Linus Torvalds 已提交
760

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

778
#ifdef CONFIG_SMP
779
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR \
780
	irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
781
apicinterrupt3 REBOOT_VECTOR \
782
	reboot_interrupt smp_reboot_interrupt
783
#endif
L
Linus Torvalds 已提交
784

N
Nick Piggin 已提交
785
#ifdef CONFIG_X86_UV
786
apicinterrupt3 UV_BAU_MESSAGE \
787
	uv_bau_message_intr1 uv_bau_message_interrupt
N
Nick Piggin 已提交
788
#endif
789 790
apicinterrupt LOCAL_TIMER_VECTOR \
	apic_timer_interrupt smp_apic_timer_interrupt
791 792
apicinterrupt X86_PLATFORM_IPI_VECTOR \
	x86_platform_ipi smp_x86_platform_ipi
793

794
#ifdef CONFIG_HAVE_KVM
795
apicinterrupt3 POSTED_INTR_VECTOR \
796 797 798
	kvm_posted_intr_ipi smp_kvm_posted_intr_ipi
#endif

799
#ifdef CONFIG_X86_MCE_THRESHOLD
800
apicinterrupt THRESHOLD_APIC_VECTOR \
801
	threshold_interrupt smp_threshold_interrupt
802 803 804
#endif

#ifdef CONFIG_X86_THERMAL_VECTOR
805 806
apicinterrupt THERMAL_APIC_VECTOR \
	thermal_interrupt smp_thermal_interrupt
807
#endif
808

809 810 811 812 813 814 815 816
#ifdef CONFIG_SMP
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \
	call_function_single_interrupt smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR \
	call_function_interrupt smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR \
	reschedule_interrupt smp_reschedule_interrupt
#endif
L
Linus Torvalds 已提交
817

818 819 820 821
apicinterrupt ERROR_APIC_VECTOR \
	error_interrupt smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR \
	spurious_interrupt smp_spurious_interrupt
822

823 824 825
#ifdef CONFIG_IRQ_WORK
apicinterrupt IRQ_WORK_VECTOR \
	irq_work_interrupt smp_irq_work_interrupt
I
Ingo Molnar 已提交
826 827
#endif

L
Linus Torvalds 已提交
828 829
/*
 * Exception entry points.
830
 */
831
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
832 833

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
834
ENTRY(\sym)
835 836 837 838 839
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

840
	ASM_CLAC
841
	PARAVIRT_ADJUST_EXCEPTION_FRAME
842 843

	.ifeq \has_error_code
844
	pushq $-1			/* ORIG_RAX: no syscall to restart */
845 846
	.endif

847
	ALLOC_PT_GPREGS_ON_STACK
848 849

	.if \paranoid
850
	.if \paranoid == 1
851
	testb	$3, CS(%rsp)		/* If coming from userspace, switch */
852 853
	jnz 1f				/* stacks. */
	.endif
854
	call paranoid_entry
855 856 857
	.else
	call error_entry
	.endif
858
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
859 860

	.if \paranoid
861 862 863
	.if \shift_ist != -1
	TRACE_IRQS_OFF_DEBUG		/* reload IDT in case of recursion */
	.else
864
	TRACE_IRQS_OFF
865
	.endif
866
	.endif
867 868 869 870 871 872 873 874 875 876

	movq %rsp,%rdi			/* pt_regs pointer */

	.if \has_error_code
	movq ORIG_RAX(%rsp),%rsi	/* get error code */
	movq $-1,ORIG_RAX(%rsp)		/* no syscall to restart */
	.else
	xorl %esi,%esi			/* no error code */
	.endif

877
	.if \shift_ist != -1
878
	subq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
879 880
	.endif

881
	call \do_sym
882

883
	.if \shift_ist != -1
884
	addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
885 886
	.endif

887
	/* these procedures expect "no swapgs" flag in ebx */
888
	.if \paranoid
889
	jmp paranoid_exit
890
	.else
891
	jmp error_exit
892 893
	.endif

894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
	call error_entry


	movq %rsp,%rdi			/* pt_regs pointer */
	call sync_regs
	movq %rax,%rsp			/* switch stack */

	movq %rsp,%rdi			/* pt_regs pointer */

	.if \has_error_code
	movq ORIG_RAX(%rsp),%rsi	/* get error code */
	movq $-1,ORIG_RAX(%rsp)		/* no syscall to restart */
	.else
	xorl %esi,%esi			/* no error code */
	.endif

	call \do_sym

	jmp error_exit			/* %ebx: no swapgs flag */
	.endif
921
END(\sym)
922
.endm
923

924
#ifdef CONFIG_TRACING
925 926 927
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
928 929
.endm
#else
930 931
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
932 933 934
.endm
#endif

935 936 937 938 939
idtentry divide_error do_divide_error has_error_code=0
idtentry overflow do_overflow has_error_code=0
idtentry bounds do_bounds has_error_code=0
idtentry invalid_op do_invalid_op has_error_code=0
idtentry device_not_available do_device_not_available has_error_code=0
940
idtentry double_fault do_double_fault has_error_code=1 paranoid=2
941 942 943 944 945 946 947
idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
idtentry invalid_TSS do_invalid_TSS has_error_code=1
idtentry segment_not_present do_segment_not_present has_error_code=1
idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0
idtentry coprocessor_error do_coprocessor_error has_error_code=0
idtentry alignment_check do_alignment_check has_error_code=1
idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
948

949

950 951
	/* Reload gs selector with exception handling */
	/* edi:  new selector */
952
ENTRY(native_load_gs_index)
953
	pushfq
954
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
955
	SWAPGS
956
gs_change:
957
	movl %edi,%gs
L
Linus Torvalds 已提交
958
2:	mfence		/* workaround */
959
	SWAPGS
960
	popfq
961
	ret
962
END(native_load_gs_index)
963

964
	_ASM_EXTABLE(gs_change,bad_gs)
965
	.section .fixup,"ax"
L
Linus Torvalds 已提交
966
	/* running with kernelgs */
967
bad_gs:
968
	SWAPGS			/* switch back to user gs */
L
Linus Torvalds 已提交
969
	xorl %eax,%eax
970 971 972
	movl %eax,%gs
	jmp  2b
	.previous
973

974
/* Call softirq on interrupt stack. Interrupts are off. */
975
ENTRY(do_softirq_own_stack)
976
	pushq %rbp
977
	mov  %rsp,%rbp
978
	incl PER_CPU_VAR(irq_count)
979
	cmove PER_CPU_VAR(irq_stack_ptr),%rsp
980
	push  %rbp			# backlink for old unwinder
981
	call __do_softirq
982
	leaveq
983
	decl PER_CPU_VAR(irq_count)
984
	ret
985
END(do_softirq_own_stack)
986

987
#ifdef CONFIG_XEN
988
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
989 990

/*
991 992 993 994 995 996 997 998 999 1000 1001 1002
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
1003
ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
1004 1005 1006 1007
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
1008
	movq %rdi, %rsp            # we don't return, adjust the stack frame
1009
11:	incl PER_CPU_VAR(irq_count)
1010
	movq %rsp,%rbp
1011
	cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
1012 1013 1014
	pushq %rbp			# backlink for old unwinder
	call xen_evtchn_do_upcall
	popq %rsp
1015
	decl PER_CPU_VAR(irq_count)
1016 1017 1018
#ifndef CONFIG_PREEMPT
	call xen_maybe_preempt_hcall
#endif
1019
	jmp  error_exit
1020
END(xen_do_hypervisor_callback)
1021 1022

/*
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1035
ENTRY(xen_failsafe_callback)
1036
	movl %ds,%ecx
1037 1038
	cmpw %cx,0x10(%rsp)
	jne 1f
1039
	movl %es,%ecx
1040 1041
	cmpw %cx,0x18(%rsp)
	jne 1f
1042
	movl %fs,%ecx
1043 1044
	cmpw %cx,0x20(%rsp)
	jne 1f
1045
	movl %gs,%ecx
1046 1047 1048 1049 1050 1051
	cmpw %cx,0x28(%rsp)
	jne 1f
	/* All segments match their saved values => Category 2 (Bad IRET). */
	movq (%rsp),%rcx
	movq 8(%rsp),%r11
	addq $0x30,%rsp
1052 1053 1054
	pushq $0	/* RIP */
	pushq %r11
	pushq %rcx
1055
	jmp general_protection
1056 1057 1058 1059
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
	movq (%rsp),%rcx
	movq 8(%rsp),%r11
	addq $0x30,%rsp
1060
	pushq $-1 /* orig_ax = -1 => not a system call */
1061 1062 1063
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
1064 1065 1066
	jmp error_exit
END(xen_failsafe_callback)

1067
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1068 1069
	xen_hvm_callback_vector xen_evtchn_do_upcall

1070
#endif /* CONFIG_XEN */
1071

1072
#if IS_ENABLED(CONFIG_HYPERV)
1073
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1074 1075 1076
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

1077 1078
idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
1079
idtentry stack_segment do_stack_segment has_error_code=1
1080
#ifdef CONFIG_XEN
1081 1082 1083
idtentry xen_debug do_debug has_error_code=0
idtentry xen_int3 do_int3 has_error_code=0
idtentry xen_stack_segment do_stack_segment has_error_code=1
1084
#endif
1085 1086
idtentry general_protection do_general_protection has_error_code=1
trace_idtentry page_fault do_page_fault has_error_code=1
G
Gleb Natapov 已提交
1087
#ifdef CONFIG_KVM_GUEST
1088
idtentry async_page_fault do_async_page_fault has_error_code=1
G
Gleb Natapov 已提交
1089
#endif
1090
#ifdef CONFIG_X86_MCE
1091
idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vector(%rip)
1092 1093
#endif

1094 1095 1096 1097 1098 1099
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
	movl $1,%ebx
	movl $MSR_GS_BASE,%ecx
	rdmsr
	testl %edx,%edx
	js 1f	/* negative -> in kernel */
	SWAPGS
	xorl %ebx,%ebx
1:	ret
1111
END(paranoid_entry)
1112

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
 */
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
1124 1125
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
1126
	TRACE_IRQS_OFF_DEBUG
1127
	testl %ebx,%ebx				/* swapgs needed? */
1128
	jnz paranoid_exit_no_swapgs
1129
	TRACE_IRQS_IRETQ
1130
	SWAPGS_UNSAFE_STACK
1131 1132
	jmp paranoid_exit_restore
paranoid_exit_no_swapgs:
1133
	TRACE_IRQS_IRETQ_DEBUG
1134
paranoid_exit_restore:
1135 1136 1137
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
1138
	INTERRUPT_RETURN
1139 1140 1141
END(paranoid_exit)

/*
1142 1143
 * Save all registers in pt_regs, and switch gs if needed.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
1144 1145 1146
 */
ENTRY(error_entry)
	cld
1147 1148
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1149
	xorl %ebx,%ebx
1150
	testb	$3, CS+8(%rsp)
1151
	jz	error_kernelspace
1152 1153 1154 1155 1156 1157
error_swapgs:
	SWAPGS
error_sti:
	TRACE_IRQS_OFF
	ret

1158 1159 1160 1161 1162 1163
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1164 1165
error_kernelspace:
	incl %ebx
1166
	leaq native_irq_return_iret(%rip),%rcx
1167
	cmpq %rcx,RIP+8(%rsp)
A
Andy Lutomirski 已提交
1168
	je error_bad_iret
1169 1170 1171
	movl %ecx,%eax	/* zero extend */
	cmpq %rax,RIP+8(%rsp)
	je bstep_iret
1172
	cmpq $gs_change,RIP+8(%rsp)
1173
	je error_swapgs
1174
	jmp error_sti
1175 1176 1177 1178

bstep_iret:
	/* Fix truncated RIP */
	movq %rcx,RIP+8(%rsp)
A
Andy Lutomirski 已提交
1179 1180 1181 1182 1183 1184 1185 1186 1187
	/* fall through */

error_bad_iret:
	SWAPGS
	mov %rsp,%rdi
	call fixup_bad_iret
	mov %rax,%rsp
	decl %ebx	/* Return to usergs */
	jmp error_sti
1188 1189 1190
END(error_entry)


1191
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
1192 1193
ENTRY(error_exit)
	movl %ebx,%eax
1194
	RESTORE_EXTRA_REGS
1195 1196 1197
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
	testl %eax,%eax
1198
	jnz retint_kernel
1199
	jmp retint_user
1200 1201
END(error_exit)

1202
/* Runs on exception stack */
1203 1204
ENTRY(nmi)
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
	 *    o Copy the interrupt frame into a "saved" location on the stack
	 *    o Copy the interrupt frame into a "copy" location on the stack
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
	 *    o Modify the "copy" location to jump to the repeate_nmi
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
	 */

1237
	/* Use %rdx as our temp variable throughout */
1238
	pushq %rdx
1239

1240 1241 1242 1243
	/*
	 * If %cs was not the kernel segment, then the NMI triggered in user
	 * space, which means it is definitely not nested.
	 */
1244
	cmpl $__KERNEL_CS, 16(%rsp)
1245 1246
	jne first_nmi

1247 1248 1249 1250
	/*
	 * Check the special variable on the stack to see if NMIs are
	 * executing.
	 */
1251
	cmpl $1, -8(%rsp)
1252 1253 1254 1255 1256 1257 1258 1259 1260
	je nested_nmi

	/*
	 * Now test if the previous stack was an NMI stack.
	 * We need the double check. We check the NMI stack to satisfy the
	 * race when the first NMI clears the variable before returning.
	 * We check the variable because the first NMI could be in a
	 * breakpoint routine using a breakpoint stack.
	 */
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
	/* Ah, it is within the NMI stack, treat it as nested */

1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
nested_nmi:
	/*
	 * Do nothing if we interrupted the fixup in repeat_nmi.
	 * It's about to repeat the NMI handler, so we are fine
	 * with ignoring this one.
	 */
	movq $repeat_nmi, %rdx
	cmpq 8(%rsp), %rdx
	ja 1f
	movq $end_repeat_nmi, %rdx
	cmpq 8(%rsp), %rdx
	ja nested_nmi_out

1:
	/* Set up the interrupted NMIs stack to jump to repeat_nmi */
1287
	leaq -1*8(%rsp), %rdx
1288
	movq %rdx, %rsp
1289
	leaq -10*8(%rsp), %rdx
1290 1291 1292 1293 1294
	pushq $__KERNEL_DS
	pushq %rdx
	pushfq
	pushq $__KERNEL_CS
	pushq $repeat_nmi
1295 1296

	/* Put stack back */
1297
	addq $(6*8), %rsp
1298 1299

nested_nmi_out:
1300
	popq %rdx
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326

	/* No need to check faults here */
	INTERRUPT_RETURN

first_nmi:
	/*
	 * Because nested NMIs will use the pushed location that we
	 * stored in rdx, we must keep that space available.
	 * Here's what our stack frame will look like:
	 * +-------------------------+
	 * | original SS             |
	 * | original Return RSP     |
	 * | original RFLAGS         |
	 * | original CS             |
	 * | original RIP            |
	 * +-------------------------+
	 * | temp storage for rdx    |
	 * +-------------------------+
	 * | NMI executing variable  |
	 * +-------------------------+
	 * | copied SS               |
	 * | copied Return RSP       |
	 * | copied RFLAGS           |
	 * | copied CS               |
	 * | copied RIP              |
	 * +-------------------------+
1327 1328 1329 1330 1331 1332
	 * | Saved SS                |
	 * | Saved Return RSP        |
	 * | Saved RFLAGS            |
	 * | Saved CS                |
	 * | Saved RIP               |
	 * +-------------------------+
1333 1334 1335
	 * | pt_regs                 |
	 * +-------------------------+
	 *
1336 1337 1338
	 * The saved stack frame is used to fix up the copied stack frame
	 * that a nested NMI may change to make the interrupted NMI iret jump
	 * to the repeat_nmi. The original stack frame and the temp storage
1339 1340
	 * is also used by nested NMIs and can not be trusted on exit.
	 */
1341
	/* Do not pop rdx, nested NMIs will corrupt that part of the stack */
1342 1343
	movq (%rsp), %rdx

1344
	/* Set the NMI executing variable on the stack. */
1345
	pushq $1
1346

1347 1348 1349 1350 1351
	/*
	 * Leave room for the "copied" frame
	 */
	subq $(5*8), %rsp

1352 1353
	/* Copy the stack frame to the Saved frame */
	.rept 5
1354
	pushq 11*8(%rsp)
1355
	.endr
1356

1357 1358
	/* Everything up to here is safe from nested NMIs */

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
	 */
repeat_nmi:
	/*
	 * Update the stack variable to say we are still in NMI (the update
	 * is benign for the non-repeat case, where 1 was pushed just above
	 * to this very stack slot).
	 */
1374
	movq $1, 10*8(%rsp)
1375 1376

	/* Make another copy, this one may be modified by nested NMIs */
1377
	addq $(10*8), %rsp
1378
	.rept 5
1379
	pushq -6*8(%rsp)
1380
	.endr
1381
	subq $(5*8), %rsp
1382
end_repeat_nmi:
1383 1384 1385

	/*
	 * Everything below this point can be preempted by a nested
1386 1387
	 * NMI if the first NMI took an exception and reset our iret stack
	 * so that we repeat another NMI.
1388
	 */
1389
	pushq $-1		/* ORIG_RAX: no syscall to restart */
1390 1391
	ALLOC_PT_GPREGS_ON_STACK

1392
	/*
1393
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1394 1395 1396 1397 1398
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1399
	call paranoid_entry
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411

	/*
	 * Save off the CR2 register. If we take a page fault in the NMI then
	 * it could corrupt the CR2 value. If the NMI preempts a page fault
	 * handler before it was able to read the CR2 register, and then the
	 * NMI itself takes a page fault, the page fault that was preempted
	 * will read the information from the NMI page fault and not the
	 * origin fault. Save it off and restore it if it changes.
	 * Use the r12 callee-saved register.
	 */
	movq %cr2, %r12

1412 1413 1414 1415
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
	movq %rsp,%rdi
	movq $-1,%rsi
	call do_nmi
1416 1417 1418 1419 1420 1421 1422

	/* Did the NMI take a page fault? Restore cr2 if it did */
	movq %cr2, %rcx
	cmpq %rcx, %r12
	je 1f
	movq %r12, %cr2
1:
1423 1424 1425 1426 1427
	testl %ebx,%ebx				/* swapgs needed? */
	jnz nmi_restore
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1428 1429
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1430
	/* Pop the extra iret frame at once */
1431
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1432

1433
	/* Clear the NMI executing stack variable */
1434
	movq $0, 5*8(%rsp)
1435 1436 1437 1438 1439 1440 1441 1442
	jmp irq_return
END(nmi)

ENTRY(ignore_sysret)
	mov $-ENOSYS,%eax
	sysret
END(ignore_sysret)