entry_64.S 38.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 *
12 13
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
L
Linus Torvalds 已提交
14 15
 * NOTE: This code handles signal-recognition, which happens every time
 * after an interrupt and after each system call.
16 17
 *
 * A note on terminology:
18
 * - iret frame: Architecture defined interrupt frame from SS to RIP
19
 * at the top of the kernel process stack.
20 21 22 23
 *
 * Some macro usage:
 * - ENTRY/END Define functions in the symbol table.
 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
24
 * - idtentry - Define exception entry points.
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
31
#include "calling.h"
32
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
33 34 35 36
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
37
#include <asm/page_types.h>
38
#include <asm/irqflags.h>
39
#include <asm/paravirt.h>
40
#include <asm/percpu.h>
41
#include <asm/asm.h>
42
#include <asm/context_tracking.h>
43
#include <asm/smap.h>
44
#include <asm/pgtable_types.h>
45
#include <linux/err.h>
L
Linus Torvalds 已提交
46

R
Roland McGrath 已提交
47 48 49 50 51 52
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
#define AUDIT_ARCH_X86_64	(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT 0x80000000
#define __AUDIT_ARCH_LE	   0x40000000

L
Linus Torvalds 已提交
53
	.code64
J
Jiri Olsa 已提交
54 55
	.section .entry.text, "ax"

56

57
#ifdef CONFIG_PARAVIRT
58
ENTRY(native_usergs_sysret64)
59 60
	swapgs
	sysretq
61
ENDPROC(native_usergs_sysret64)
62 63
#endif /* CONFIG_PARAVIRT */

64

65
.macro TRACE_IRQS_IRETQ
66
#ifdef CONFIG_TRACE_IRQFLAGS
67
	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
68 69 70 71 72 73
	jnc  1f
	TRACE_IRQS_ON
1:
#endif
.endm

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
	call debug_stack_set_zero
	TRACE_IRQS_OFF
	call debug_stack_reset
.endm

.macro TRACE_IRQS_ON_DEBUG
	call debug_stack_set_zero
	TRACE_IRQS_ON
	call debug_stack_reset
.endm

99 100
.macro TRACE_IRQS_IRETQ_DEBUG
	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
101 102 103 104 105 106 107 108 109 110 111
	jnc  1f
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
# define TRACE_IRQS_OFF_DEBUG		TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG		TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG		TRACE_IRQS_IRETQ
#endif

L
Linus Torvalds 已提交
112
/*
113
 * 64bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
114
 *
115 116 117 118 119 120 121
 * 64bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
122
 * rax  system call number
123 124
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
125 126
 * rdi  arg0
 * rsi  arg1
127
 * rdx  arg2
128
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
129 130
 * r8   arg4
 * r9   arg5
131
 * (note: r12-r15,rbp,rbx are callee-preserved in C ABI)
132
 *
L
Linus Torvalds 已提交
133 134
 * Only called from user space.
 *
135
 * When user can change pt_regs->foo always force IRET. That is because
136 137
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
138
 */
L
Linus Torvalds 已提交
139 140

ENTRY(system_call)
141 142 143 144 145
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
146 147 148 149 150 151
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
152
GLOBAL(system_call_after_swapgs)
153

154
	movq	%rsp,PER_CPU_VAR(rsp_scratch)
155
	movq	PER_CPU_VAR(cpu_current_top_of_stack),%rsp
156 157

	/* Construct struct pt_regs on stack */
158 159
	pushq $__USER_DS			/* pt_regs->ss */
	pushq PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
160
	/*
161 162 163 164 165
	 * Re-enable interrupts.
	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
	 * must execute atomically in the face of possible interrupt-driven
	 * task preemption. We must enable interrupts only after we're done
	 * with using rsp_scratch:
166 167
	 */
	ENABLE_INTERRUPTS(CLBR_NONE)
168 169 170 171 172 173 174 175 176 177 178 179 180
	pushq	%r11			/* pt_regs->flags */
	pushq	$__USER_CS		/* pt_regs->cs */
	pushq	%rcx			/* pt_regs->ip */
	pushq	%rax			/* pt_regs->orig_ax */
	pushq	%rdi			/* pt_regs->di */
	pushq	%rsi			/* pt_regs->si */
	pushq	%rdx			/* pt_regs->dx */
	pushq	%rcx			/* pt_regs->cx */
	pushq	$-ENOSYS		/* pt_regs->ax */
	pushq	%r8			/* pt_regs->r8 */
	pushq	%r9			/* pt_regs->r9 */
	pushq	%r10			/* pt_regs->r10 */
	pushq	%r11			/* pt_regs->r11 */
181
	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
182

183
	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
L
Linus Torvalds 已提交
184
	jnz tracesys
R
Roland McGrath 已提交
185
system_call_fastpath:
186
#if __SYSCALL_MASK == ~0
L
Linus Torvalds 已提交
187
	cmpq $__NR_syscall_max,%rax
188 189 190 191
#else
	andl $__SYSCALL_MASK,%eax
	cmpl $__NR_syscall_max,%eax
#endif
192
	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
L
Linus Torvalds 已提交
193
	movq %r10,%rcx
194
	call *sys_call_table(,%rax,8)
195
	movq %rax,RAX(%rsp)
196
1:
L
Linus Torvalds 已提交
197
/*
198 199
 * Syscall return path ending with SYSRET (fast path).
 * Has incompletely filled pt_regs.
200
 */
201
	LOCKDEP_SYS_EXIT
202 203 204 205
	/*
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
206
	DISABLE_INTERRUPTS(CLBR_NONE)
207 208 209 210 211 212 213 214 215

	/*
	 * We must check ti flags with interrupts (or at least preemption)
	 * off because we must *never* return to userspace without
	 * processing exit work that is enqueued if we're preempted here.
	 * In particular, returning to userspace with any of the one-shot
	 * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
	 * very bad.
	 */
216 217
	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz int_ret_from_sys_call_irqs_off	/* Go to the slow path */
218

219 220 221
	RESTORE_C_REGS_EXCEPT_RCX_R11
	movq	RIP(%rsp),%rcx
	movq	EFLAGS(%rsp),%r11
222
	movq	RSP(%rsp),%rsp
223 224 225 226
	/*
	 * 64bit SYSRET restores rip from rcx,
	 * rflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
227
	 * Restoration of rflags re-enables interrupts.
228 229 230 231 232 233 234 235 236
	 *
	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
	 * descriptor is not reinitialized.  This means that we should
	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
	 * exit the kernel, and re-enter using an interrupt vector.  (All
	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
	 * from happening by reloading SS in __switch_to.  (Actually
	 * detecting the failure in 64-bit userspace is tricky but can be
	 * done.)
237
	 */
238
	USERGS_SYSRET64
L
Linus Torvalds 已提交
239

240
	/* Do syscall entry tracing */
241
tracesys:
242
	movq %rsp, %rdi
243
	movl $AUDIT_ARCH_X86_64, %esi
244 245 246
	call syscall_trace_enter_phase1
	test %rax, %rax
	jnz tracesys_phase2		/* if needed, run the slow path */
247
	RESTORE_C_REGS_EXCEPT_RAX	/* else restore clobbered regs */
248
	movq ORIG_RAX(%rsp), %rax
249 250 251
	jmp system_call_fastpath	/*      and return to the fast path */

tracesys_phase2:
252
	SAVE_EXTRA_REGS
253
	movq %rsp, %rdi
254
	movl $AUDIT_ARCH_X86_64, %esi
255 256 257
	movq %rax,%rdx
	call syscall_trace_enter_phase2

258
	/*
D
Denys Vlasenko 已提交
259
	 * Reload registers from stack in case ptrace changed them.
260
	 * We don't reload %rax because syscall_trace_entry_phase2() returned
261 262
	 * the value it wants us to use in the table lookup.
	 */
263 264
	RESTORE_C_REGS_EXCEPT_RAX
	RESTORE_EXTRA_REGS
265
#if __SYSCALL_MASK == ~0
L
Linus Torvalds 已提交
266
	cmpq $__NR_syscall_max,%rax
267 268 269 270
#else
	andl $__SYSCALL_MASK,%eax
	cmpl $__NR_syscall_max,%eax
#endif
271
	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
L
Linus Torvalds 已提交
272 273
	movq %r10,%rcx	/* fixup for C */
	call *sys_call_table(,%rax,8)
274
	movq %rax,RAX(%rsp)
275
1:
276
	/* Use IRET because user could have changed pt_regs->foo */
277 278

/*
L
Linus Torvalds 已提交
279
 * Syscall return path ending with IRET.
280
 * Has correct iret frame.
281
 */
282
GLOBAL(int_ret_from_sys_call)
283
	DISABLE_INTERRUPTS(CLBR_NONE)
284
int_ret_from_sys_call_irqs_off: /* jumps come here from the irqs-off SYSRET path */
285
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
286 287
	movl $_TIF_ALLWORK_MASK,%edi
	/* edi:	mask to check */
288
GLOBAL(int_with_check)
289
	LOCKDEP_SYS_EXIT_IRQ
L
Linus Torvalds 已提交
290
	GET_THREAD_INFO(%rcx)
G
Glauber Costa 已提交
291
	movl TI_flags(%rcx),%edx
L
Linus Torvalds 已提交
292 293
	andl %edi,%edx
	jnz   int_careful
294 295
	andl	$~TS_COMPAT,TI_status(%rcx)
	jmp	syscall_return
L
Linus Torvalds 已提交
296 297 298 299 300 301 302

	/* Either reschedule or signal or syscall exit tracking needed. */
	/* First do a reschedule test. */
	/* edx:	work, edi: workmask */
int_careful:
	bt $TIF_NEED_RESCHED,%edx
	jnc  int_very_careful
303
	TRACE_IRQS_ON
304
	ENABLE_INTERRUPTS(CLBR_NONE)
305
	pushq %rdi
306
	SCHEDULE_USER
307
	popq %rdi
308
	DISABLE_INTERRUPTS(CLBR_NONE)
309
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
310 311
	jmp int_with_check

312
	/* handle signals and tracing -- both require a full pt_regs */
L
Linus Torvalds 已提交
313
int_very_careful:
314
	TRACE_IRQS_ON
315
	ENABLE_INTERRUPTS(CLBR_NONE)
316
	SAVE_EXTRA_REGS
317
	/* Check for syscall exit trace */
318
	testl $_TIF_WORK_SYSCALL_EXIT,%edx
L
Linus Torvalds 已提交
319
	jz int_signal
320
	pushq %rdi
321
	leaq 8(%rsp),%rdi	# &ptregs -> arg1
L
Linus Torvalds 已提交
322
	call syscall_trace_leave
323
	popq %rdi
324
	andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
L
Linus Torvalds 已提交
325
	jmp int_restore_rest
326

L
Linus Torvalds 已提交
327
int_signal:
P
Peter Zijlstra 已提交
328
	testl $_TIF_DO_NOTIFY_MASK,%edx
L
Linus Torvalds 已提交
329 330 331 332
	jz 1f
	movq %rsp,%rdi		# &ptregs -> arg1
	xorl %esi,%esi		# oldset -> arg2
	call do_notify_resume
R
Roland McGrath 已提交
333
1:	movl $_TIF_WORK_MASK,%edi
L
Linus Torvalds 已提交
334
int_restore_rest:
335
	RESTORE_EXTRA_REGS
336
	DISABLE_INTERRUPTS(CLBR_NONE)
337
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
338
	jmp int_with_check
339 340 341 342 343 344 345 346 347 348 349

syscall_return:
	/* The IRETQ could re-enable interrupts: */
	DISABLE_INTERRUPTS(CLBR_ANY)
	TRACE_IRQS_IRETQ

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
	movq RCX(%rsp),%rcx
350 351
	movq RIP(%rsp),%r11
	cmpq %rcx,%r11			/* RCX == RIP */
352 353 354 355 356
	jne opportunistic_sysret_failed

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
357
	 * the kernel, since userspace controls RSP.
358
	 *
359
	 * If width of "canonical tail" ever becomes variable, this will need
360 361 362 363 364
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
365 366 367 368 369 370
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405

	cmpq $__USER_CS,CS(%rsp)	/* CS must match SYSRET */
	jne opportunistic_sysret_failed

	movq R11(%rsp),%r11
	cmpq %r11,EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne opportunistic_sysret_failed

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
	 *           movq $stuck_here,%rcx
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
	testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz opportunistic_sysret_failed

	/* nothing to check for RSP */

	cmpq $__USER_DS,SS(%rsp)	/* SS must match SYSRET */
	jne opportunistic_sysret_failed

	/*
	 * We win!  This label is here just for ease of understanding
	 * perf profiles.  Nothing jumps here.
	 */
syscall_return_via_sysret:
406 407
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
408 409 410 411 412 413
	movq RSP(%rsp),%rsp
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
414
END(system_call)
415

416

417 418
	.macro FORK_LIKE func
ENTRY(stub_\func)
419
	SAVE_EXTRA_REGS 8
420
	jmp sys_\func
421 422 423 424 425 426
END(stub_\func)
	.endm

	FORK_LIKE  clone
	FORK_LIKE  fork
	FORK_LIKE  vfork
L
Linus Torvalds 已提交
427 428

ENTRY(stub_execve)
429 430 431 432 433 434 435 436 437 438 439 440
	call	sys_execve
return_from_execve:
	testl	%eax, %eax
	jz	1f
	/* exec failed, can use fast SYSRET code path in this case */
	ret
1:
	/* must use IRET code path (pt_regs->cs may have changed) */
	addq	$8, %rsp
	ZERO_EXTRA_REGS
	movq	%rax,RAX(%rsp)
	jmp	int_ret_from_sys_call
441
END(stub_execve)
442 443 444 445 446 447
/*
 * Remaining execve stubs are only 7 bytes long.
 * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
 */
	.align	8
GLOBAL(stub_execveat)
448 449
	call	sys_execveat
	jmp	return_from_execve
D
David Drysdale 已提交
450 451
END(stub_execveat)

452
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
453 454
	.align	8
GLOBAL(stub_x32_execve)
455
GLOBAL(stub32_execve)
456 457
	call	compat_sys_execve
	jmp	return_from_execve
458
END(stub32_execve)
459
END(stub_x32_execve)
460 461 462
	.align	8
GLOBAL(stub_x32_execveat)
GLOBAL(stub32_execveat)
463 464 465
	call	compat_sys_execveat
	jmp	return_from_execve
END(stub32_execveat)
466
END(stub_x32_execveat)
467 468
#endif

L
Linus Torvalds 已提交
469 470 471
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
472
 */
L
Linus Torvalds 已提交
473
ENTRY(stub_rt_sigreturn)
474 475 476 477 478 479 480 481
	/*
	 * SAVE_EXTRA_REGS result is not normally needed:
	 * sigreturn overwrites all pt_regs->GPREGS.
	 * But sigreturn can fail (!), and there is no easy way to detect that.
	 * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
	 * we SAVE_EXTRA_REGS here.
	 */
	SAVE_EXTRA_REGS 8
L
Linus Torvalds 已提交
482
	call sys_rt_sigreturn
483 484
return_from_stub:
	addq	$8, %rsp
485
	RESTORE_EXTRA_REGS
486
	movq %rax,RAX(%rsp)
L
Linus Torvalds 已提交
487
	jmp int_ret_from_sys_call
488
END(stub_rt_sigreturn)
L
Linus Torvalds 已提交
489

490 491
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
492
	SAVE_EXTRA_REGS 8
493
	call sys32_x32_rt_sigreturn
494
	jmp  return_from_stub
495 496 497
END(stub_x32_rt_sigreturn)
#endif

498 499 500 501 502 503 504 505 506
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)

	LOCK ; btr $TIF_FORK,TI_flags(%r8)

507 508
	pushq $0x0002
	popfq				# reset kernel eflags
509 510 511 512 513

	call schedule_tail			# rdi: 'prev' task parameter

	RESTORE_EXTRA_REGS

514
	testb	$3, CS(%rsp)			# from kernel_thread?
515

516 517 518 519
	/*
	 * By the time we get here, we have no idea whether our pt_regs,
	 * ti flags, and ti status came from the 64-bit SYSCALL fast path,
	 * the slow path, or one of the ia32entry paths.
520
	 * Use IRET code path to return, since it can safely handle
521 522
	 * all of the above.
	 */
523
	jnz	int_ret_from_sys_call
524

525 526
	/* We came from kernel_thread */
	/* nb: we depend on RESTORE_EXTRA_REGS above */
527 528 529 530 531 532 533
	movq %rbp, %rdi
	call *%rbx
	movl $0, RAX(%rsp)
	RESTORE_EXTRA_REGS
	jmp int_ret_from_sys_call
END(ret_from_fork)

534
/*
535 536
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
537
 */
538
	.align 8
539
ENTRY(irq_entries_start)
540 541
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
542
	pushq $(~vector+0x80)	/* Note: always in signed byte range */
543 544 545 546
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
547 548
END(irq_entries_start)

549
/*
L
Linus Torvalds 已提交
550 551 552
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
553 554 555
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
556

557
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
558
	.macro interrupt func
559
	cld
D
Denys Vlasenko 已提交
560 561 562 563 564 565 566
	/*
	 * Since nothing in interrupt handling code touches r12...r15 members
	 * of "struct pt_regs", and since interrupts can nest, we can save
	 * four stack slots and simultaneously provide
	 * an unwind-friendly stack layout by saving "truncated" pt_regs
	 * exactly up to rbp slot, without these members.
	 */
567 568 569 570 571 572
	ALLOC_PT_GPREGS_ON_STACK -RBP
	SAVE_C_REGS -RBP
	/* this goes to 0(%rsp) for unwinder, not for saving the value: */
	SAVE_EXTRA_REGS_RBP -RBP

	leaq -RBP(%rsp),%rdi	/* arg1 for \func (pointer to pt_regs) */
573

574
	testb	$3, CS-RBP(%rsp)
575
	jz	1f
576
	SWAPGS
577
1:
578
	/*
D
Denys Vlasenko 已提交
579
	 * Save previous stack pointer, optionally switch to interrupt stack.
580 581 582 583 584
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
585 586
	movq %rsp, %rsi
	incl PER_CPU_VAR(irq_count)
587 588 589 590 591
	cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
	pushq %rsi
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

L
Linus Torvalds 已提交
592 593 594
	call \func
	.endm

595 596 597 598
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
599 600
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
601
	ASM_CLAC
602
	addq $-0x80,(%rsp)		/* Adjust vector to [-256,-1] range */
L
Linus Torvalds 已提交
603
	interrupt do_IRQ
604
	/* 0(%rsp): old RSP */
605
ret_from_intr:
606
	DISABLE_INTERRUPTS(CLBR_NONE)
607
	TRACE_IRQS_OFF
608
	decl PER_CPU_VAR(irq_count)
609

610 611
	/* Restore saved previous stack */
	popq %rsi
D
Denys Vlasenko 已提交
612
	/* return code expects complete pt_regs - adjust rsp accordingly: */
613
	leaq -RBP(%rsi),%rsp
614

615
	testb	$3, CS(%rsp)
616
	jz	retint_kernel
L
Linus Torvalds 已提交
617
	/* Interrupt came from user space */
618
retint_user:
619
	GET_THREAD_INFO(%rcx)
L
Linus Torvalds 已提交
620 621
	/*
	 * %rcx: thread info. Interrupts off.
622
	 */
L
Linus Torvalds 已提交
623 624
retint_with_reschedule:
	movl $_TIF_WORK_MASK,%edi
625
retint_check:
626
	LOCKDEP_SYS_EXIT_IRQ
G
Glauber Costa 已提交
627
	movl TI_flags(%rcx),%edx
L
Linus Torvalds 已提交
628 629
	andl %edi,%edx
	jnz  retint_careful
630 631

retint_swapgs:		/* return to user-space */
632 633 634
	/*
	 * The iretq could re-enable interrupts:
	 */
635
	DISABLE_INTERRUPTS(CLBR_ANY)
636
	TRACE_IRQS_IRETQ
637

638
	SWAPGS
639
	jmp	restore_c_regs_and_iret
640

641
/* Returning to kernel space */
642
retint_kernel:
643 644 645 646
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
647
	jnc	1f
648 649
0:	cmpl	$0,PER_CPU_VAR(__preempt_count)
	jnz	1f
650
	call	preempt_schedule_irq
651
	jmp	0b
652
1:
653
#endif
654 655 656 657
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
658 659 660 661 662 663

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
restore_c_regs_and_iret:
664 665
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
666 667 668
	INTERRUPT_RETURN

ENTRY(native_iret)
669 670 671 672
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
673
#ifdef CONFIG_X86_ESPFIX64
674
	testb $4,(SS-RIP)(%rsp)
675
	jnz native_irq_return_ldt
676
#endif
677

678
.global native_irq_return_iret
679
native_irq_return_iret:
A
Andy Lutomirski 已提交
680 681 682 683 684 685
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
686
	iretq
I
Ingo Molnar 已提交
687

688
#ifdef CONFIG_X86_ESPFIX64
689
native_irq_return_ldt:
690 691
	pushq %rax
	pushq %rdi
692 693 694 695 696 697 698 699 700 701 702 703 704 705
	SWAPGS
	movq PER_CPU_VAR(espfix_waddr),%rdi
	movq %rax,(0*8)(%rdi)	/* RAX */
	movq (2*8)(%rsp),%rax	/* RIP */
	movq %rax,(1*8)(%rdi)
	movq (3*8)(%rsp),%rax	/* CS */
	movq %rax,(2*8)(%rdi)
	movq (4*8)(%rsp),%rax	/* RFLAGS */
	movq %rax,(3*8)(%rdi)
	movq (6*8)(%rsp),%rax	/* SS */
	movq %rax,(5*8)(%rdi)
	movq (5*8)(%rsp),%rax	/* RSP */
	movq %rax,(4*8)(%rdi)
	andl $0xffff0000,%eax
706
	popq %rdi
707 708 709
	orq PER_CPU_VAR(espfix_stack),%rax
	SWAPGS
	movq %rax,%rsp
710
	popq %rax
711
	jmp native_irq_return_iret
712
#endif
713

714
	/* edi: workmask, edx: work */
L
Linus Torvalds 已提交
715 716 717
retint_careful:
	bt    $TIF_NEED_RESCHED,%edx
	jnc   retint_signal
718
	TRACE_IRQS_ON
719
	ENABLE_INTERRUPTS(CLBR_NONE)
720
	pushq %rdi
721
	SCHEDULE_USER
722
	popq %rdi
L
Linus Torvalds 已提交
723
	GET_THREAD_INFO(%rcx)
724
	DISABLE_INTERRUPTS(CLBR_NONE)
725
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
726
	jmp retint_check
727

L
Linus Torvalds 已提交
728
retint_signal:
P
Peter Zijlstra 已提交
729
	testl $_TIF_DO_NOTIFY_MASK,%edx
730
	jz    retint_swapgs
731
	TRACE_IRQS_ON
732
	ENABLE_INTERRUPTS(CLBR_NONE)
733
	SAVE_EXTRA_REGS
734
	movq $-1,ORIG_RAX(%rsp)
735
	xorl %esi,%esi		# oldset
L
Linus Torvalds 已提交
736 737
	movq %rsp,%rdi		# &pt_regs
	call do_notify_resume
738
	RESTORE_EXTRA_REGS
739
	DISABLE_INTERRUPTS(CLBR_NONE)
740
	TRACE_IRQS_OFF
741
	GET_THREAD_INFO(%rcx)
R
Roland McGrath 已提交
742
	jmp retint_with_reschedule
L
Linus Torvalds 已提交
743

744
END(common_interrupt)
745

L
Linus Torvalds 已提交
746 747
/*
 * APIC interrupts.
748
 */
749
.macro apicinterrupt3 num sym do_sym
750
ENTRY(\sym)
751
	ASM_CLAC
752
	pushq $~(\num)
753
.Lcommon_\sym:
754
	interrupt \do_sym
L
Linus Torvalds 已提交
755
	jmp ret_from_intr
756 757
END(\sym)
.endm
L
Linus Torvalds 已提交
758

759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

776
#ifdef CONFIG_SMP
777
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR \
778
	irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
779
apicinterrupt3 REBOOT_VECTOR \
780
	reboot_interrupt smp_reboot_interrupt
781
#endif
L
Linus Torvalds 已提交
782

N
Nick Piggin 已提交
783
#ifdef CONFIG_X86_UV
784
apicinterrupt3 UV_BAU_MESSAGE \
785
	uv_bau_message_intr1 uv_bau_message_interrupt
N
Nick Piggin 已提交
786
#endif
787 788
apicinterrupt LOCAL_TIMER_VECTOR \
	apic_timer_interrupt smp_apic_timer_interrupt
789 790
apicinterrupt X86_PLATFORM_IPI_VECTOR \
	x86_platform_ipi smp_x86_platform_ipi
791

792
#ifdef CONFIG_HAVE_KVM
793
apicinterrupt3 POSTED_INTR_VECTOR \
794 795 796
	kvm_posted_intr_ipi smp_kvm_posted_intr_ipi
#endif

797
#ifdef CONFIG_X86_MCE_THRESHOLD
798
apicinterrupt THRESHOLD_APIC_VECTOR \
799
	threshold_interrupt smp_threshold_interrupt
800 801 802
#endif

#ifdef CONFIG_X86_THERMAL_VECTOR
803 804
apicinterrupt THERMAL_APIC_VECTOR \
	thermal_interrupt smp_thermal_interrupt
805
#endif
806

807 808 809 810 811 812 813 814
#ifdef CONFIG_SMP
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \
	call_function_single_interrupt smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR \
	call_function_interrupt smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR \
	reschedule_interrupt smp_reschedule_interrupt
#endif
L
Linus Torvalds 已提交
815

816 817 818 819
apicinterrupt ERROR_APIC_VECTOR \
	error_interrupt smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR \
	spurious_interrupt smp_spurious_interrupt
820

821 822 823
#ifdef CONFIG_IRQ_WORK
apicinterrupt IRQ_WORK_VECTOR \
	irq_work_interrupt smp_irq_work_interrupt
I
Ingo Molnar 已提交
824 825
#endif

L
Linus Torvalds 已提交
826 827
/*
 * Exception entry points.
828
 */
829
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
830 831

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
832
ENTRY(\sym)
833 834 835 836 837
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

838
	ASM_CLAC
839
	PARAVIRT_ADJUST_EXCEPTION_FRAME
840 841

	.ifeq \has_error_code
842
	pushq $-1			/* ORIG_RAX: no syscall to restart */
843 844
	.endif

845
	ALLOC_PT_GPREGS_ON_STACK
846 847

	.if \paranoid
848
	.if \paranoid == 1
849
	testb	$3, CS(%rsp)		/* If coming from userspace, switch */
850 851
	jnz 1f				/* stacks. */
	.endif
852
	call paranoid_entry
853 854 855
	.else
	call error_entry
	.endif
856
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
857 858

	.if \paranoid
859 860 861
	.if \shift_ist != -1
	TRACE_IRQS_OFF_DEBUG		/* reload IDT in case of recursion */
	.else
862
	TRACE_IRQS_OFF
863
	.endif
864
	.endif
865 866 867 868 869 870 871 872 873 874

	movq %rsp,%rdi			/* pt_regs pointer */

	.if \has_error_code
	movq ORIG_RAX(%rsp),%rsi	/* get error code */
	movq $-1,ORIG_RAX(%rsp)		/* no syscall to restart */
	.else
	xorl %esi,%esi			/* no error code */
	.endif

875
	.if \shift_ist != -1
876
	subq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
877 878
	.endif

879
	call \do_sym
880

881
	.if \shift_ist != -1
882
	addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
883 884
	.endif

885
	/* these procedures expect "no swapgs" flag in ebx */
886
	.if \paranoid
887
	jmp paranoid_exit
888
	.else
889
	jmp error_exit
890 891
	.endif

892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
	call error_entry


	movq %rsp,%rdi			/* pt_regs pointer */
	call sync_regs
	movq %rax,%rsp			/* switch stack */

	movq %rsp,%rdi			/* pt_regs pointer */

	.if \has_error_code
	movq ORIG_RAX(%rsp),%rsi	/* get error code */
	movq $-1,ORIG_RAX(%rsp)		/* no syscall to restart */
	.else
	xorl %esi,%esi			/* no error code */
	.endif

	call \do_sym

	jmp error_exit			/* %ebx: no swapgs flag */
	.endif
919
END(\sym)
920
.endm
921

922
#ifdef CONFIG_TRACING
923 924 925
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
926 927
.endm
#else
928 929
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
930 931 932
.endm
#endif

933 934 935 936 937
idtentry divide_error do_divide_error has_error_code=0
idtentry overflow do_overflow has_error_code=0
idtentry bounds do_bounds has_error_code=0
idtentry invalid_op do_invalid_op has_error_code=0
idtentry device_not_available do_device_not_available has_error_code=0
938
idtentry double_fault do_double_fault has_error_code=1 paranoid=2
939 940 941 942 943 944 945
idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
idtentry invalid_TSS do_invalid_TSS has_error_code=1
idtentry segment_not_present do_segment_not_present has_error_code=1
idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0
idtentry coprocessor_error do_coprocessor_error has_error_code=0
idtentry alignment_check do_alignment_check has_error_code=1
idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
946

947

948 949
	/* Reload gs selector with exception handling */
	/* edi:  new selector */
950
ENTRY(native_load_gs_index)
951
	pushfq
952
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
953
	SWAPGS
954
gs_change:
955
	movl %edi,%gs
L
Linus Torvalds 已提交
956
2:	mfence		/* workaround */
957
	SWAPGS
958
	popfq
959
	ret
960
END(native_load_gs_index)
961

962
	_ASM_EXTABLE(gs_change,bad_gs)
963
	.section .fixup,"ax"
L
Linus Torvalds 已提交
964
	/* running with kernelgs */
965
bad_gs:
966
	SWAPGS			/* switch back to user gs */
L
Linus Torvalds 已提交
967
	xorl %eax,%eax
968 969 970
	movl %eax,%gs
	jmp  2b
	.previous
971

972
/* Call softirq on interrupt stack. Interrupts are off. */
973
ENTRY(do_softirq_own_stack)
974
	pushq %rbp
975
	mov  %rsp,%rbp
976
	incl PER_CPU_VAR(irq_count)
977
	cmove PER_CPU_VAR(irq_stack_ptr),%rsp
978
	push  %rbp			# backlink for old unwinder
979
	call __do_softirq
980
	leaveq
981
	decl PER_CPU_VAR(irq_count)
982
	ret
983
END(do_softirq_own_stack)
984

985
#ifdef CONFIG_XEN
986
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
987 988

/*
989 990 991 992 993 994 995 996 997 998 999 1000
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
1001
ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
1002 1003 1004 1005
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
1006
	movq %rdi, %rsp            # we don't return, adjust the stack frame
1007
11:	incl PER_CPU_VAR(irq_count)
1008
	movq %rsp,%rbp
1009
	cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
1010 1011 1012
	pushq %rbp			# backlink for old unwinder
	call xen_evtchn_do_upcall
	popq %rsp
1013
	decl PER_CPU_VAR(irq_count)
1014 1015 1016
#ifndef CONFIG_PREEMPT
	call xen_maybe_preempt_hcall
#endif
1017
	jmp  error_exit
1018
END(xen_do_hypervisor_callback)
1019 1020

/*
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1033
ENTRY(xen_failsafe_callback)
1034
	movl %ds,%ecx
1035 1036
	cmpw %cx,0x10(%rsp)
	jne 1f
1037
	movl %es,%ecx
1038 1039
	cmpw %cx,0x18(%rsp)
	jne 1f
1040
	movl %fs,%ecx
1041 1042
	cmpw %cx,0x20(%rsp)
	jne 1f
1043
	movl %gs,%ecx
1044 1045 1046 1047 1048 1049
	cmpw %cx,0x28(%rsp)
	jne 1f
	/* All segments match their saved values => Category 2 (Bad IRET). */
	movq (%rsp),%rcx
	movq 8(%rsp),%r11
	addq $0x30,%rsp
1050 1051 1052
	pushq $0	/* RIP */
	pushq %r11
	pushq %rcx
1053
	jmp general_protection
1054 1055 1056 1057
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
	movq (%rsp),%rcx
	movq 8(%rsp),%r11
	addq $0x30,%rsp
1058
	pushq $-1 /* orig_ax = -1 => not a system call */
1059 1060 1061
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
1062 1063 1064
	jmp error_exit
END(xen_failsafe_callback)

1065
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1066 1067
	xen_hvm_callback_vector xen_evtchn_do_upcall

1068
#endif /* CONFIG_XEN */
1069

1070
#if IS_ENABLED(CONFIG_HYPERV)
1071
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1072 1073 1074
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

1075 1076
idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
1077
idtentry stack_segment do_stack_segment has_error_code=1
1078
#ifdef CONFIG_XEN
1079 1080 1081
idtentry xen_debug do_debug has_error_code=0
idtentry xen_int3 do_int3 has_error_code=0
idtentry xen_stack_segment do_stack_segment has_error_code=1
1082
#endif
1083 1084
idtentry general_protection do_general_protection has_error_code=1
trace_idtentry page_fault do_page_fault has_error_code=1
G
Gleb Natapov 已提交
1085
#ifdef CONFIG_KVM_GUEST
1086
idtentry async_page_fault do_async_page_fault has_error_code=1
G
Gleb Natapov 已提交
1087
#endif
1088
#ifdef CONFIG_X86_MCE
1089
idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vector(%rip)
1090 1091
#endif

1092 1093 1094 1095 1096 1097
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
	movl $1,%ebx
	movl $MSR_GS_BASE,%ecx
	rdmsr
	testl %edx,%edx
	js 1f	/* negative -> in kernel */
	SWAPGS
	xorl %ebx,%ebx
1:	ret
1109
END(paranoid_entry)
1110

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
 */
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
1122 1123
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
1124
	TRACE_IRQS_OFF_DEBUG
1125
	testl %ebx,%ebx				/* swapgs needed? */
1126
	jnz paranoid_exit_no_swapgs
1127
	TRACE_IRQS_IRETQ
1128
	SWAPGS_UNSAFE_STACK
1129 1130
	jmp paranoid_exit_restore
paranoid_exit_no_swapgs:
1131
	TRACE_IRQS_IRETQ_DEBUG
1132
paranoid_exit_restore:
1133 1134 1135
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
1136
	INTERRUPT_RETURN
1137 1138 1139
END(paranoid_exit)

/*
1140 1141
 * Save all registers in pt_regs, and switch gs if needed.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
1142 1143 1144
 */
ENTRY(error_entry)
	cld
1145 1146
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1147
	xorl %ebx,%ebx
1148
	testb	$3, CS+8(%rsp)
1149
	jz	error_kernelspace
1150 1151 1152 1153 1154 1155
error_swapgs:
	SWAPGS
error_sti:
	TRACE_IRQS_OFF
	ret

1156 1157 1158 1159 1160 1161
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1162 1163
error_kernelspace:
	incl %ebx
1164
	leaq native_irq_return_iret(%rip),%rcx
1165
	cmpq %rcx,RIP+8(%rsp)
A
Andy Lutomirski 已提交
1166
	je error_bad_iret
1167 1168 1169
	movl %ecx,%eax	/* zero extend */
	cmpq %rax,RIP+8(%rsp)
	je bstep_iret
1170
	cmpq $gs_change,RIP+8(%rsp)
1171
	je error_swapgs
1172
	jmp error_sti
1173 1174 1175 1176

bstep_iret:
	/* Fix truncated RIP */
	movq %rcx,RIP+8(%rsp)
A
Andy Lutomirski 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185
	/* fall through */

error_bad_iret:
	SWAPGS
	mov %rsp,%rdi
	call fixup_bad_iret
	mov %rax,%rsp
	decl %ebx	/* Return to usergs */
	jmp error_sti
1186 1187 1188
END(error_entry)


1189
/* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) */
1190 1191
ENTRY(error_exit)
	movl %ebx,%eax
1192
	RESTORE_EXTRA_REGS
1193 1194 1195
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
	testl %eax,%eax
1196
	jnz retint_kernel
1197
	jmp retint_user
1198 1199
END(error_exit)

1200
/* Runs on exception stack */
1201 1202
ENTRY(nmi)
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
	 *    o Copy the interrupt frame into a "saved" location on the stack
	 *    o Copy the interrupt frame into a "copy" location on the stack
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
	 *    o Modify the "copy" location to jump to the repeate_nmi
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
	 */

1235
	/* Use %rdx as our temp variable throughout */
1236
	pushq %rdx
1237

1238 1239 1240 1241
	/*
	 * If %cs was not the kernel segment, then the NMI triggered in user
	 * space, which means it is definitely not nested.
	 */
1242
	cmpl $__KERNEL_CS, 16(%rsp)
1243 1244
	jne first_nmi

1245 1246 1247 1248
	/*
	 * Check the special variable on the stack to see if NMIs are
	 * executing.
	 */
1249
	cmpl $1, -8(%rsp)
1250 1251 1252 1253 1254 1255 1256 1257 1258
	je nested_nmi

	/*
	 * Now test if the previous stack was an NMI stack.
	 * We need the double check. We check the NMI stack to satisfy the
	 * race when the first NMI clears the variable before returning.
	 * We check the variable because the first NMI could be in a
	 * breakpoint routine using a breakpoint stack.
	 */
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
	/* Ah, it is within the NMI stack, treat it as nested */

1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
nested_nmi:
	/*
	 * Do nothing if we interrupted the fixup in repeat_nmi.
	 * It's about to repeat the NMI handler, so we are fine
	 * with ignoring this one.
	 */
	movq $repeat_nmi, %rdx
	cmpq 8(%rsp), %rdx
	ja 1f
	movq $end_repeat_nmi, %rdx
	cmpq 8(%rsp), %rdx
	ja nested_nmi_out

1:
	/* Set up the interrupted NMIs stack to jump to repeat_nmi */
1285
	leaq -1*8(%rsp), %rdx
1286
	movq %rdx, %rsp
1287
	leaq -10*8(%rsp), %rdx
1288 1289 1290 1291 1292
	pushq $__KERNEL_DS
	pushq %rdx
	pushfq
	pushq $__KERNEL_CS
	pushq $repeat_nmi
1293 1294

	/* Put stack back */
1295
	addq $(6*8), %rsp
1296 1297

nested_nmi_out:
1298
	popq %rdx
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324

	/* No need to check faults here */
	INTERRUPT_RETURN

first_nmi:
	/*
	 * Because nested NMIs will use the pushed location that we
	 * stored in rdx, we must keep that space available.
	 * Here's what our stack frame will look like:
	 * +-------------------------+
	 * | original SS             |
	 * | original Return RSP     |
	 * | original RFLAGS         |
	 * | original CS             |
	 * | original RIP            |
	 * +-------------------------+
	 * | temp storage for rdx    |
	 * +-------------------------+
	 * | NMI executing variable  |
	 * +-------------------------+
	 * | copied SS               |
	 * | copied Return RSP       |
	 * | copied RFLAGS           |
	 * | copied CS               |
	 * | copied RIP              |
	 * +-------------------------+
1325 1326 1327 1328 1329 1330
	 * | Saved SS                |
	 * | Saved Return RSP        |
	 * | Saved RFLAGS            |
	 * | Saved CS                |
	 * | Saved RIP               |
	 * +-------------------------+
1331 1332 1333
	 * | pt_regs                 |
	 * +-------------------------+
	 *
1334 1335 1336
	 * The saved stack frame is used to fix up the copied stack frame
	 * that a nested NMI may change to make the interrupted NMI iret jump
	 * to the repeat_nmi. The original stack frame and the temp storage
1337 1338
	 * is also used by nested NMIs and can not be trusted on exit.
	 */
1339
	/* Do not pop rdx, nested NMIs will corrupt that part of the stack */
1340 1341
	movq (%rsp), %rdx

1342
	/* Set the NMI executing variable on the stack. */
1343
	pushq $1
1344

1345 1346 1347 1348 1349
	/*
	 * Leave room for the "copied" frame
	 */
	subq $(5*8), %rsp

1350 1351
	/* Copy the stack frame to the Saved frame */
	.rept 5
1352
	pushq 11*8(%rsp)
1353
	.endr
1354

1355 1356
	/* Everything up to here is safe from nested NMIs */

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
	 */
repeat_nmi:
	/*
	 * Update the stack variable to say we are still in NMI (the update
	 * is benign for the non-repeat case, where 1 was pushed just above
	 * to this very stack slot).
	 */
1372
	movq $1, 10*8(%rsp)
1373 1374

	/* Make another copy, this one may be modified by nested NMIs */
1375
	addq $(10*8), %rsp
1376
	.rept 5
1377
	pushq -6*8(%rsp)
1378
	.endr
1379
	subq $(5*8), %rsp
1380
end_repeat_nmi:
1381 1382 1383

	/*
	 * Everything below this point can be preempted by a nested
1384 1385
	 * NMI if the first NMI took an exception and reset our iret stack
	 * so that we repeat another NMI.
1386
	 */
1387
	pushq $-1		/* ORIG_RAX: no syscall to restart */
1388 1389
	ALLOC_PT_GPREGS_ON_STACK

1390
	/*
1391
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1392 1393 1394 1395 1396
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1397
	call paranoid_entry
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409

	/*
	 * Save off the CR2 register. If we take a page fault in the NMI then
	 * it could corrupt the CR2 value. If the NMI preempts a page fault
	 * handler before it was able to read the CR2 register, and then the
	 * NMI itself takes a page fault, the page fault that was preempted
	 * will read the information from the NMI page fault and not the
	 * origin fault. Save it off and restore it if it changes.
	 * Use the r12 callee-saved register.
	 */
	movq %cr2, %r12

1410 1411 1412 1413
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
	movq %rsp,%rdi
	movq $-1,%rsi
	call do_nmi
1414 1415 1416 1417 1418 1419 1420

	/* Did the NMI take a page fault? Restore cr2 if it did */
	movq %cr2, %rcx
	cmpq %rcx, %r12
	je 1f
	movq %r12, %cr2
1:
1421 1422 1423 1424 1425
	testl %ebx,%ebx				/* swapgs needed? */
	jnz nmi_restore
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1426 1427
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1428
	/* Pop the extra iret frame at once */
1429
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1430

1431
	/* Clear the NMI executing stack variable */
1432
	movq $0, 5*8(%rsp)
1433
	INTERRUPT_RETURN
1434 1435 1436 1437 1438 1439 1440
END(nmi)

ENTRY(ignore_sysret)
	mov $-ENOSYS,%eax
	sysret
END(ignore_sysret)