entry_64.S 39.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
7
 *
L
Linus Torvalds 已提交
8 9
 * entry.S contains the system-call and fault low-level handling routines.
 *
10 11
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
12
 * A note on terminology:
13 14
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
15 16
 *
 * Some macro usage:
17 18 19
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
20 21 22 23 24
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
25
#include "calling.h"
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/smap.h>
37
#include <asm/pgtable_types.h>
38
#include <linux/err.h>
L
Linus Torvalds 已提交
39

R
Roland McGrath 已提交
40 41
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
42 43 44
#define AUDIT_ARCH_X86_64			(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT			0x80000000
#define __AUDIT_ARCH_LE				0x40000000
J
Jiri Olsa 已提交
45

46 47
.code64
.section .entry.text, "ax"
48

49
#ifdef CONFIG_PARAVIRT
50
ENTRY(native_usergs_sysret64)
51 52
	swapgs
	sysretq
53
ENDPROC(native_usergs_sysret64)
54 55
#endif /* CONFIG_PARAVIRT */

56
.macro TRACE_IRQS_IRETQ
57
#ifdef CONFIG_TRACE_IRQFLAGS
58 59
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
60 61 62 63 64
	TRACE_IRQS_ON
1:
#endif
.endm

65 66 67 68 69 70 71 72 73 74 75 76 77 78
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
79
	call	debug_stack_set_zero
80
	TRACE_IRQS_OFF
81
	call	debug_stack_reset
82 83 84
.endm

.macro TRACE_IRQS_ON_DEBUG
85
	call	debug_stack_set_zero
86
	TRACE_IRQS_ON
87
	call	debug_stack_reset
88 89
.endm

90
.macro TRACE_IRQS_IRETQ_DEBUG
91 92
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
93 94 95 96 97
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
98 99 100
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
101 102
#endif

L
Linus Torvalds 已提交
103
/*
104
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
105
 *
106
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
107 108 109 110 111 112
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
113
 * rax  system call number
114 115
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
116 117
 * rdi  arg0
 * rsi  arg1
118
 * rdx  arg2
119
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
120 121
 * r8   arg4
 * r9   arg5
122
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
123
 *
L
Linus Torvalds 已提交
124 125
 * Only called from user space.
 *
126
 * When user can change pt_regs->foo always force IRET. That is because
127 128
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
129
 */
L
Linus Torvalds 已提交
130

131
ENTRY(entry_SYSCALL_64)
132 133 134 135 136
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
137 138 139 140 141 142
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
143
GLOBAL(entry_SYSCALL_64_after_swapgs)
144

145 146
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
147

148 149
	TRACE_IRQS_OFF

150
	/* Construct struct pt_regs on stack */
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */

168 169 170 171 172 173 174
	/*
	 * If we need to do entry work or if we guess we'll need to do
	 * exit work, go straight to the slow path.
	 */
	testl	$_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	entry_SYSCALL64_slow_path

175
entry_SYSCALL_64_fastpath:
176 177 178 179 180 181 182
	/*
	 * Easy case: enable interrupts and issue the syscall.  If the syscall
	 * needs pt_regs, we'll call a stub that disables interrupts again
	 * and jumps to the slow path.
	 */
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
183
#if __SYSCALL_MASK == ~0
184
	cmpq	$__NR_syscall_max, %rax
185
#else
186 187
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
188
#endif
189 190
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
191 192 193

	/*
	 * This call instruction is handled specially in stub_ptregs_64.
194 195
	 * It might end up jumping to the slow path.  If it jumps, RAX
	 * and all argument registers are clobbered.
196
	 */
197
	call	*sys_call_table(, %rax, 8)
198 199
.Lentry_SYSCALL_64_after_fastpath_call:

200
	movq	%rax, RAX(%rsp)
201
1:
202 203

	/*
204 205 206
	 * If we get here, then we know that pt_regs is clean for SYSRET64.
	 * If we see that no exit work is required (which we are required
	 * to check with IRQs off), then we can go straight to SYSRET64.
207
	 */
208 209
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
210
	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
211
	jnz	1f
212

213 214 215
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON		/* user mode is traced as IRQs on */
	RESTORE_C_REGS
216
	movq	RSP(%rsp), %rsp
217
	USERGS_SYSRET64
L
Linus Torvalds 已提交
218

219 220 221 222 223 224
1:
	/*
	 * The fast path looked good when we started, but something changed
	 * along the way and we need to switch to the slow path.  Calling
	 * raise(3) will trigger this, for example.  IRQs are off.
	 */
225 226
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
227
	SAVE_EXTRA_REGS
228
	movq	%rsp, %rdi
229 230
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	jmp	return_from_SYSCALL_64
231

232 233
entry_SYSCALL64_slow_path:
	/* IRQs are off. */
234
	SAVE_EXTRA_REGS
235
	movq	%rsp, %rdi
236 237 238
	call	do_syscall_64		/* returns with IRQs disabled */

return_from_SYSCALL_64:
239
	RESTORE_EXTRA_REGS
240
	TRACE_IRQS_IRETQ		/* we're about to change IF */
241 242 243 244 245

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
246 247 248 249
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
	cmpq	%rcx, %r11			/* RCX == RIP */
	jne	opportunistic_sysret_failed
250 251 252 253

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
254
	 * the kernel, since userspace controls RSP.
255
	 *
256
	 * If width of "canonical tail" ever becomes variable, this will need
257 258 259 260 261
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
262

263 264 265
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
266

267 268 269
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
270

271 272
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
	jne	opportunistic_sysret_failed
273

274 275 276
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne	opportunistic_sysret_failed
277 278 279 280 281 282 283 284

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
285
	 *           movq	$stuck_here, %rcx
286 287 288 289 290 291
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
292 293
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz	opportunistic_sysret_failed
294 295 296

	/* nothing to check for RSP */

297 298
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
	jne	opportunistic_sysret_failed
299 300

	/*
301 302
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
303 304
	 */
syscall_return_via_sysret:
305 306
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
307
	movq	RSP(%rsp), %rsp
308 309 310 311 312
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
313
END(entry_SYSCALL_64)
314

315 316 317
ENTRY(stub_ptregs_64)
	/*
	 * Syscalls marked as needing ptregs land here.
318 319 320
	 * If we are on the fast path, we need to save the extra regs,
	 * which we achieve by trying again on the slow path.  If we are on
	 * the slow path, the extra regs are already saved.
321 322
	 *
	 * RAX stores a pointer to the C function implementing the syscall.
323
	 * IRQs are on.
324 325 326 327
	 */
	cmpq	$.Lentry_SYSCALL_64_after_fastpath_call, (%rsp)
	jne	1f

328 329 330 331 332 333
	/*
	 * Called from fast path -- disable IRQs again, pop return address
	 * and jump to slow path
	 */
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
334
	popq	%rax
335
	jmp	entry_SYSCALL64_slow_path
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

1:
	/* Called from C */
	jmp	*%rax				/* called from C */
END(stub_ptregs_64)

.macro ptregs_stub func
ENTRY(ptregs_\func)
	leaq	\func(%rip), %rax
	jmp	stub_ptregs_64
END(ptregs_\func)
.endm

/* Instantiate ptregs_stub for each ptregs-using syscall */
#define __SYSCALL_64_QUAL_(sym)
#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym
#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym)
#include <asm/syscalls_64.h>
354

355 356 357 358 359 360
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)
361
	LOCK ; btr $TIF_FORK, TI_flags(%r8)
362

363 364
	pushq	$0x0002
	popfq					/* reset kernel eflags */
365

366
	call	schedule_tail			/* rdi: 'prev' task parameter */
367

368
	testb	$3, CS(%rsp)			/* from kernel_thread? */
369
	jnz	1f
370

371
	/*
372 373 374 375 376 377
	 * We came from kernel_thread.  This code path is quite twisted, and
	 * someone should clean it up.
	 *
	 * copy_thread_tls stashes the function pointer in RBX and the
	 * parameter to be passed in RBP.  The called function is permitted
	 * to call do_execve and thereby jump to user mode.
378
	 */
379 380 381
	movq	RBP(%rsp), %rdi
	call	*RBX(%rsp)
	movl	$0, RAX(%rsp)
382

383
	/*
384 385
	 * Fall through as though we're exiting a syscall.  This makes a
	 * twisted sort of sense if we just called do_execve.
386
	 */
387 388 389 390 391 392 393

1:
	movq	%rsp, %rdi
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
	SWAPGS
	jmp	restore_regs_and_iret
394 395
END(ret_from_fork)

396
/*
397 398
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
399
 */
400
	.align 8
401
ENTRY(irq_entries_start)
402 403
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
404
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
405 406 407 408
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
409 410
END(irq_entries_start)

411
/*
L
Linus Torvalds 已提交
412 413 414
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
415 416 417
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
418

419
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
420
	.macro interrupt func
421
	cld
422 423 424
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
425

426
	testb	$3, CS(%rsp)
427
	jz	1f
428 429 430 431 432

	/*
	 * IRQ from user mode.  Switch to kernel gsbase and inform context
	 * tracking that we're in kernel mode.
	 */
433
	SWAPGS
434 435 436 437 438 439 440 441 442 443 444

	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
	 * the simplest way to handle it is to just call it twice if
	 * we enter from user mode.  There's no reason to optimize this since
	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
	 */
	TRACE_IRQS_OFF

445
	CALL_enter_from_user_mode
446

447
1:
448
	/*
D
Denys Vlasenko 已提交
449
	 * Save previous stack pointer, optionally switch to interrupt stack.
450 451 452 453 454
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
455
	movq	%rsp, %rdi
456 457
	incl	PER_CPU_VAR(irq_count)
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
458
	pushq	%rdi
459 460 461
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

462
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
463 464
	.endm

465 466 467 468
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
469 470
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
471
	ASM_CLAC
472
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
473
	interrupt do_IRQ
474
	/* 0(%rsp): old RSP */
475
ret_from_intr:
476
	DISABLE_INTERRUPTS(CLBR_NONE)
477
	TRACE_IRQS_OFF
478
	decl	PER_CPU_VAR(irq_count)
479

480
	/* Restore saved previous stack */
481
	popq	%rsp
482

483
	testb	$3, CS(%rsp)
484
	jz	retint_kernel
485

486 487 488 489
	/* Interrupt came from user space */
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
490
	TRACE_IRQS_IRETQ
491
	SWAPGS
492
	jmp	restore_regs_and_iret
493

494
/* Returning to kernel space */
495
retint_kernel:
496 497 498
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
499
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
500
	jnc	1f
501
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
502
	jnz	1f
503
	call	preempt_schedule_irq
504
	jmp	0b
505
1:
506
#endif
507 508 509 510
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
511 512 513 514 515

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
516
GLOBAL(restore_regs_and_iret)
517
	RESTORE_EXTRA_REGS
518
restore_c_regs_and_iret:
519 520
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
521 522 523
	INTERRUPT_RETURN

ENTRY(native_iret)
524 525 526 527
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
528
#ifdef CONFIG_X86_ESPFIX64
529 530
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
531
#endif
532

533
.global native_irq_return_iret
534
native_irq_return_iret:
A
Andy Lutomirski 已提交
535 536 537 538 539 540
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
541
	iretq
I
Ingo Molnar 已提交
542

543
#ifdef CONFIG_X86_ESPFIX64
544
native_irq_return_ldt:
545 546
	pushq	%rax
	pushq	%rdi
547
	SWAPGS
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
	movq	PER_CPU_VAR(espfix_waddr), %rdi
	movq	%rax, (0*8)(%rdi)		/* RAX */
	movq	(2*8)(%rsp), %rax		/* RIP */
	movq	%rax, (1*8)(%rdi)
	movq	(3*8)(%rsp), %rax		/* CS */
	movq	%rax, (2*8)(%rdi)
	movq	(4*8)(%rsp), %rax		/* RFLAGS */
	movq	%rax, (3*8)(%rdi)
	movq	(6*8)(%rsp), %rax		/* SS */
	movq	%rax, (5*8)(%rdi)
	movq	(5*8)(%rsp), %rax		/* RSP */
	movq	%rax, (4*8)(%rdi)
	andl	$0xffff0000, %eax
	popq	%rdi
	orq	PER_CPU_VAR(espfix_stack), %rax
563
	SWAPGS
564 565 566
	movq	%rax, %rsp
	popq	%rax
	jmp	native_irq_return_iret
567
#endif
568
END(common_interrupt)
569

L
Linus Torvalds 已提交
570 571
/*
 * APIC interrupts.
572
 */
573
.macro apicinterrupt3 num sym do_sym
574
ENTRY(\sym)
575
	ASM_CLAC
576
	pushq	$~(\num)
577
.Lcommon_\sym:
578
	interrupt \do_sym
579
	jmp	ret_from_intr
580 581
END(\sym)
.endm
L
Linus Torvalds 已提交
582

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

600
#ifdef CONFIG_SMP
601 602
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
603
#endif
L
Linus Torvalds 已提交
604

N
Nick Piggin 已提交
605
#ifdef CONFIG_X86_UV
606
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
607
#endif
608 609 610

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
611

612
#ifdef CONFIG_HAVE_KVM
613 614
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
615 616
#endif

617
#ifdef CONFIG_X86_MCE_THRESHOLD
618
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
619 620
#endif

621
#ifdef CONFIG_X86_MCE_AMD
622
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
623 624
#endif

625
#ifdef CONFIG_X86_THERMAL_VECTOR
626
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
627
#endif
628

629
#ifdef CONFIG_SMP
630 631 632
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
633
#endif
L
Linus Torvalds 已提交
634

635 636
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
637

638
#ifdef CONFIG_IRQ_WORK
639
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
640 641
#endif

L
Linus Torvalds 已提交
642 643
/*
 * Exception entry points.
644
 */
645
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
646 647

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
648
ENTRY(\sym)
649 650 651 652 653
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

654
	ASM_CLAC
655
	PARAVIRT_ADJUST_EXCEPTION_FRAME
656 657

	.ifeq \has_error_code
658
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
659 660
	.endif

661
	ALLOC_PT_GPREGS_ON_STACK
662 663

	.if \paranoid
664
	.if \paranoid == 1
665 666
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
	jnz	1f
667
	.endif
668
	call	paranoid_entry
669
	.else
670
	call	error_entry
671
	.endif
672
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
673 674

	.if \paranoid
675
	.if \shift_ist != -1
676
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
677
	.else
678
	TRACE_IRQS_OFF
679
	.endif
680
	.endif
681

682
	movq	%rsp, %rdi			/* pt_regs pointer */
683 684

	.if \has_error_code
685 686
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
687
	.else
688
	xorl	%esi, %esi			/* no error code */
689 690
	.endif

691
	.if \shift_ist != -1
692
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
693 694
	.endif

695
	call	\do_sym
696

697
	.if \shift_ist != -1
698
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
699 700
	.endif

701
	/* these procedures expect "no swapgs" flag in ebx */
702
	.if \paranoid
703
	jmp	paranoid_exit
704
	.else
705
	jmp	error_exit
706 707
	.endif

708 709 710 711 712 713 714
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
715
	call	error_entry
716 717


718 719 720
	movq	%rsp, %rdi			/* pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
721

722
	movq	%rsp, %rdi			/* pt_regs pointer */
723 724

	.if \has_error_code
725 726
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
727
	.else
728
	xorl	%esi, %esi			/* no error code */
729 730
	.endif

731
	call	\do_sym
732

733
	jmp	error_exit			/* %ebx: no swapgs flag */
734
	.endif
735
END(\sym)
736
.endm
737

738
#ifdef CONFIG_TRACING
739 740 741
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
742 743
.endm
#else
744 745
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
746 747 748
.endm
#endif

749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
768
ENTRY(native_load_gs_index)
769
	pushfq
770
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
771
	SWAPGS
772
gs_change:
773 774
	movl	%edi, %gs
2:	mfence					/* workaround */
775
	SWAPGS
776
	popfq
777
	ret
778
END(native_load_gs_index)
779

780 781
	_ASM_EXTABLE(gs_change, bad_gs)
	.section .fixup, "ax"
L
Linus Torvalds 已提交
782
	/* running with kernelgs */
783
bad_gs:
784 785 786 787
	SWAPGS					/* switch back to user gs */
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
788
	.previous
789

790
/* Call softirq on interrupt stack. Interrupts are off. */
791
ENTRY(do_softirq_own_stack)
792 793 794 795 796 797
	pushq	%rbp
	mov	%rsp, %rbp
	incl	PER_CPU_VAR(irq_count)
	cmove	PER_CPU_VAR(irq_stack_ptr), %rsp
	push	%rbp				/* frame pointer backlink */
	call	__do_softirq
798
	leaveq
799
	decl	PER_CPU_VAR(irq_count)
800
	ret
801
END(do_softirq_own_stack)
802

803
#ifdef CONFIG_XEN
804
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
805 806

/*
807 808 809 810 811 812 813 814 815 816 817 818
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
819 820
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

821 822 823 824
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
825 826 827 828 829 830 831 832
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
11:	incl	PER_CPU_VAR(irq_count)
	movq	%rsp, %rbp
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
	pushq	%rbp				/* frame pointer backlink */
	call	xen_evtchn_do_upcall
	popq	%rsp
	decl	PER_CPU_VAR(irq_count)
833
#ifndef CONFIG_PREEMPT
834
	call	xen_maybe_preempt_hcall
835
#endif
836
	jmp	error_exit
837
END(xen_do_hypervisor_callback)
838 839

/*
840 841 842 843 844 845 846 847 848 849 850 851
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
852
ENTRY(xen_failsafe_callback)
853 854 855 856 857 858 859 860 861 862 863 864
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
865
	/* All segments match their saved values => Category 2 (Bad IRET). */
866 867 868 869 870 871 872
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
	pushq	%r11
	pushq	%rcx
	jmp	general_protection
873
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
874 875 876 877
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$-1 /* orig_ax = -1 => not a system call */
878 879 880
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
881
	jmp	error_exit
882 883
END(xen_failsafe_callback)

884
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
885 886
	xen_hvm_callback_vector xen_evtchn_do_upcall

887
#endif /* CONFIG_XEN */
888

889
#if IS_ENABLED(CONFIG_HYPERV)
890
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
891 892 893
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

894 895 896 897
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

898
#ifdef CONFIG_XEN
899 900 901
idtentry xen_debug		do_debug		has_error_code=0
idtentry xen_int3		do_int3			has_error_code=0
idtentry xen_stack_segment	do_stack_segment	has_error_code=1
902
#endif
903 904 905 906

idtentry general_protection	do_general_protection	has_error_code=1
trace_idtentry page_fault	do_page_fault		has_error_code=1

G
Gleb Natapov 已提交
907
#ifdef CONFIG_KVM_GUEST
908
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
909
#endif
910

911
#ifdef CONFIG_X86_MCE
912
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
913 914
#endif

915 916 917 918 919 920
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
921 922 923
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
924 925
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
926
	rdmsr
927 928
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
929
	SWAPGS
930
	xorl	%ebx, %ebx
931
1:	ret
932
END(paranoid_entry)
933

934 935 936 937 938 939 940 941 942
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
943 944
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
945
 */
946 947
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
948
	TRACE_IRQS_OFF_DEBUG
949 950
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	paranoid_exit_no_swapgs
951
	TRACE_IRQS_IRETQ
952
	SWAPGS_UNSAFE_STACK
953
	jmp	paranoid_exit_restore
954
paranoid_exit_no_swapgs:
955
	TRACE_IRQS_IRETQ_DEBUG
956
paranoid_exit_restore:
957 958 959
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
960
	INTERRUPT_RETURN
961 962 963
END(paranoid_exit)

/*
964
 * Save all registers in pt_regs, and switch gs if needed.
965
 * Return: EBX=0: came from user mode; EBX=1: otherwise
966 967 968
 */
ENTRY(error_entry)
	cld
969 970
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
971
	xorl	%ebx, %ebx
972
	testb	$3, CS+8(%rsp)
973
	jz	.Lerror_kernelspace
974

975 976 977 978 979
.Lerror_entry_from_usermode_swapgs:
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
980
	SWAPGS
981

982
.Lerror_entry_from_usermode_after_swapgs:
983 984 985 986 987 988
	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).
	 */
	TRACE_IRQS_OFF
989
	CALL_enter_from_user_mode
990
	ret
991

992
.Lerror_entry_done:
993 994 995
	TRACE_IRQS_OFF
	ret

996 997 998 999 1000 1001
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1002
.Lerror_kernelspace:
1003 1004 1005
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1006
	je	.Lerror_bad_iret
1007 1008
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1009
	je	.Lbstep_iret
1010
	cmpq	$gs_change, RIP+8(%rsp)
1011
	jne	.Lerror_entry_done
1012 1013 1014 1015 1016 1017

	/*
	 * hack: gs_change can fail with user gsbase.  If this happens, fix up
	 * gsbase and proceed.  We'll fix up the exception and land in
	 * gs_change's error handler with kernel gsbase.
	 */
1018
	jmp	.Lerror_entry_from_usermode_swapgs
1019

1020
.Lbstep_iret:
1021
	/* Fix truncated RIP */
1022
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1023 1024
	/* fall through */

1025
.Lerror_bad_iret:
1026 1027 1028 1029
	/*
	 * We came from an IRET to user mode, so we have user gsbase.
	 * Switch to kernel gsbase:
	 */
A
Andy Lutomirski 已提交
1030
	SWAPGS
1031 1032 1033 1034 1035 1036

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1037 1038 1039
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1040
	decl	%ebx
1041
	jmp	.Lerror_entry_from_usermode_after_swapgs
1042 1043 1044
END(error_entry)


1045 1046 1047 1048 1049
/*
 * On entry, EBS is a "return to kernel mode" flag:
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1050
ENTRY(error_exit)
1051
	movl	%ebx, %eax
1052 1053
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
1054 1055 1056
	testl	%eax, %eax
	jnz	retint_kernel
	jmp	retint_user
1057 1058
END(error_exit)

1059
/* Runs on exception stack */
1060
ENTRY(nmi)
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
	/*
	 * Fix up the exception frame if we're on Xen.
	 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
	 * one value to the stack on native, so it may clobber the rdx
	 * scratch slot, but it won't clobber any of the important
	 * slots past it.
	 *
	 * Xen is a different story, because the Xen frame itself overlaps
	 * the "NMI executing" variable.
	 */
1071
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1072

1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1090 1091 1092
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1093 1094
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1095
	 *    o Modify the "iret" location to jump to the repeat_nmi
1096 1097 1098 1099 1100 1101 1102 1103
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1104 1105 1106 1107 1108
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1109 1110
	 */

1111
	/* Use %rdx as our temp variable throughout */
1112
	pushq	%rdx
1113

1114 1115 1116 1117 1118 1119 1120 1121 1122
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
1123 1124 1125
	 *
	 * We also must not push anything to the stack before switching
	 * stacks lest we corrupt the "NMI executing" variable.
1126 1127
	 */

1128
	SWAPGS_UNSAFE_STACK
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
	cld
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1164
	/*
1165 1166 1167
	 * Return back to user mode.  We must *not* do the normal exit
	 * work, because we don't want to enable interrupts.  Fortunately,
	 * do_nmi doesn't modify pt_regs.
1168
	 */
1169 1170
	SWAPGS
	jmp	restore_c_regs_and_iret
1171

1172
.Lnmi_from_kernel:
1173
	/*
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1214
	/*
1215 1216
	 * Determine whether we're a nested NMI.
	 *
1217 1218 1219 1220 1221 1222
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1223
	 */
1224 1225 1226 1227 1228 1229 1230 1231

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1232

1233
	/*
1234
	 * Now check "NMI executing".  If it's set, then we're nested.
1235 1236
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1237
	 */
1238 1239
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1240 1241

	/*
1242 1243
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1244 1245 1246 1247 1248 1249 1250 1251
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1252
	 */
1253 1254 1255 1256 1257
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1258

1259 1260 1261 1262
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1263 1264 1265 1266 1267 1268 1269

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1270

1271 1272
nested_nmi:
	/*
1273 1274
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1275
	 */
1276
	subq	$8, %rsp
1277 1278 1279
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1280
	pushfq
1281 1282
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1283 1284

	/* Put stack back */
1285
	addq	$(6*8), %rsp
1286 1287

nested_nmi_out:
1288
	popq	%rdx
1289

1290
	/* We are returning to kernel mode, so this cannot result in a fault. */
1291 1292 1293
	INTERRUPT_RETURN

first_nmi:
1294
	/* Restore rdx. */
1295
	movq	(%rsp), %rdx
1296

1297 1298
	/* Make room for "NMI executing". */
	pushq	$0
1299

1300
	/* Leave room for the "iret" frame */
1301
	subq	$(5*8), %rsp
1302

1303
	/* Copy the "original" frame to the "outermost" frame */
1304
	.rept 5
1305
	pushq	11*8(%rsp)
1306
	.endr
1307

1308 1309
	/* Everything up to here is safe from nested NMIs */

1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
	INTERRUPT_RETURN	/* continues at repeat_nmi below */
1:
#endif

1325
repeat_nmi:
1326 1327 1328 1329 1330 1331 1332 1333
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1334 1335 1336 1337
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1338 1339
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1340
	 */
1341
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1342

1343
	/*
1344 1345 1346
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1347
	 */
1348
	addq	$(10*8), %rsp
1349
	.rept 5
1350
	pushq	-6*8(%rsp)
1351
	.endr
1352
	subq	$(5*8), %rsp
1353
end_repeat_nmi:
1354 1355

	/*
1356 1357 1358
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1359
	 */
1360
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1361 1362
	ALLOC_PT_GPREGS_ON_STACK

1363
	/*
1364
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1365 1366 1367 1368 1369
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1370
	call	paranoid_entry
1371

1372
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1373 1374 1375
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1376

1377 1378
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1379 1380 1381
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1382 1383
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1384 1385

	/* Point RSP at the "iret" frame. */
1386
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1387

1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
	 * the SYSCALL entry and exit paths.  On a native kernel, we
	 * could just inspect RIP, but, on paravirt kernels,
	 * INTERRUPT_RETURN can translate into a jump into a
	 * hypercall page.
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1398 1399 1400 1401 1402 1403

	/*
	 * INTERRUPT_RETURN reads the "iret" frame and exits the NMI
	 * stack in a single instruction.  We are returning to kernel
	 * mode, so this cannot result in a fault.
	 */
1404
	INTERRUPT_RETURN
1405 1406 1407
END(nmi)

ENTRY(ignore_sysret)
1408
	mov	$-ENOSYS, %eax
1409 1410
	sysret
END(ignore_sysret)