entry_64.S 41.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
7
 *
L
Linus Torvalds 已提交
8 9
 * entry.S contains the system-call and fault low-level handling routines.
 *
10 11
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
12
 * A note on terminology:
13 14
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
15 16
 *
 * Some macro usage:
17 18 19
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
20 21 22 23 24
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
25
#include "calling.h"
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/smap.h>
37
#include <asm/pgtable_types.h>
38
#include <linux/err.h>
L
Linus Torvalds 已提交
39

R
Roland McGrath 已提交
40 41
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
42 43 44
#define AUDIT_ARCH_X86_64			(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT			0x80000000
#define __AUDIT_ARCH_LE				0x40000000
J
Jiri Olsa 已提交
45

46 47
.code64
.section .entry.text, "ax"
48

49
#ifdef CONFIG_PARAVIRT
50
ENTRY(native_usergs_sysret64)
51 52
	swapgs
	sysretq
53
ENDPROC(native_usergs_sysret64)
54 55
#endif /* CONFIG_PARAVIRT */

56
.macro TRACE_IRQS_IRETQ
57
#ifdef CONFIG_TRACE_IRQFLAGS
58 59
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
60 61 62 63 64
	TRACE_IRQS_ON
1:
#endif
.endm

65 66 67 68 69 70 71 72 73 74 75 76 77 78
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
79
	call	debug_stack_set_zero
80
	TRACE_IRQS_OFF
81
	call	debug_stack_reset
82 83 84
.endm

.macro TRACE_IRQS_ON_DEBUG
85
	call	debug_stack_set_zero
86
	TRACE_IRQS_ON
87
	call	debug_stack_reset
88 89
.endm

90
.macro TRACE_IRQS_IRETQ_DEBUG
91 92
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
93 94 95 96 97
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
98 99 100
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
101 102
#endif

L
Linus Torvalds 已提交
103
/*
104
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
105
 *
106
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
107 108 109 110 111 112
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
113
 * rax  system call number
114 115
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
116 117
 * rdi  arg0
 * rsi  arg1
118
 * rdx  arg2
119
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
120 121
 * r8   arg4
 * r9   arg5
122
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
123
 *
L
Linus Torvalds 已提交
124 125
 * Only called from user space.
 *
126
 * When user can change pt_regs->foo always force IRET. That is because
127 128
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
129
 */
L
Linus Torvalds 已提交
130

131
ENTRY(entry_SYSCALL_64)
132 133 134 135 136
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
137 138 139 140 141 142
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
143
GLOBAL(entry_SYSCALL_64_after_swapgs)
144

145 146
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
147 148

	/* Construct struct pt_regs on stack */
149 150
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
151
	/*
152 153 154 155 156
	 * Re-enable interrupts.
	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
	 * must execute atomically in the face of possible interrupt-driven
	 * task preemption. We must enable interrupts only after we're done
	 * with using rsp_scratch:
157 158
	 */
	ENABLE_INTERRUPTS(CLBR_NONE)
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */

	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	tracesys
176
entry_SYSCALL_64_fastpath:
177
#if __SYSCALL_MASK == ~0
178
	cmpq	$__NR_syscall_max, %rax
179
#else
180 181
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
182
#endif
183 184 185 186
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
187
1:
L
Linus Torvalds 已提交
188
/*
189 190
 * Syscall return path ending with SYSRET (fast path).
 * Has incompletely filled pt_regs.
191
 */
192
	LOCKDEP_SYS_EXIT
193 194 195 196
	/*
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
197
	DISABLE_INTERRUPTS(CLBR_NONE)
198 199 200 201 202 203 204 205 206

	/*
	 * We must check ti flags with interrupts (or at least preemption)
	 * off because we must *never* return to userspace without
	 * processing exit work that is enqueued if we're preempted here.
	 * In particular, returning to userspace with any of the one-shot
	 * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
	 * very bad.
	 */
207 208
	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	int_ret_from_sys_call_irqs_off	/* Go to the slow path */
209

210
	RESTORE_C_REGS_EXCEPT_RCX_R11
211 212 213
	movq	RIP(%rsp), %rcx
	movq	EFLAGS(%rsp), %r11
	movq	RSP(%rsp), %rsp
214
	/*
215
	 * 64-bit SYSRET restores rip from rcx,
216 217
	 * rflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
218
	 * Restoration of rflags re-enables interrupts.
219 220 221 222 223 224 225 226 227
	 *
	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
	 * descriptor is not reinitialized.  This means that we should
	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
	 * exit the kernel, and re-enter using an interrupt vector.  (All
	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
	 * from happening by reloading SS in __switch_to.  (Actually
	 * detecting the failure in 64-bit userspace is tricky but can be
	 * done.)
228
	 */
229
	USERGS_SYSRET64
L
Linus Torvalds 已提交
230

231 232 233 234 235
GLOBAL(int_ret_from_sys_call_irqs_off)
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
	jmp int_ret_from_sys_call

236
	/* Do syscall entry tracing */
237
tracesys:
238 239 240 241 242 243 244 245
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	call	syscall_trace_enter_phase1
	test	%rax, %rax
	jnz	tracesys_phase2			/* if needed, run the slow path */
	RESTORE_C_REGS_EXCEPT_RAX		/* else restore clobbered regs */
	movq	ORIG_RAX(%rsp), %rax
	jmp	entry_SYSCALL_64_fastpath	/* and return to the fast path */
246 247

tracesys_phase2:
248
	SAVE_EXTRA_REGS
249 250 251 252
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	movq	%rax, %rdx
	call	syscall_trace_enter_phase2
253

254
	/*
D
Denys Vlasenko 已提交
255
	 * Reload registers from stack in case ptrace changed them.
256
	 * We don't reload %rax because syscall_trace_entry_phase2() returned
257 258
	 * the value it wants us to use in the table lookup.
	 */
259 260
	RESTORE_C_REGS_EXCEPT_RAX
	RESTORE_EXTRA_REGS
261
#if __SYSCALL_MASK == ~0
262
	cmpq	$__NR_syscall_max, %rax
263
#else
264 265
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
266
#endif
267 268 269 270
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx			/* fixup for C */
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
271
1:
272
	/* Use IRET because user could have changed pt_regs->foo */
273 274

/*
L
Linus Torvalds 已提交
275
 * Syscall return path ending with IRET.
276
 * Has correct iret frame.
277
 */
278
GLOBAL(int_ret_from_sys_call)
279
	SAVE_EXTRA_REGS
280 281
	movq	%rsp, %rdi
	call	syscall_return_slowpath	/* returns with IRQs disabled */
282
	RESTORE_EXTRA_REGS
283
	TRACE_IRQS_IRETQ		/* we're about to change IF */
284 285 286 287 288

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
289 290 291 292
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
	cmpq	%rcx, %r11			/* RCX == RIP */
	jne	opportunistic_sysret_failed
293 294 295 296

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
297
	 * the kernel, since userspace controls RSP.
298
	 *
299
	 * If width of "canonical tail" ever becomes variable, this will need
300 301 302 303 304
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
305

306 307 308
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
309

310 311 312
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
313

314 315
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
	jne	opportunistic_sysret_failed
316

317 318 319
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne	opportunistic_sysret_failed
320 321 322 323 324 325 326 327

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
328
	 *           movq	$stuck_here, %rcx
329 330 331 332 333 334
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
335 336
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz	opportunistic_sysret_failed
337 338 339

	/* nothing to check for RSP */

340 341
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
	jne	opportunistic_sysret_failed
342 343

	/*
344 345
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
346 347
	 */
syscall_return_via_sysret:
348 349
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
350
	movq	RSP(%rsp), %rsp
351 352 353 354 355
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
356
END(entry_SYSCALL_64)
357

358

359 360
	.macro FORK_LIKE func
ENTRY(stub_\func)
361
	SAVE_EXTRA_REGS 8
362
	jmp	sys_\func
363 364 365 366 367 368
END(stub_\func)
	.endm

	FORK_LIKE  clone
	FORK_LIKE  fork
	FORK_LIKE  vfork
L
Linus Torvalds 已提交
369 370

ENTRY(stub_execve)
371 372 373 374 375 376 377 378 379 380
	call	sys_execve
return_from_execve:
	testl	%eax, %eax
	jz	1f
	/* exec failed, can use fast SYSRET code path in this case */
	ret
1:
	/* must use IRET code path (pt_regs->cs may have changed) */
	addq	$8, %rsp
	ZERO_EXTRA_REGS
381
	movq	%rax, RAX(%rsp)
382
	jmp	int_ret_from_sys_call
383
END(stub_execve)
384 385 386 387 388 389
/*
 * Remaining execve stubs are only 7 bytes long.
 * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
 */
	.align	8
GLOBAL(stub_execveat)
390 391
	call	sys_execveat
	jmp	return_from_execve
D
David Drysdale 已提交
392 393
END(stub_execveat)

394
#if defined(CONFIG_X86_X32_ABI)
395 396
	.align	8
GLOBAL(stub_x32_execve)
397 398 399
	call	compat_sys_execve
	jmp	return_from_execve
END(stub_x32_execve)
400 401
	.align	8
GLOBAL(stub_x32_execveat)
402 403
	call	compat_sys_execveat
	jmp	return_from_execve
404
END(stub_x32_execveat)
405 406
#endif

L
Linus Torvalds 已提交
407 408 409
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
410
 */
L
Linus Torvalds 已提交
411
ENTRY(stub_rt_sigreturn)
412 413 414 415 416 417 418 419
	/*
	 * SAVE_EXTRA_REGS result is not normally needed:
	 * sigreturn overwrites all pt_regs->GPREGS.
	 * But sigreturn can fail (!), and there is no easy way to detect that.
	 * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
	 * we SAVE_EXTRA_REGS here.
	 */
	SAVE_EXTRA_REGS 8
420
	call	sys_rt_sigreturn
421 422
return_from_stub:
	addq	$8, %rsp
423
	RESTORE_EXTRA_REGS
424 425
	movq	%rax, RAX(%rsp)
	jmp	int_ret_from_sys_call
426
END(stub_rt_sigreturn)
L
Linus Torvalds 已提交
427

428 429
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
430
	SAVE_EXTRA_REGS 8
431 432
	call	sys32_x32_rt_sigreturn
	jmp	return_from_stub
433 434 435
END(stub_x32_rt_sigreturn)
#endif

436 437 438 439 440 441 442
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)

443
	LOCK ; btr $TIF_FORK, TI_flags(%r8)
444

445 446
	pushq	$0x0002
	popfq					/* reset kernel eflags */
447

448
	call	schedule_tail			/* rdi: 'prev' task parameter */
449 450 451

	RESTORE_EXTRA_REGS

452
	testb	$3, CS(%rsp)			/* from kernel_thread? */
453

454 455 456
	/*
	 * By the time we get here, we have no idea whether our pt_regs,
	 * ti flags, and ti status came from the 64-bit SYSCALL fast path,
457
	 * the slow path, or one of the 32-bit compat paths.
458
	 * Use IRET code path to return, since it can safely handle
459 460
	 * all of the above.
	 */
461
	jnz	int_ret_from_sys_call
462

463 464 465 466 467 468 469
	/*
	 * We came from kernel_thread
	 * nb: we depend on RESTORE_EXTRA_REGS above
	 */
	movq	%rbp, %rdi
	call	*%rbx
	movl	$0, RAX(%rsp)
470
	RESTORE_EXTRA_REGS
471
	jmp	int_ret_from_sys_call
472 473
END(ret_from_fork)

474
/*
475 476
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
477
 */
478
	.align 8
479
ENTRY(irq_entries_start)
480 481
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
482
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
483 484 485 486
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
487 488
END(irq_entries_start)

489
/*
L
Linus Torvalds 已提交
490 491 492
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
493 494 495
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
496

497
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
498
	.macro interrupt func
499
	cld
500 501 502
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
503

504
	testb	$3, CS(%rsp)
505
	jz	1f
506 507 508 509 510

	/*
	 * IRQ from user mode.  Switch to kernel gsbase and inform context
	 * tracking that we're in kernel mode.
	 */
511
	SWAPGS
512 513 514 515
#ifdef CONFIG_CONTEXT_TRACKING
	call enter_from_user_mode
#endif

516
1:
517
	/*
D
Denys Vlasenko 已提交
518
	 * Save previous stack pointer, optionally switch to interrupt stack.
519 520 521 522 523
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
524
	movq	%rsp, %rdi
525 526
	incl	PER_CPU_VAR(irq_count)
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
527
	pushq	%rdi
528 529 530
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

531
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
532 533
	.endm

534 535 536 537
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
538 539
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
540
	ASM_CLAC
541
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
542
	interrupt do_IRQ
543
	/* 0(%rsp): old RSP */
544
ret_from_intr:
545
	DISABLE_INTERRUPTS(CLBR_NONE)
546
	TRACE_IRQS_OFF
547
	decl	PER_CPU_VAR(irq_count)
548

549
	/* Restore saved previous stack */
550
	popq	%rsp
551

552
	testb	$3, CS(%rsp)
553
	jz	retint_kernel
554

555 556 557 558
	/* Interrupt came from user space */
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
559
	TRACE_IRQS_IRETQ
560
	SWAPGS
561
	jmp	restore_regs_and_iret
562

563
/* Returning to kernel space */
564
retint_kernel:
565 566 567
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
568
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
569
	jnc	1f
570
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
571
	jnz	1f
572
	call	preempt_schedule_irq
573
	jmp	0b
574
1:
575
#endif
576 577 578 579
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
580 581 582 583 584

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
585
GLOBAL(restore_regs_and_iret)
586
	RESTORE_EXTRA_REGS
587
restore_c_regs_and_iret:
588 589
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
590 591 592
	INTERRUPT_RETURN

ENTRY(native_iret)
593 594 595 596
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
597
#ifdef CONFIG_X86_ESPFIX64
598 599
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
600
#endif
601

602
.global native_irq_return_iret
603
native_irq_return_iret:
A
Andy Lutomirski 已提交
604 605 606 607 608 609
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
610
	iretq
I
Ingo Molnar 已提交
611

612
#ifdef CONFIG_X86_ESPFIX64
613
native_irq_return_ldt:
614 615
	pushq	%rax
	pushq	%rdi
616
	SWAPGS
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
	movq	PER_CPU_VAR(espfix_waddr), %rdi
	movq	%rax, (0*8)(%rdi)		/* RAX */
	movq	(2*8)(%rsp), %rax		/* RIP */
	movq	%rax, (1*8)(%rdi)
	movq	(3*8)(%rsp), %rax		/* CS */
	movq	%rax, (2*8)(%rdi)
	movq	(4*8)(%rsp), %rax		/* RFLAGS */
	movq	%rax, (3*8)(%rdi)
	movq	(6*8)(%rsp), %rax		/* SS */
	movq	%rax, (5*8)(%rdi)
	movq	(5*8)(%rsp), %rax		/* RSP */
	movq	%rax, (4*8)(%rdi)
	andl	$0xffff0000, %eax
	popq	%rdi
	orq	PER_CPU_VAR(espfix_stack), %rax
632
	SWAPGS
633 634 635
	movq	%rax, %rsp
	popq	%rax
	jmp	native_irq_return_iret
636
#endif
637
END(common_interrupt)
638

L
Linus Torvalds 已提交
639 640
/*
 * APIC interrupts.
641
 */
642
.macro apicinterrupt3 num sym do_sym
643
ENTRY(\sym)
644
	ASM_CLAC
645
	pushq	$~(\num)
646
.Lcommon_\sym:
647
	interrupt \do_sym
648
	jmp	ret_from_intr
649 650
END(\sym)
.endm
L
Linus Torvalds 已提交
651

652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

669
#ifdef CONFIG_SMP
670 671
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
672
#endif
L
Linus Torvalds 已提交
673

N
Nick Piggin 已提交
674
#ifdef CONFIG_X86_UV
675
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
676
#endif
677 678 679

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
680

681
#ifdef CONFIG_HAVE_KVM
682 683
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
684 685
#endif

686
#ifdef CONFIG_X86_MCE_THRESHOLD
687
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
688 689
#endif

690
#ifdef CONFIG_X86_MCE_AMD
691
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
692 693
#endif

694
#ifdef CONFIG_X86_THERMAL_VECTOR
695
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
696
#endif
697

698
#ifdef CONFIG_SMP
699 700 701
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
702
#endif
L
Linus Torvalds 已提交
703

704 705
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
706

707
#ifdef CONFIG_IRQ_WORK
708
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
709 710
#endif

L
Linus Torvalds 已提交
711 712
/*
 * Exception entry points.
713
 */
714
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
715 716

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
717
ENTRY(\sym)
718 719 720 721 722
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

723
	ASM_CLAC
724
	PARAVIRT_ADJUST_EXCEPTION_FRAME
725 726

	.ifeq \has_error_code
727
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
728 729
	.endif

730
	ALLOC_PT_GPREGS_ON_STACK
731 732

	.if \paranoid
733
	.if \paranoid == 1
734 735
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
	jnz	1f
736
	.endif
737
	call	paranoid_entry
738
	.else
739
	call	error_entry
740
	.endif
741
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
742 743

	.if \paranoid
744
	.if \shift_ist != -1
745
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
746
	.else
747
	TRACE_IRQS_OFF
748
	.endif
749
	.endif
750

751
	movq	%rsp, %rdi			/* pt_regs pointer */
752 753

	.if \has_error_code
754 755
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
756
	.else
757
	xorl	%esi, %esi			/* no error code */
758 759
	.endif

760
	.if \shift_ist != -1
761
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
762 763
	.endif

764
	call	\do_sym
765

766
	.if \shift_ist != -1
767
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
768 769
	.endif

770
	/* these procedures expect "no swapgs" flag in ebx */
771
	.if \paranoid
772
	jmp	paranoid_exit
773
	.else
774
	jmp	error_exit
775 776
	.endif

777 778 779 780 781 782 783
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
784
	call	error_entry
785 786


787 788 789
	movq	%rsp, %rdi			/* pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
790

791
	movq	%rsp, %rdi			/* pt_regs pointer */
792 793

	.if \has_error_code
794 795
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
796
	.else
797
	xorl	%esi, %esi			/* no error code */
798 799
	.endif

800
	call	\do_sym
801

802
	jmp	error_exit			/* %ebx: no swapgs flag */
803
	.endif
804
END(\sym)
805
.endm
806

807
#ifdef CONFIG_TRACING
808 809 810
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
811 812
.endm
#else
813 814
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
815 816 817
.endm
#endif

818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
837
ENTRY(native_load_gs_index)
838
	pushfq
839
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
840
	SWAPGS
841
gs_change:
842 843
	movl	%edi, %gs
2:	mfence					/* workaround */
844
	SWAPGS
845
	popfq
846
	ret
847
END(native_load_gs_index)
848

849 850
	_ASM_EXTABLE(gs_change, bad_gs)
	.section .fixup, "ax"
L
Linus Torvalds 已提交
851
	/* running with kernelgs */
852
bad_gs:
853 854 855 856
	SWAPGS					/* switch back to user gs */
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
857
	.previous
858

859
/* Call softirq on interrupt stack. Interrupts are off. */
860
ENTRY(do_softirq_own_stack)
861 862 863 864 865 866
	pushq	%rbp
	mov	%rsp, %rbp
	incl	PER_CPU_VAR(irq_count)
	cmove	PER_CPU_VAR(irq_stack_ptr), %rsp
	push	%rbp				/* frame pointer backlink */
	call	__do_softirq
867
	leaveq
868
	decl	PER_CPU_VAR(irq_count)
869
	ret
870
END(do_softirq_own_stack)
871

872
#ifdef CONFIG_XEN
873
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
874 875

/*
876 877 878 879 880 881 882 883 884 885 886 887
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
888 889
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

890 891 892 893
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
894 895 896 897 898 899 900 901
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
11:	incl	PER_CPU_VAR(irq_count)
	movq	%rsp, %rbp
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
	pushq	%rbp				/* frame pointer backlink */
	call	xen_evtchn_do_upcall
	popq	%rsp
	decl	PER_CPU_VAR(irq_count)
902
#ifndef CONFIG_PREEMPT
903
	call	xen_maybe_preempt_hcall
904
#endif
905
	jmp	error_exit
906
END(xen_do_hypervisor_callback)
907 908

/*
909 910 911 912 913 914 915 916 917 918 919 920
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
921
ENTRY(xen_failsafe_callback)
922 923 924 925 926 927 928 929 930 931 932 933
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
934
	/* All segments match their saved values => Category 2 (Bad IRET). */
935 936 937 938 939 940 941
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
	pushq	%r11
	pushq	%rcx
	jmp	general_protection
942
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
943 944 945 946
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$-1 /* orig_ax = -1 => not a system call */
947 948 949
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
950
	jmp	error_exit
951 952
END(xen_failsafe_callback)

953
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
954 955
	xen_hvm_callback_vector xen_evtchn_do_upcall

956
#endif /* CONFIG_XEN */
957

958
#if IS_ENABLED(CONFIG_HYPERV)
959
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
960 961 962
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

963 964 965 966
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

967
#ifdef CONFIG_XEN
968 969 970
idtentry xen_debug		do_debug		has_error_code=0
idtentry xen_int3		do_int3			has_error_code=0
idtentry xen_stack_segment	do_stack_segment	has_error_code=1
971
#endif
972 973 974 975

idtentry general_protection	do_general_protection	has_error_code=1
trace_idtentry page_fault	do_page_fault		has_error_code=1

G
Gleb Natapov 已提交
976
#ifdef CONFIG_KVM_GUEST
977
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
978
#endif
979

980
#ifdef CONFIG_X86_MCE
981
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
982 983
#endif

984 985 986 987 988 989
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
990 991 992
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
993 994
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
995
	rdmsr
996 997
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
998
	SWAPGS
999
	xorl	%ebx, %ebx
1000
1:	ret
1001
END(paranoid_entry)
1002

1003 1004 1005 1006 1007 1008 1009 1010 1011
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1012 1013
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1014
 */
1015 1016
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
1017
	TRACE_IRQS_OFF_DEBUG
1018 1019
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	paranoid_exit_no_swapgs
1020
	TRACE_IRQS_IRETQ
1021
	SWAPGS_UNSAFE_STACK
1022
	jmp	paranoid_exit_restore
1023
paranoid_exit_no_swapgs:
1024
	TRACE_IRQS_IRETQ_DEBUG
1025
paranoid_exit_restore:
1026 1027 1028
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
1029
	INTERRUPT_RETURN
1030 1031 1032
END(paranoid_exit)

/*
1033
 * Save all registers in pt_regs, and switch gs if needed.
1034
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1035 1036 1037
 */
ENTRY(error_entry)
	cld
1038 1039
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1040
	xorl	%ebx, %ebx
1041
	testb	$3, CS+8(%rsp)
1042
	jz	.Lerror_kernelspace
1043

1044 1045 1046 1047 1048
.Lerror_entry_from_usermode_swapgs:
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
1049
	SWAPGS
1050

1051
.Lerror_entry_from_usermode_after_swapgs:
1052 1053 1054 1055
#ifdef CONFIG_CONTEXT_TRACKING
	call enter_from_user_mode
#endif

1056
.Lerror_entry_done:
1057

1058 1059 1060
	TRACE_IRQS_OFF
	ret

1061 1062 1063 1064 1065 1066
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1067
.Lerror_kernelspace:
1068 1069 1070
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1071
	je	.Lerror_bad_iret
1072 1073
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1074
	je	.Lbstep_iret
1075
	cmpq	$gs_change, RIP+8(%rsp)
1076
	jne	.Lerror_entry_done
1077 1078 1079 1080 1081 1082

	/*
	 * hack: gs_change can fail with user gsbase.  If this happens, fix up
	 * gsbase and proceed.  We'll fix up the exception and land in
	 * gs_change's error handler with kernel gsbase.
	 */
1083
	jmp	.Lerror_entry_from_usermode_swapgs
1084

1085
.Lbstep_iret:
1086
	/* Fix truncated RIP */
1087
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1088 1089
	/* fall through */

1090
.Lerror_bad_iret:
1091 1092 1093 1094
	/*
	 * We came from an IRET to user mode, so we have user gsbase.
	 * Switch to kernel gsbase:
	 */
A
Andy Lutomirski 已提交
1095
	SWAPGS
1096 1097 1098 1099 1100 1101

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1102 1103 1104
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1105
	decl	%ebx
1106
	jmp	.Lerror_entry_from_usermode_after_swapgs
1107 1108 1109
END(error_entry)


1110 1111 1112 1113 1114
/*
 * On entry, EBS is a "return to kernel mode" flag:
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1115
ENTRY(error_exit)
1116
	movl	%ebx, %eax
1117 1118
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
1119 1120 1121
	testl	%eax, %eax
	jnz	retint_kernel
	jmp	retint_user
1122 1123
END(error_exit)

1124
/* Runs on exception stack */
1125
ENTRY(nmi)
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
	/*
	 * Fix up the exception frame if we're on Xen.
	 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
	 * one value to the stack on native, so it may clobber the rdx
	 * scratch slot, but it won't clobber any of the important
	 * slots past it.
	 *
	 * Xen is a different story, because the Xen frame itself overlaps
	 * the "NMI executing" variable.
	 */
1136
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1137

1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1155 1156 1157
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1158 1159
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1160
	 *    o Modify the "iret" location to jump to the repeat_nmi
1161 1162 1163 1164 1165 1166 1167 1168
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1169 1170 1171 1172 1173
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1174 1175
	 */

1176
	/* Use %rdx as our temp variable throughout */
1177
	pushq	%rdx
1178

1179 1180 1181 1182 1183 1184 1185 1186 1187
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
1188 1189 1190
	 *
	 * We also must not push anything to the stack before switching
	 * stacks lest we corrupt the "NMI executing" variable.
1191 1192
	 */

1193
	SWAPGS_UNSAFE_STACK
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
	cld
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1229
	/*
1230 1231 1232
	 * Return back to user mode.  We must *not* do the normal exit
	 * work, because we don't want to enable interrupts.  Fortunately,
	 * do_nmi doesn't modify pt_regs.
1233
	 */
1234 1235
	SWAPGS
	jmp	restore_c_regs_and_iret
1236

1237
.Lnmi_from_kernel:
1238
	/*
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1279
	/*
1280 1281
	 * Determine whether we're a nested NMI.
	 *
1282 1283 1284 1285 1286 1287
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1288
	 */
1289 1290 1291 1292 1293 1294 1295 1296

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1297

1298
	/*
1299
	 * Now check "NMI executing".  If it's set, then we're nested.
1300 1301
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1302
	 */
1303 1304
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1305 1306

	/*
1307 1308
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1309 1310 1311 1312 1313 1314 1315 1316
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1317
	 */
1318 1319 1320 1321 1322
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1323

1324 1325 1326 1327
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1328 1329 1330 1331 1332 1333 1334

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1335

1336 1337
nested_nmi:
	/*
1338 1339
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1340
	 */
1341
	subq	$8, %rsp
1342 1343 1344
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1345
	pushfq
1346 1347
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1348 1349

	/* Put stack back */
1350
	addq	$(6*8), %rsp
1351 1352

nested_nmi_out:
1353
	popq	%rdx
1354

1355
	/* We are returning to kernel mode, so this cannot result in a fault. */
1356 1357 1358
	INTERRUPT_RETURN

first_nmi:
1359
	/* Restore rdx. */
1360
	movq	(%rsp), %rdx
1361

1362 1363
	/* Make room for "NMI executing". */
	pushq	$0
1364

1365
	/* Leave room for the "iret" frame */
1366
	subq	$(5*8), %rsp
1367

1368
	/* Copy the "original" frame to the "outermost" frame */
1369
	.rept 5
1370
	pushq	11*8(%rsp)
1371
	.endr
1372

1373 1374
	/* Everything up to here is safe from nested NMIs */

1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
	INTERRUPT_RETURN	/* continues at repeat_nmi below */
1:
#endif

1390
repeat_nmi:
1391 1392 1393 1394 1395 1396 1397 1398
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1399 1400 1401 1402
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1403 1404
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1405
	 */
1406
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1407

1408
	/*
1409 1410 1411
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1412
	 */
1413
	addq	$(10*8), %rsp
1414
	.rept 5
1415
	pushq	-6*8(%rsp)
1416
	.endr
1417
	subq	$(5*8), %rsp
1418
end_repeat_nmi:
1419 1420

	/*
1421 1422 1423
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1424
	 */
1425
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1426 1427
	ALLOC_PT_GPREGS_ON_STACK

1428
	/*
1429
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1430 1431 1432 1433 1434
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1435
	call	paranoid_entry
1436

1437
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1438 1439 1440
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1441

1442 1443
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1444 1445 1446
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1447 1448
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1449 1450

	/* Point RSP at the "iret" frame. */
1451
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1452

1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
	 * the SYSCALL entry and exit paths.  On a native kernel, we
	 * could just inspect RIP, but, on paravirt kernels,
	 * INTERRUPT_RETURN can translate into a jump into a
	 * hypercall page.
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1463 1464 1465 1466 1467 1468

	/*
	 * INTERRUPT_RETURN reads the "iret" frame and exits the NMI
	 * stack in a single instruction.  We are returning to kernel
	 * mode, so this cannot result in a fault.
	 */
1469
	INTERRUPT_RETURN
1470 1471 1472
END(nmi)

ENTRY(ignore_sysret)
1473
	mov	$-ENOSYS, %eax
1474 1475
	sysret
END(ignore_sysret)