entry_64.S 40.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
7
 *
L
Linus Torvalds 已提交
8 9
 * entry.S contains the system-call and fault low-level handling routines.
 *
10 11
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
12
 * A note on terminology:
13 14
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
15 16
 *
 * Some macro usage:
17 18 19
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
20 21 22 23 24
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
25
#include "calling.h"
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/smap.h>
37
#include <asm/pgtable_types.h>
38
#include <linux/err.h>
L
Linus Torvalds 已提交
39

R
Roland McGrath 已提交
40 41
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
42 43 44
#define AUDIT_ARCH_X86_64			(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT			0x80000000
#define __AUDIT_ARCH_LE				0x40000000
J
Jiri Olsa 已提交
45

46 47
.code64
.section .entry.text, "ax"
48

49
#ifdef CONFIG_PARAVIRT
50
ENTRY(native_usergs_sysret64)
51 52
	swapgs
	sysretq
53
ENDPROC(native_usergs_sysret64)
54 55
#endif /* CONFIG_PARAVIRT */

56
.macro TRACE_IRQS_IRETQ
57
#ifdef CONFIG_TRACE_IRQFLAGS
58 59
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
60 61 62 63 64
	TRACE_IRQS_ON
1:
#endif
.endm

65 66 67 68 69 70 71 72 73 74 75 76 77 78
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
79
	call	debug_stack_set_zero
80
	TRACE_IRQS_OFF
81
	call	debug_stack_reset
82 83 84
.endm

.macro TRACE_IRQS_ON_DEBUG
85
	call	debug_stack_set_zero
86
	TRACE_IRQS_ON
87
	call	debug_stack_reset
88 89
.endm

90
.macro TRACE_IRQS_IRETQ_DEBUG
91 92
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
93 94 95 96 97
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
98 99 100
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
101 102
#endif

L
Linus Torvalds 已提交
103
/*
104
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
105
 *
106
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
107 108 109 110 111 112
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
113
 * rax  system call number
114 115
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
116 117
 * rdi  arg0
 * rsi  arg1
118
 * rdx  arg2
119
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
120 121
 * r8   arg4
 * r9   arg5
122
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
123
 *
L
Linus Torvalds 已提交
124 125
 * Only called from user space.
 *
126
 * When user can change pt_regs->foo always force IRET. That is because
127 128
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
129
 */
L
Linus Torvalds 已提交
130

131
ENTRY(entry_SYSCALL_64)
132 133 134 135 136
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
137 138 139 140 141 142
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
143
GLOBAL(entry_SYSCALL_64_after_swapgs)
144

145 146
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
147 148

	/* Construct struct pt_regs on stack */
149 150
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
151
	/*
152 153 154 155 156
	 * Re-enable interrupts.
	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
	 * must execute atomically in the face of possible interrupt-driven
	 * task preemption. We must enable interrupts only after we're done
	 * with using rsp_scratch:
157 158
	 */
	ENABLE_INTERRUPTS(CLBR_NONE)
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */

	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	tracesys
176
entry_SYSCALL_64_fastpath:
177
#if __SYSCALL_MASK == ~0
178
	cmpq	$__NR_syscall_max, %rax
179
#else
180 181
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
182
#endif
183 184 185 186
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
187
1:
L
Linus Torvalds 已提交
188
/*
189 190
 * Syscall return path ending with SYSRET (fast path).
 * Has incompletely filled pt_regs.
191
 */
192
	LOCKDEP_SYS_EXIT
193 194 195 196
	/*
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
197
	DISABLE_INTERRUPTS(CLBR_NONE)
198 199 200 201 202 203 204 205 206

	/*
	 * We must check ti flags with interrupts (or at least preemption)
	 * off because we must *never* return to userspace without
	 * processing exit work that is enqueued if we're preempted here.
	 * In particular, returning to userspace with any of the one-shot
	 * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
	 * very bad.
	 */
207 208
	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	int_ret_from_sys_call_irqs_off	/* Go to the slow path */
209

210
	RESTORE_C_REGS_EXCEPT_RCX_R11
211 212 213
	movq	RIP(%rsp), %rcx
	movq	EFLAGS(%rsp), %r11
	movq	RSP(%rsp), %rsp
214
	/*
215
	 * 64-bit SYSRET restores rip from rcx,
216 217
	 * rflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
218
	 * Restoration of rflags re-enables interrupts.
219 220 221 222 223 224 225 226 227
	 *
	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
	 * descriptor is not reinitialized.  This means that we should
	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
	 * exit the kernel, and re-enter using an interrupt vector.  (All
	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
	 * from happening by reloading SS in __switch_to.  (Actually
	 * detecting the failure in 64-bit userspace is tricky but can be
	 * done.)
228
	 */
229
	USERGS_SYSRET64
L
Linus Torvalds 已提交
230

231 232 233 234 235
GLOBAL(int_ret_from_sys_call_irqs_off)
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
	jmp int_ret_from_sys_call

236
	/* Do syscall entry tracing */
237
tracesys:
238 239 240 241 242 243 244 245
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	call	syscall_trace_enter_phase1
	test	%rax, %rax
	jnz	tracesys_phase2			/* if needed, run the slow path */
	RESTORE_C_REGS_EXCEPT_RAX		/* else restore clobbered regs */
	movq	ORIG_RAX(%rsp), %rax
	jmp	entry_SYSCALL_64_fastpath	/* and return to the fast path */
246 247

tracesys_phase2:
248
	SAVE_EXTRA_REGS
249 250 251 252
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	movq	%rax, %rdx
	call	syscall_trace_enter_phase2
253

254
	/*
D
Denys Vlasenko 已提交
255
	 * Reload registers from stack in case ptrace changed them.
256
	 * We don't reload %rax because syscall_trace_entry_phase2() returned
257 258
	 * the value it wants us to use in the table lookup.
	 */
259 260
	RESTORE_C_REGS_EXCEPT_RAX
	RESTORE_EXTRA_REGS
261
#if __SYSCALL_MASK == ~0
262
	cmpq	$__NR_syscall_max, %rax
263
#else
264 265
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
266
#endif
267 268 269 270
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx			/* fixup for C */
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
271
1:
272
	/* Use IRET because user could have changed pt_regs->foo */
273 274

/*
L
Linus Torvalds 已提交
275
 * Syscall return path ending with IRET.
276
 * Has correct iret frame.
277
 */
278
GLOBAL(int_ret_from_sys_call)
279
	SAVE_EXTRA_REGS
280 281
	movq	%rsp, %rdi
	call	syscall_return_slowpath	/* returns with IRQs disabled */
282
	RESTORE_EXTRA_REGS
283
	TRACE_IRQS_IRETQ		/* we're about to change IF */
284 285 286 287 288

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
289 290 291 292
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
	cmpq	%rcx, %r11			/* RCX == RIP */
	jne	opportunistic_sysret_failed
293 294 295 296

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
297
	 * the kernel, since userspace controls RSP.
298
	 *
299
	 * If width of "canonical tail" ever becomes variable, this will need
300 301 302 303 304
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
305

306 307 308
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
309

310 311 312
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
313

314 315
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
	jne	opportunistic_sysret_failed
316

317 318 319
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne	opportunistic_sysret_failed
320 321 322 323 324 325 326 327

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
328
	 *           movq	$stuck_here, %rcx
329 330 331 332 333 334
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
335 336
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz	opportunistic_sysret_failed
337 338 339

	/* nothing to check for RSP */

340 341
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
	jne	opportunistic_sysret_failed
342 343

	/*
344 345
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
346 347
	 */
syscall_return_via_sysret:
348 349
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
350
	movq	RSP(%rsp), %rsp
351 352 353 354 355
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
356
END(entry_SYSCALL_64)
357

358

359 360
	.macro FORK_LIKE func
ENTRY(stub_\func)
361
	SAVE_EXTRA_REGS 8
362
	jmp	sys_\func
363 364 365 366 367 368
END(stub_\func)
	.endm

	FORK_LIKE  clone
	FORK_LIKE  fork
	FORK_LIKE  vfork
L
Linus Torvalds 已提交
369 370

ENTRY(stub_execve)
371 372 373 374 375 376 377 378 379 380
	call	sys_execve
return_from_execve:
	testl	%eax, %eax
	jz	1f
	/* exec failed, can use fast SYSRET code path in this case */
	ret
1:
	/* must use IRET code path (pt_regs->cs may have changed) */
	addq	$8, %rsp
	ZERO_EXTRA_REGS
381
	movq	%rax, RAX(%rsp)
382
	jmp	int_ret_from_sys_call
383
END(stub_execve)
384 385 386 387 388 389
/*
 * Remaining execve stubs are only 7 bytes long.
 * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
 */
	.align	8
GLOBAL(stub_execveat)
390 391
	call	sys_execveat
	jmp	return_from_execve
D
David Drysdale 已提交
392 393
END(stub_execveat)

394
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
395 396
	.align	8
GLOBAL(stub_x32_execve)
397
GLOBAL(stub32_execve)
398 399
	call	compat_sys_execve
	jmp	return_from_execve
400
END(stub32_execve)
401
END(stub_x32_execve)
402 403 404
	.align	8
GLOBAL(stub_x32_execveat)
GLOBAL(stub32_execveat)
405 406 407
	call	compat_sys_execveat
	jmp	return_from_execve
END(stub32_execveat)
408
END(stub_x32_execveat)
409 410
#endif

L
Linus Torvalds 已提交
411 412 413
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
414
 */
L
Linus Torvalds 已提交
415
ENTRY(stub_rt_sigreturn)
416 417 418 419 420 421 422 423
	/*
	 * SAVE_EXTRA_REGS result is not normally needed:
	 * sigreturn overwrites all pt_regs->GPREGS.
	 * But sigreturn can fail (!), and there is no easy way to detect that.
	 * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
	 * we SAVE_EXTRA_REGS here.
	 */
	SAVE_EXTRA_REGS 8
424
	call	sys_rt_sigreturn
425 426
return_from_stub:
	addq	$8, %rsp
427
	RESTORE_EXTRA_REGS
428 429
	movq	%rax, RAX(%rsp)
	jmp	int_ret_from_sys_call
430
END(stub_rt_sigreturn)
L
Linus Torvalds 已提交
431

432 433
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
434
	SAVE_EXTRA_REGS 8
435 436
	call	sys32_x32_rt_sigreturn
	jmp	return_from_stub
437 438 439
END(stub_x32_rt_sigreturn)
#endif

440 441 442 443 444 445 446
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)

447
	LOCK ; btr $TIF_FORK, TI_flags(%r8)
448

449 450
	pushq	$0x0002
	popfq					/* reset kernel eflags */
451

452
	call	schedule_tail			/* rdi: 'prev' task parameter */
453 454 455

	RESTORE_EXTRA_REGS

456
	testb	$3, CS(%rsp)			/* from kernel_thread? */
457

458 459 460
	/*
	 * By the time we get here, we have no idea whether our pt_regs,
	 * ti flags, and ti status came from the 64-bit SYSCALL fast path,
461
	 * the slow path, or one of the 32-bit compat paths.
462
	 * Use IRET code path to return, since it can safely handle
463 464
	 * all of the above.
	 */
465
	jnz	int_ret_from_sys_call
466

467 468 469 470 471 472 473
	/*
	 * We came from kernel_thread
	 * nb: we depend on RESTORE_EXTRA_REGS above
	 */
	movq	%rbp, %rdi
	call	*%rbx
	movl	$0, RAX(%rsp)
474
	RESTORE_EXTRA_REGS
475
	jmp	int_ret_from_sys_call
476 477
END(ret_from_fork)

478
/*
479 480
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
481
 */
482
	.align 8
483
ENTRY(irq_entries_start)
484 485
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
486
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
487 488 489 490
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
491 492
END(irq_entries_start)

493
/*
L
Linus Torvalds 已提交
494 495 496
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
497 498 499
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
500

501
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
502
	.macro interrupt func
503
	cld
504 505 506
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
507

508
	testb	$3, CS(%rsp)
509
	jz	1f
510 511 512 513 514

	/*
	 * IRQ from user mode.  Switch to kernel gsbase and inform context
	 * tracking that we're in kernel mode.
	 */
515
	SWAPGS
516 517 518 519
#ifdef CONFIG_CONTEXT_TRACKING
	call enter_from_user_mode
#endif

520
1:
521
	/*
D
Denys Vlasenko 已提交
522
	 * Save previous stack pointer, optionally switch to interrupt stack.
523 524 525 526 527
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
528
	movq	%rsp, %rdi
529 530
	incl	PER_CPU_VAR(irq_count)
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
531
	pushq	%rdi
532 533 534
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

535
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
536 537
	.endm

538 539 540 541
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
542 543
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
544
	ASM_CLAC
545
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
546
	interrupt do_IRQ
547
	/* 0(%rsp): old RSP */
548
ret_from_intr:
549
	DISABLE_INTERRUPTS(CLBR_NONE)
550
	TRACE_IRQS_OFF
551
	decl	PER_CPU_VAR(irq_count)
552

553
	/* Restore saved previous stack */
554
	popq	%rsp
555

556
	testb	$3, CS(%rsp)
557
	jz	retint_kernel
558

559
	/* Interrupt came from user space */
560
	LOCKDEP_SYS_EXIT_IRQ
561 562 563
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
564
	TRACE_IRQS_IRETQ
565
	SWAPGS
566
	jmp	restore_regs_and_iret
567

568
/* Returning to kernel space */
569
retint_kernel:
570 571 572
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
573
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
574
	jnc	1f
575
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
576
	jnz	1f
577
	call	preempt_schedule_irq
578
	jmp	0b
579
1:
580
#endif
581 582 583 584
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
585 586 587 588 589

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
590 591
restore_regs_and_iret:
	RESTORE_EXTRA_REGS
592
restore_c_regs_and_iret:
593 594
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
595 596 597
	INTERRUPT_RETURN

ENTRY(native_iret)
598 599 600 601
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
602
#ifdef CONFIG_X86_ESPFIX64
603 604
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
605
#endif
606

607
.global native_irq_return_iret
608
native_irq_return_iret:
A
Andy Lutomirski 已提交
609 610 611 612 613 614
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
615
	iretq
I
Ingo Molnar 已提交
616

617
#ifdef CONFIG_X86_ESPFIX64
618
native_irq_return_ldt:
619 620
	pushq	%rax
	pushq	%rdi
621
	SWAPGS
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
	movq	PER_CPU_VAR(espfix_waddr), %rdi
	movq	%rax, (0*8)(%rdi)		/* RAX */
	movq	(2*8)(%rsp), %rax		/* RIP */
	movq	%rax, (1*8)(%rdi)
	movq	(3*8)(%rsp), %rax		/* CS */
	movq	%rax, (2*8)(%rdi)
	movq	(4*8)(%rsp), %rax		/* RFLAGS */
	movq	%rax, (3*8)(%rdi)
	movq	(6*8)(%rsp), %rax		/* SS */
	movq	%rax, (5*8)(%rdi)
	movq	(5*8)(%rsp), %rax		/* RSP */
	movq	%rax, (4*8)(%rdi)
	andl	$0xffff0000, %eax
	popq	%rdi
	orq	PER_CPU_VAR(espfix_stack), %rax
637
	SWAPGS
638 639 640
	movq	%rax, %rsp
	popq	%rax
	jmp	native_irq_return_iret
641
#endif
642
END(common_interrupt)
643

L
Linus Torvalds 已提交
644 645
/*
 * APIC interrupts.
646
 */
647
.macro apicinterrupt3 num sym do_sym
648
ENTRY(\sym)
649
	ASM_CLAC
650
	pushq	$~(\num)
651
.Lcommon_\sym:
652
	interrupt \do_sym
653
	jmp	ret_from_intr
654 655
END(\sym)
.endm
L
Linus Torvalds 已提交
656

657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

674
#ifdef CONFIG_SMP
675 676
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
677
#endif
L
Linus Torvalds 已提交
678

N
Nick Piggin 已提交
679
#ifdef CONFIG_X86_UV
680
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
681
#endif
682 683 684

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
685

686
#ifdef CONFIG_HAVE_KVM
687 688
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
689 690
#endif

691
#ifdef CONFIG_X86_MCE_THRESHOLD
692
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
693 694
#endif

695
#ifdef CONFIG_X86_MCE_AMD
696
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
697 698
#endif

699
#ifdef CONFIG_X86_THERMAL_VECTOR
700
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
701
#endif
702

703
#ifdef CONFIG_SMP
704 705 706
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
707
#endif
L
Linus Torvalds 已提交
708

709 710
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
711

712
#ifdef CONFIG_IRQ_WORK
713
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
714 715
#endif

L
Linus Torvalds 已提交
716 717
/*
 * Exception entry points.
718
 */
719
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
720 721

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
722
ENTRY(\sym)
723 724 725 726 727
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

728
	ASM_CLAC
729
	PARAVIRT_ADJUST_EXCEPTION_FRAME
730 731

	.ifeq \has_error_code
732
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
733 734
	.endif

735
	ALLOC_PT_GPREGS_ON_STACK
736 737

	.if \paranoid
738
	.if \paranoid == 1
739 740
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
	jnz	1f
741
	.endif
742
	call	paranoid_entry
743
	.else
744
	call	error_entry
745
	.endif
746
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
747 748

	.if \paranoid
749
	.if \shift_ist != -1
750
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
751
	.else
752
	TRACE_IRQS_OFF
753
	.endif
754
	.endif
755

756
	movq	%rsp, %rdi			/* pt_regs pointer */
757 758

	.if \has_error_code
759 760
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
761
	.else
762
	xorl	%esi, %esi			/* no error code */
763 764
	.endif

765
	.if \shift_ist != -1
766
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
767 768
	.endif

769
	call	\do_sym
770

771
	.if \shift_ist != -1
772
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
773 774
	.endif

775
	/* these procedures expect "no swapgs" flag in ebx */
776
	.if \paranoid
777
	jmp	paranoid_exit
778
	.else
779
	jmp	error_exit
780 781
	.endif

782 783 784 785 786 787 788
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
789
	call	error_entry
790 791


792 793 794
	movq	%rsp, %rdi			/* pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
795

796
	movq	%rsp, %rdi			/* pt_regs pointer */
797 798

	.if \has_error_code
799 800
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
801
	.else
802
	xorl	%esi, %esi			/* no error code */
803 804
	.endif

805
	call	\do_sym
806

807
	jmp	error_exit			/* %ebx: no swapgs flag */
808
	.endif
809
END(\sym)
810
.endm
811

812
#ifdef CONFIG_TRACING
813 814 815
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
816 817
.endm
#else
818 819
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
820 821 822
.endm
#endif

823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
842
ENTRY(native_load_gs_index)
843
	pushfq
844
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
845
	SWAPGS
846
gs_change:
847 848
	movl	%edi, %gs
2:	mfence					/* workaround */
849
	SWAPGS
850
	popfq
851
	ret
852
END(native_load_gs_index)
853

854 855
	_ASM_EXTABLE(gs_change, bad_gs)
	.section .fixup, "ax"
L
Linus Torvalds 已提交
856
	/* running with kernelgs */
857
bad_gs:
858 859 860 861
	SWAPGS					/* switch back to user gs */
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
862
	.previous
863

864
/* Call softirq on interrupt stack. Interrupts are off. */
865
ENTRY(do_softirq_own_stack)
866 867 868 869 870 871
	pushq	%rbp
	mov	%rsp, %rbp
	incl	PER_CPU_VAR(irq_count)
	cmove	PER_CPU_VAR(irq_stack_ptr), %rsp
	push	%rbp				/* frame pointer backlink */
	call	__do_softirq
872
	leaveq
873
	decl	PER_CPU_VAR(irq_count)
874
	ret
875
END(do_softirq_own_stack)
876

877
#ifdef CONFIG_XEN
878
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
879 880

/*
881 882 883 884 885 886 887 888 889 890 891 892
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
893 894
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

895 896 897 898
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
899 900 901 902 903 904 905 906
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
11:	incl	PER_CPU_VAR(irq_count)
	movq	%rsp, %rbp
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
	pushq	%rbp				/* frame pointer backlink */
	call	xen_evtchn_do_upcall
	popq	%rsp
	decl	PER_CPU_VAR(irq_count)
907
#ifndef CONFIG_PREEMPT
908
	call	xen_maybe_preempt_hcall
909
#endif
910
	jmp	error_exit
911
END(xen_do_hypervisor_callback)
912 913

/*
914 915 916 917 918 919 920 921 922 923 924 925
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
926
ENTRY(xen_failsafe_callback)
927 928 929 930 931 932 933 934 935 936 937 938
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
939
	/* All segments match their saved values => Category 2 (Bad IRET). */
940 941 942 943 944 945 946
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
	pushq	%r11
	pushq	%rcx
	jmp	general_protection
947
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
948 949 950 951
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$-1 /* orig_ax = -1 => not a system call */
952 953 954
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
955
	jmp	error_exit
956 957
END(xen_failsafe_callback)

958
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
959 960
	xen_hvm_callback_vector xen_evtchn_do_upcall

961
#endif /* CONFIG_XEN */
962

963
#if IS_ENABLED(CONFIG_HYPERV)
964
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
965 966 967
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

968 969 970 971
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

972
#ifdef CONFIG_XEN
973 974 975
idtentry xen_debug		do_debug		has_error_code=0
idtentry xen_int3		do_int3			has_error_code=0
idtentry xen_stack_segment	do_stack_segment	has_error_code=1
976
#endif
977 978 979 980

idtentry general_protection	do_general_protection	has_error_code=1
trace_idtentry page_fault	do_page_fault		has_error_code=1

G
Gleb Natapov 已提交
981
#ifdef CONFIG_KVM_GUEST
982
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
983
#endif
984

985
#ifdef CONFIG_X86_MCE
986
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
987 988
#endif

989 990 991 992 993 994
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
995 996 997
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
998 999
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
1000
	rdmsr
1001 1002
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
1003
	SWAPGS
1004
	xorl	%ebx, %ebx
1005
1:	ret
1006
END(paranoid_entry)
1007

1008 1009 1010 1011 1012 1013 1014 1015 1016
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1017 1018
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1019
 */
1020 1021
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
1022
	TRACE_IRQS_OFF_DEBUG
1023 1024
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	paranoid_exit_no_swapgs
1025
	TRACE_IRQS_IRETQ
1026
	SWAPGS_UNSAFE_STACK
1027
	jmp	paranoid_exit_restore
1028
paranoid_exit_no_swapgs:
1029
	TRACE_IRQS_IRETQ_DEBUG
1030
paranoid_exit_restore:
1031 1032 1033
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
1034
	INTERRUPT_RETURN
1035 1036 1037
END(paranoid_exit)

/*
1038
 * Save all registers in pt_regs, and switch gs if needed.
1039
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1040 1041 1042
 */
ENTRY(error_entry)
	cld
1043 1044
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1045
	xorl	%ebx, %ebx
1046
	testb	$3, CS+8(%rsp)
1047
	jz	.Lerror_kernelspace
1048

1049 1050 1051 1052 1053
.Lerror_entry_from_usermode_swapgs:
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
1054
	SWAPGS
1055

1056
.Lerror_entry_from_usermode_after_swapgs:
1057 1058 1059 1060
#ifdef CONFIG_CONTEXT_TRACKING
	call enter_from_user_mode
#endif

1061
.Lerror_entry_done:
1062

1063 1064 1065
	TRACE_IRQS_OFF
	ret

1066 1067 1068 1069 1070 1071
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1072
.Lerror_kernelspace:
1073 1074 1075
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1076
	je	.Lerror_bad_iret
1077 1078
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1079
	je	.Lbstep_iret
1080
	cmpq	$gs_change, RIP+8(%rsp)
1081
	jne	.Lerror_entry_done
1082 1083 1084 1085 1086 1087

	/*
	 * hack: gs_change can fail with user gsbase.  If this happens, fix up
	 * gsbase and proceed.  We'll fix up the exception and land in
	 * gs_change's error handler with kernel gsbase.
	 */
1088
	jmp	.Lerror_entry_from_usermode_swapgs
1089

1090
.Lbstep_iret:
1091
	/* Fix truncated RIP */
1092
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1093 1094
	/* fall through */

1095
.Lerror_bad_iret:
1096 1097 1098 1099
	/*
	 * We came from an IRET to user mode, so we have user gsbase.
	 * Switch to kernel gsbase:
	 */
A
Andy Lutomirski 已提交
1100
	SWAPGS
1101 1102 1103 1104 1105 1106

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1107 1108 1109
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1110
	decl	%ebx
1111
	jmp	.Lerror_entry_from_usermode_after_swapgs
1112 1113 1114
END(error_entry)


1115 1116 1117 1118 1119
/*
 * On entry, EBS is a "return to kernel mode" flag:
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1120
ENTRY(error_exit)
1121
	movl	%ebx, %eax
1122 1123
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
1124 1125 1126
	testl	%eax, %eax
	jnz	retint_kernel
	jmp	retint_user
1127 1128
END(error_exit)

1129
/* Runs on exception stack */
1130
ENTRY(nmi)
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
	/*
	 * Fix up the exception frame if we're on Xen.
	 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
	 * one value to the stack on native, so it may clobber the rdx
	 * scratch slot, but it won't clobber any of the important
	 * slots past it.
	 *
	 * Xen is a different story, because the Xen frame itself overlaps
	 * the "NMI executing" variable.
	 */
1141
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1142

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1160 1161 1162
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1163 1164
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1165
	 *    o Modify the "iret" location to jump to the repeat_nmi
1166 1167 1168 1169 1170 1171 1172 1173
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1174 1175 1176 1177 1178
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1179 1180
	 */

1181
	/* Use %rdx as our temp variable throughout */
1182
	pushq	%rdx
1183

1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
	 */

	SWAPGS
	cld
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1231
	/*
1232 1233 1234
	 * Return back to user mode.  We must *not* do the normal exit
	 * work, because we don't want to enable interrupts.  Fortunately,
	 * do_nmi doesn't modify pt_regs.
1235
	 */
1236 1237
	SWAPGS
	jmp	restore_c_regs_and_iret
1238

1239
.Lnmi_from_kernel:
1240
	/*
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1281
	/*
1282 1283
	 * Determine whether we're a nested NMI.
	 *
1284 1285 1286 1287 1288 1289
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1290
	 */
1291 1292 1293 1294 1295 1296 1297 1298

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1299

1300
	/*
1301
	 * Now check "NMI executing".  If it's set, then we're nested.
1302 1303
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1304
	 */
1305 1306
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1307 1308

	/*
1309 1310
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1311 1312 1313 1314 1315 1316 1317 1318
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1319
	 */
1320 1321 1322 1323 1324
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1325

1326 1327 1328 1329
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1330 1331 1332 1333 1334 1335 1336

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1337

1338 1339
nested_nmi:
	/*
1340 1341
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1342
	 */
1343
	subq	$8, %rsp
1344 1345 1346
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1347
	pushfq
1348 1349
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1350 1351

	/* Put stack back */
1352
	addq	$(6*8), %rsp
1353 1354

nested_nmi_out:
1355
	popq	%rdx
1356

1357
	/* We are returning to kernel mode, so this cannot result in a fault. */
1358 1359 1360
	INTERRUPT_RETURN

first_nmi:
1361
	/* Restore rdx. */
1362
	movq	(%rsp), %rdx
1363

1364 1365
	/* Make room for "NMI executing". */
	pushq	$0
1366

1367
	/* Leave room for the "iret" frame */
1368
	subq	$(5*8), %rsp
1369

1370
	/* Copy the "original" frame to the "outermost" frame */
1371
	.rept 5
1372
	pushq	11*8(%rsp)
1373
	.endr
1374

1375 1376
	/* Everything up to here is safe from nested NMIs */

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
	INTERRUPT_RETURN	/* continues at repeat_nmi below */
1:
#endif

1392
repeat_nmi:
1393 1394 1395 1396 1397 1398 1399 1400
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1401 1402 1403 1404
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1405 1406
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1407
	 */
1408
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1409

1410
	/*
1411 1412 1413
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1414
	 */
1415
	addq	$(10*8), %rsp
1416
	.rept 5
1417
	pushq	-6*8(%rsp)
1418
	.endr
1419
	subq	$(5*8), %rsp
1420
end_repeat_nmi:
1421 1422

	/*
1423 1424 1425
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1426
	 */
1427
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1428 1429
	ALLOC_PT_GPREGS_ON_STACK

1430
	/*
1431
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1432 1433 1434 1435 1436
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1437
	call	paranoid_entry
1438

1439
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1440 1441 1442
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1443

1444 1445
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1446 1447 1448
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1449 1450
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1451 1452

	/* Point RSP at the "iret" frame. */
1453
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1454

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
	 * the SYSCALL entry and exit paths.  On a native kernel, we
	 * could just inspect RIP, but, on paravirt kernels,
	 * INTERRUPT_RETURN can translate into a jump into a
	 * hypercall page.
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1465 1466 1467 1468 1469 1470

	/*
	 * INTERRUPT_RETURN reads the "iret" frame and exits the NMI
	 * stack in a single instruction.  We are returning to kernel
	 * mode, so this cannot result in a fault.
	 */
1471
	INTERRUPT_RETURN
1472 1473 1474
END(nmi)

ENTRY(ignore_sysret)
1475
	mov	$-ENOSYS, %eax
1476 1477
	sysret
END(ignore_sysret)