entry_64.S 49.2 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4 5 6 7
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
8
 *
L
Linus Torvalds 已提交
9 10
 * entry.S contains the system-call and fault low-level handling routines.
 *
11 12
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
13
 * A note on terminology:
14 15
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
16 17
 *
 * Some macro usage:
18 19 20
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
21 22 23 24 25
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/smap.h>
37
#include <asm/pgtable_types.h>
38
#include <asm/export.h>
39
#include <asm/frame.h>
40
#include <asm/nospec-branch.h>
41
#include <linux/err.h>
L
Linus Torvalds 已提交
42

43 44
#include "calling.h"

45 46
.code64
.section .entry.text, "ax"
47

48
#ifdef CONFIG_PARAVIRT
49
ENTRY(native_usergs_sysret64)
50
	UNWIND_HINT_EMPTY
51 52
	swapgs
	sysretq
53
END(native_usergs_sysret64)
54 55
#endif /* CONFIG_PARAVIRT */

56
.macro TRACE_IRQS_IRETQ
57
#ifdef CONFIG_TRACE_IRQFLAGS
58 59
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
60 61 62 63 64
	TRACE_IRQS_ON
1:
#endif
.endm

65 66 67 68 69 70 71 72 73 74 75 76 77 78
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
79
	call	debug_stack_set_zero
80
	TRACE_IRQS_OFF
81
	call	debug_stack_reset
82 83 84
.endm

.macro TRACE_IRQS_ON_DEBUG
85
	call	debug_stack_set_zero
86
	TRACE_IRQS_ON
87
	call	debug_stack_reset
88 89
.endm

90
.macro TRACE_IRQS_IRETQ_DEBUG
91 92
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
93 94 95 96 97
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
98 99 100
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
101 102
#endif

L
Linus Torvalds 已提交
103
/*
104
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
105
 *
106 107 108 109 110 111 112 113 114 115
 * This is the only entry point used for 64-bit system calls.  The
 * hardware interface is reasonably well designed and the register to
 * argument mapping Linux uses fits well with the registers that are
 * available when SYSCALL is used.
 *
 * SYSCALL instructions can be found inlined in libc implementations as
 * well as some other programs and libraries.  There are also a handful
 * of SYSCALL instructions in the vDSO used, for example, as a
 * clock_gettimeofday fallback.
 *
116
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
117 118 119 120 121 122
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
123
 * rax  system call number
124 125
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
126 127
 * rdi  arg0
 * rsi  arg1
128
 * rdx  arg2
129
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
130 131
 * r8   arg4
 * r9   arg5
132
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
133
 *
L
Linus Torvalds 已提交
134 135
 * Only called from user space.
 *
136
 * When user can change pt_regs->foo always force IRET. That is because
137 138
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
139
 */
L
Linus Torvalds 已提交
140

141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
	.pushsection .entry_trampoline, "ax"

/*
 * The code in here gets remapped into cpu_entry_area's trampoline.  This means
 * that the assembler and linker have the wrong idea as to where this code
 * lives (and, in fact, it's mapped more than once, so it's not even at a
 * fixed address).  So we can't reference any symbols outside the entry
 * trampoline and expect it to work.
 *
 * Instead, we carefully abuse %rip-relative addressing.
 * _entry_trampoline(%rip) refers to the start of the remapped) entry
 * trampoline.  We can thus find cpu_entry_area with this macro:
 */

#define CPU_ENTRY_AREA \
	_entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip)

/* The top word of the SYSENTER stack is hot and is usable as scratch space. */
159 160
#define RSP_SCRATCH	CPU_ENTRY_AREA_entry_stack + \
			SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA
161 162 163 164 165 166 167 168

ENTRY(entry_SYSCALL_64_trampoline)
	UNWIND_HINT_EMPTY
	swapgs

	/* Stash the user RSP. */
	movq	%rsp, RSP_SCRATCH

169 170 171
	/* Note: using %rsp as a scratch reg. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp

172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
	/* Load the top of the task stack into RSP */
	movq	CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp

	/* Start building the simulated IRET frame. */
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	RSP_SCRATCH			/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */

	/*
	 * x86 lacks a near absolute jump, and we can't jump to the real
	 * entry text with a relative jump.  We could push the target
	 * address and then use retq, but this destroys the pipeline on
	 * many CPUs (wasting over 20 cycles on Sandy Bridge).  Instead,
	 * spill RDI and restore it in a second-stage trampoline.
	 */
	pushq	%rdi
	movq	$entry_SYSCALL_64_stage2, %rdi
191
	JMP_NOSPEC %rdi
192 193 194 195 196 197 198 199 200 201
END(entry_SYSCALL_64_trampoline)

	.popsection

ENTRY(entry_SYSCALL_64_stage2)
	UNWIND_HINT_EMPTY
	popq	%rdi
	jmp	entry_SYSCALL_64_after_hwframe
END(entry_SYSCALL_64_stage2)

202
ENTRY(entry_SYSCALL_64)
203
	UNWIND_HINT_EMPTY
204 205 206 207 208
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
209

210
	swapgs
211 212 213 214
	/*
	 * This path is not taken when PAGE_TABLE_ISOLATION is disabled so it
	 * is not required to switch CR3.
	 */
215 216
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
217

218 219
	TRACE_IRQS_OFF

220
	/* Construct struct pt_regs on stack */
221 222 223 224 225
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
226
GLOBAL(entry_SYSCALL_64_after_hwframe)
227 228 229 230 231 232 233 234 235 236 237
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
238
	UNWIND_HINT_REGS extra=0
239

240 241 242 243
	/*
	 * If we need to do entry work or if we guess we'll need to do
	 * exit work, go straight to the slow path.
	 */
244 245
	movq	PER_CPU_VAR(current_task), %r11
	testl	$_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
246 247
	jnz	entry_SYSCALL64_slow_path

248
entry_SYSCALL_64_fastpath:
249 250 251 252 253 254 255
	/*
	 * Easy case: enable interrupts and issue the syscall.  If the syscall
	 * needs pt_regs, we'll call a stub that disables interrupts again
	 * and jumps to the slow path.
	 */
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
256
#if __SYSCALL_MASK == ~0
257
	cmpq	$__NR_syscall_max, %rax
258
#else
259 260
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
261
#endif
262 263
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
264 265 266

	/*
	 * This call instruction is handled specially in stub_ptregs_64.
267 268
	 * It might end up jumping to the slow path.  If it jumps, RAX
	 * and all argument registers are clobbered.
269
	 */
270 271 272 273
#ifdef CONFIG_RETPOLINE
	movq	sys_call_table(, %rax, 8), %rax
	call	__x86_indirect_thunk_rax
#else
274
	call	*sys_call_table(, %rax, 8)
275
#endif
276 277
.Lentry_SYSCALL_64_after_fastpath_call:

278
	movq	%rax, RAX(%rsp)
279
1:
280 281

	/*
282 283 284
	 * If we get here, then we know that pt_regs is clean for SYSRET64.
	 * If we see that no exit work is required (which we are required
	 * to check with IRQs off), then we can go straight to SYSRET64.
285
	 */
286
	DISABLE_INTERRUPTS(CLBR_ANY)
287
	TRACE_IRQS_OFF
288 289
	movq	PER_CPU_VAR(current_task), %r11
	testl	$_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
290
	jnz	1f
291

292 293
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON		/* user mode is traced as IRQs on */
294 295
	movq	RIP(%rsp), %rcx
	movq	EFLAGS(%rsp), %r11
296
	addq	$6*8, %rsp	/* skip extra regs -- they were preserved */
297
	UNWIND_HINT_EMPTY
298
	jmp	.Lpop_c_regs_except_rcx_r11_and_sysret
L
Linus Torvalds 已提交
299

300 301 302 303 304 305
1:
	/*
	 * The fast path looked good when we started, but something changed
	 * along the way and we need to switch to the slow path.  Calling
	 * raise(3) will trigger this, for example.  IRQs are off.
	 */
306
	TRACE_IRQS_ON
307
	ENABLE_INTERRUPTS(CLBR_ANY)
308
	SAVE_EXTRA_REGS
309
	movq	%rsp, %rdi
310 311
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	jmp	return_from_SYSCALL_64
312

313 314
entry_SYSCALL64_slow_path:
	/* IRQs are off. */
315
	SAVE_EXTRA_REGS
316
	movq	%rsp, %rdi
317 318 319
	call	do_syscall_64		/* returns with IRQs disabled */

return_from_SYSCALL_64:
320
	TRACE_IRQS_IRETQ		/* we're about to change IF */
321 322 323

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
324 325
	 * a completely clean 64-bit userspace context.  If we're not,
	 * go to the slow exit path.
326
	 */
327 328
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
329 330 331

	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
	jne	swapgs_restore_regs_and_return_to_usermode
332 333 334 335

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
336
	 * the kernel, since userspace controls RSP.
337
	 *
338
	 * If width of "canonical tail" ever becomes variable, this will need
339
	 * to be updated to remain correct on both old and new CPUs.
340
	 *
341 342
	 * Change top bits to match most significant bit (47th or 56th bit
	 * depending on paging mode) in the address.
343
	 */
344 345
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
346

347 348
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
349
	jne	swapgs_restore_regs_and_return_to_usermode
350

351
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
352
	jne	swapgs_restore_regs_and_return_to_usermode
353

354 355
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
356
	jne	swapgs_restore_regs_and_return_to_usermode
357 358

	/*
359 360 361 362 363 364 365 366 367
	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
	 * restore RF properly. If the slowpath sets it for whatever reason, we
	 * need to restore it correctly.
	 *
	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
	 * trap from userspace immediately after SYSRET.  This would cause an
	 * infinite loop whenever #DB happens with register state that satisfies
	 * the opportunistic SYSRET conditions.  For example, single-stepping
	 * this user code:
368
	 *
369
	 *           movq	$stuck_here, %rcx
370 371 372 373 374 375
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
376
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
377
	jnz	swapgs_restore_regs_and_return_to_usermode
378 379 380

	/* nothing to check for RSP */

381
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
382
	jne	swapgs_restore_regs_and_return_to_usermode
383 384

	/*
385 386
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
387 388
	 */
syscall_return_via_sysret:
389
	/* rcx and r11 are already restored (see code above) */
390
	UNWIND_HINT_EMPTY
391
	POP_EXTRA_REGS
392
.Lpop_c_regs_except_rcx_r11_and_sysret:
393 394 395 396 397 398 399 400
	popq	%rsi	/* skip r11 */
	popq	%r10
	popq	%r9
	popq	%r8
	popq	%rax
	popq	%rsi	/* skip rcx */
	popq	%rdx
	popq	%rsi
401 402 403 404 405 406

	/*
	 * Now all regs are restored except RSP and RDI.
	 * Save old stack pointer and switch to trampoline stack.
	 */
	movq	%rsp, %rdi
407
	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
408 409 410 411 412 413 414 415

	pushq	RSP-RDI(%rdi)	/* RSP */
	pushq	(%rdi)		/* RDI */

	/*
	 * We are on the trampoline stack.  All regs except RDI are live.
	 * We can do future final exit work right here.
	 */
416
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
417

418
	popq	%rdi
419
	popq	%rsp
420
	USERGS_SYSRET64
421
END(entry_SYSCALL_64)
422

423 424 425
ENTRY(stub_ptregs_64)
	/*
	 * Syscalls marked as needing ptregs land here.
426 427 428
	 * If we are on the fast path, we need to save the extra regs,
	 * which we achieve by trying again on the slow path.  If we are on
	 * the slow path, the extra regs are already saved.
429 430
	 *
	 * RAX stores a pointer to the C function implementing the syscall.
431
	 * IRQs are on.
432 433 434 435
	 */
	cmpq	$.Lentry_SYSCALL_64_after_fastpath_call, (%rsp)
	jne	1f

436 437 438 439
	/*
	 * Called from fast path -- disable IRQs again, pop return address
	 * and jump to slow path
	 */
440
	DISABLE_INTERRUPTS(CLBR_ANY)
441
	TRACE_IRQS_OFF
442
	popq	%rax
443
	UNWIND_HINT_REGS extra=0
444
	jmp	entry_SYSCALL64_slow_path
445 446

1:
447
	JMP_NOSPEC %rax				/* Called from C */
448 449 450 451
END(stub_ptregs_64)

.macro ptregs_stub func
ENTRY(ptregs_\func)
452
	UNWIND_HINT_FUNC
453 454 455 456 457 458 459 460 461 462
	leaq	\func(%rip), %rax
	jmp	stub_ptregs_64
END(ptregs_\func)
.endm

/* Instantiate ptregs_stub for each ptregs-using syscall */
#define __SYSCALL_64_QUAL_(sym)
#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym
#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym)
#include <asm/syscalls_64.h>
463

464 465 466 467 468
/*
 * %rdi: prev task
 * %rsi: next task
 */
ENTRY(__switch_to_asm)
469
	UNWIND_HINT_FUNC
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
	/*
	 * Save callee-saved registers
	 * This must match the order in inactive_task_frame
	 */
	pushq	%rbp
	pushq	%rbx
	pushq	%r12
	pushq	%r13
	pushq	%r14
	pushq	%r15

	/* switch stack */
	movq	%rsp, TASK_threadsp(%rdi)
	movq	TASK_threadsp(%rsi), %rsp

#ifdef CONFIG_CC_STACKPROTECTOR
	movq	TASK_stack_canary(%rsi), %rbx
	movq	%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
#endif

490 491 492 493 494 495 496 497 498 499 500
#ifdef CONFIG_RETPOLINE
	/*
	 * When switching from a shallower to a deeper call stack
	 * the RSB may either underflow or use entries populated
	 * with userspace addresses. On CPUs where those concerns
	 * exist, overwrite the RSB with entries which capture
	 * speculative execution to prevent attack.
	 */
	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
#endif

501 502 503 504 505 506 507 508 509 510 511
	/* restore callee-saved registers */
	popq	%r15
	popq	%r14
	popq	%r13
	popq	%r12
	popq	%rbx
	popq	%rbp

	jmp	__switch_to
END(__switch_to_asm)

512 513 514
/*
 * A newly forked process directly context switches into this address.
 *
515
 * rax: prev task we switched from
516 517
 * rbx: kernel thread func (NULL for user thread)
 * r12: kernel thread arg
518 519
 */
ENTRY(ret_from_fork)
520
	UNWIND_HINT_EMPTY
521
	movq	%rax, %rdi
522
	call	schedule_tail			/* rdi: 'prev' task parameter */
523

524 525
	testq	%rbx, %rbx			/* from kernel_thread? */
	jnz	1f				/* kernel threads are uncommon */
526

527
2:
528
	UNWIND_HINT_REGS
529
	movq	%rsp, %rdi
530 531
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
532
	jmp	swapgs_restore_regs_and_return_to_usermode
533 534 535 536

1:
	/* kernel thread */
	movq	%r12, %rdi
537
	CALL_NOSPEC %rbx
538 539 540 541 542 543 544
	/*
	 * A kernel thread is allowed to return here after successfully
	 * calling do_execve().  Exit to userspace to complete the execve()
	 * syscall.
	 */
	movq	$0, RAX(%rsp)
	jmp	2b
545 546
END(ret_from_fork)

547
/*
548 549
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
550
 */
551
	.align 8
552
ENTRY(irq_entries_start)
553 554
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
555
	UNWIND_HINT_IRET_REGS
556
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
557 558
	jmp	common_interrupt
	.align	8
559
	vector=vector+1
560
    .endr
561 562
END(irq_entries_start)

563 564
.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
#ifdef CONFIG_DEBUG_ENTRY
565 566 567
	pushq %rax
	SAVE_FLAGS(CLBR_RAX)
	testl $X86_EFLAGS_IF, %eax
568 569 570
	jz .Lokay_\@
	ud2
.Lokay_\@:
571
	popq %rax
572 573 574 575 576 577 578 579 580 581
#endif
.endm

/*
 * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
 * Requires kernel GSBASE.
 *
 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
 */
582
.macro ENTER_IRQ_STACK regs=1 old_rsp
583 584
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	movq	%rsp, \old_rsp
585 586 587 588 589

	.if \regs
	UNWIND_HINT_REGS base=\old_rsp
	.endif

590
	incl	PER_CPU_VAR(irq_count)
591
	jnz	.Lirq_stack_push_old_rsp_\@
592 593 594 595 596 597 598 599 600

	/*
	 * Right now, if we just incremented irq_count to zero, we've
	 * claimed the IRQ stack but we haven't switched to it yet.
	 *
	 * If anything is added that can interrupt us here without using IST,
	 * it must be *extremely* careful to limit its stack usage.  This
	 * could include kprobes and a hypothetical future IST-less #DB
	 * handler.
601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
	 *
	 * The OOPS unwinder relies on the word at the top of the IRQ
	 * stack linking back to the previous RSP for the entire time we're
	 * on the IRQ stack.  For this to work reliably, we need to write
	 * it before we actually move ourselves to the IRQ stack.
	 */

	movq	\old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
	movq	PER_CPU_VAR(irq_stack_ptr), %rsp

#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * If the first movq above becomes wrong due to IRQ stack layout
	 * changes, the only way we'll notice is if we try to unwind right
	 * here.  Assert that we set up the stack right to catch this type
	 * of bug quickly.
617
	 */
618 619 620 621 622
	cmpq	-8(%rsp), \old_rsp
	je	.Lirq_stack_okay\@
	ud2
	.Lirq_stack_okay\@:
#endif
623

624
.Lirq_stack_push_old_rsp_\@:
625
	pushq	\old_rsp
626 627 628 629

	.if \regs
	UNWIND_HINT_REGS indirect=1
	.endif
630 631 632 633 634
.endm

/*
 * Undoes ENTER_IRQ_STACK.
 */
635
.macro LEAVE_IRQ_STACK regs=1
636 637 638 639
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	/* We need to be off the IRQ stack before decrementing irq_count. */
	popq	%rsp

640 641 642 643
	.if \regs
	UNWIND_HINT_REGS
	.endif

644 645 646 647 648 649 650 651
	/*
	 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
	 * the irq stack but we're not on it.
	 */

	decl	PER_CPU_VAR(irq_count)
.endm

652
/*
L
Linus Torvalds 已提交
653 654 655
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
656 657 658
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
659

660
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
661
	.macro interrupt func
662
	cld
663 664 665 666 667 668 669

	testb	$3, CS-ORIG_RAX(%rsp)
	jz	1f
	SWAPGS
	call	switch_to_thread_stack
1:

670 671 672
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
673
	ENCODE_FRAME_POINTER
674

675
	testb	$3, CS(%rsp)
676
	jz	1f
677 678

	/*
679 680
	 * IRQ from user mode.
	 *
681 682 683 684 685 686 687 688 689
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
	 * the simplest way to handle it is to just call it twice if
	 * we enter from user mode.  There's no reason to optimize this since
	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
	 */
	TRACE_IRQS_OFF

690
	CALL_enter_from_user_mode
691

692
1:
693
	ENTER_IRQ_STACK old_rsp=%rdi
694 695 696
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

697
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
698 699
	.endm

700 701 702 703
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
704 705
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
706
	ASM_CLAC
707
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
708
	interrupt do_IRQ
709
	/* 0(%rsp): old RSP */
710
ret_from_intr:
711
	DISABLE_INTERRUPTS(CLBR_ANY)
712
	TRACE_IRQS_OFF
713

714
	LEAVE_IRQ_STACK
715

716
	testb	$3, CS(%rsp)
717
	jz	retint_kernel
718

719 720 721 722
	/* Interrupt came from user space */
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
723
	TRACE_IRQS_IRETQ
724

725
GLOBAL(swapgs_restore_regs_and_return_to_usermode)
726 727
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates user mode. */
728
	testb	$3, CS(%rsp)
729 730 731 732
	jnz	1f
	ud2
1:
#endif
733
	POP_EXTRA_REGS
734 735 736 737 738 739 740 741 742 743 744 745 746 747
	popq	%r11
	popq	%r10
	popq	%r9
	popq	%r8
	popq	%rax
	popq	%rcx
	popq	%rdx
	popq	%rsi

	/*
	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
	 * Save old stack pointer and switch to trampoline stack.
	 */
	movq	%rsp, %rdi
748
	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764

	/* Copy the IRET frame to the trampoline stack. */
	pushq	6*8(%rdi)	/* SS */
	pushq	5*8(%rdi)	/* RSP */
	pushq	4*8(%rdi)	/* EFLAGS */
	pushq	3*8(%rdi)	/* CS */
	pushq	2*8(%rdi)	/* RIP */

	/* Push user RDI on the trampoline stack. */
	pushq	(%rdi)

	/*
	 * We are on the trampoline stack.  All regs except RDI are live.
	 * We can do future final exit work right here.
	 */

765
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
766

767 768 769
	/* Restore RDI. */
	popq	%rdi
	SWAPGS
770 771
	INTERRUPT_RETURN

772

773
/* Returning to kernel space */
774
retint_kernel:
775 776 777
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
778
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
779
	jnc	1f
780
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
781
	jnz	1f
782
	call	preempt_schedule_irq
783
	jmp	0b
784
1:
785
#endif
786 787 788 789
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
790

791 792 793
GLOBAL(restore_regs_and_return_to_kernel)
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates kernel mode. */
794
	testb	$3, CS(%rsp)
795 796 797 798
	jz	1f
	ud2
1:
#endif
799 800 801
	POP_EXTRA_REGS
	POP_C_REGS
	addq	$8, %rsp	/* skip regs->orig_ax */
802 803 804
	INTERRUPT_RETURN

ENTRY(native_iret)
805
	UNWIND_HINT_IRET_REGS
806 807 808 809
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
810
#ifdef CONFIG_X86_ESPFIX64
811 812
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
813
#endif
814

815
.global native_irq_return_iret
816
native_irq_return_iret:
A
Andy Lutomirski 已提交
817 818 819 820 821 822
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
823
	iretq
I
Ingo Molnar 已提交
824

825
#ifdef CONFIG_X86_ESPFIX64
826
native_irq_return_ldt:
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
	/*
	 * We are running with user GSBASE.  All GPRs contain their user
	 * values.  We have a percpu ESPFIX stack that is eight slots
	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
	 * of the ESPFIX stack.
	 *
	 * We clobber RAX and RDI in this code.  We stash RDI on the
	 * normal stack and RAX on the ESPFIX stack.
	 *
	 * The ESPFIX stack layout we set up looks like this:
	 *
	 * --- top of ESPFIX stack ---
	 * SS
	 * RSP
	 * RFLAGS
	 * CS
	 * RIP  <-- RSP points here when we're done
	 * RAX  <-- espfix_waddr points here
	 * --- bottom of ESPFIX stack ---
	 */

	pushq	%rdi				/* Stash user RDI */
849 850 851
	SWAPGS					/* to kernel GS */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */

852
	movq	PER_CPU_VAR(espfix_waddr), %rdi
853 854
	movq	%rax, (0*8)(%rdi)		/* user RAX */
	movq	(1*8)(%rsp), %rax		/* user RIP */
855
	movq	%rax, (1*8)(%rdi)
856
	movq	(2*8)(%rsp), %rax		/* user CS */
857
	movq	%rax, (2*8)(%rdi)
858
	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
859
	movq	%rax, (3*8)(%rdi)
860
	movq	(5*8)(%rsp), %rax		/* user SS */
861
	movq	%rax, (5*8)(%rdi)
862
	movq	(4*8)(%rsp), %rax		/* user RSP */
863
	movq	%rax, (4*8)(%rdi)
864 865 866 867 868 869 870 871 872 873 874 875
	/* Now RAX == RSP. */

	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */

	/*
	 * espfix_stack[31:16] == 0.  The page tables are set up such that
	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
	 * the same page.  Set up RSP so that RSP[31:16] contains the
	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
	 * still points to an RO alias of the ESPFIX stack.
	 */
876
	orq	PER_CPU_VAR(espfix_stack), %rax
877

878
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
879 880 881
	SWAPGS					/* to user GS */
	popq	%rdi				/* Restore user RDI */

882
	movq	%rax, %rsp
883
	UNWIND_HINT_IRET_REGS offset=8
884 885 886 887 888 889 890 891 892 893 894 895

	/*
	 * At this point, we cannot write to the stack any more, but we can
	 * still read.
	 */
	popq	%rax				/* Restore user RAX */

	/*
	 * RSP now points to an ordinary IRET frame, except that the page
	 * is read-only and RSP[31:16] are preloaded with the userspace
	 * values.  We can now IRET back to userspace.
	 */
896
	jmp	native_irq_return_iret
897
#endif
898
END(common_interrupt)
899

L
Linus Torvalds 已提交
900 901
/*
 * APIC interrupts.
902
 */
903
.macro apicinterrupt3 num sym do_sym
904
ENTRY(\sym)
905
	UNWIND_HINT_IRET_REGS
906
	ASM_CLAC
907
	pushq	$~(\num)
908
.Lcommon_\sym:
909
	interrupt \do_sym
910
	jmp	ret_from_intr
911 912
END(\sym)
.endm
L
Linus Torvalds 已提交
913

914
/* Make sure APIC interrupt handlers end up in the irqentry section: */
915 916
#define PUSH_SECTION_IRQENTRY	.pushsection .irqentry.text, "ax"
#define POP_SECTION_IRQENTRY	.popsection
917

918
.macro apicinterrupt num sym do_sym
919
PUSH_SECTION_IRQENTRY
920
apicinterrupt3 \num \sym \do_sym
921
POP_SECTION_IRQENTRY
922 923
.endm

924
#ifdef CONFIG_SMP
925 926
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
927
#endif
L
Linus Torvalds 已提交
928

N
Nick Piggin 已提交
929
#ifdef CONFIG_X86_UV
930
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
931
#endif
932 933 934

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
935

936
#ifdef CONFIG_HAVE_KVM
937 938
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
939
apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
940 941
#endif

942
#ifdef CONFIG_X86_MCE_THRESHOLD
943
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
944 945
#endif

946
#ifdef CONFIG_X86_MCE_AMD
947
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
948 949
#endif

950
#ifdef CONFIG_X86_THERMAL_VECTOR
951
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
952
#endif
953

954
#ifdef CONFIG_SMP
955 956 957
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
958
#endif
L
Linus Torvalds 已提交
959

960 961
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
962

963
#ifdef CONFIG_IRQ_WORK
964
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
965 966
#endif

L
Linus Torvalds 已提交
967 968
/*
 * Exception entry points.
969
 */
970
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8)
971

972 973 974 975 976 977 978 979 980
/*
 * Switch to the thread stack.  This is called with the IRET frame and
 * orig_ax on the stack.  (That is, RDI..R12 are not on the stack and
 * space has not been allocated for them.)
 */
ENTRY(switch_to_thread_stack)
	UNWIND_HINT_FUNC

	pushq	%rdi
981 982
	/* Need to switch before accessing the thread stack. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
	movq	%rsp, %rdi
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI

	pushq	7*8(%rdi)		/* regs->ss */
	pushq	6*8(%rdi)		/* regs->rsp */
	pushq	5*8(%rdi)		/* regs->eflags */
	pushq	4*8(%rdi)		/* regs->cs */
	pushq	3*8(%rdi)		/* regs->ip */
	pushq	2*8(%rdi)		/* regs->orig_ax */
	pushq	8(%rdi)			/* return address */
	UNWIND_HINT_FUNC

	movq	(%rdi), %rdi
	ret
END(switch_to_thread_stack)

1000
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
1001
ENTRY(\sym)
1002
	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
1003

1004 1005 1006 1007 1008
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

1009
	ASM_CLAC
1010

1011
	.if \has_error_code == 0
1012
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1013 1014
	.endif

1015
	ALLOC_PT_GPREGS_ON_STACK
1016

1017
	.if \paranoid < 2
1018
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
1019
	jnz	.Lfrom_usermode_switch_stack_\@
1020
	.endif
1021 1022

	.if \paranoid
1023
	call	paranoid_entry
1024
	.else
1025
	call	error_entry
1026
	.endif
1027
	UNWIND_HINT_REGS
1028
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
1029 1030

	.if \paranoid
1031
	.if \shift_ist != -1
1032
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
1033
	.else
1034
	TRACE_IRQS_OFF
1035
	.endif
1036
	.endif
1037

1038
	movq	%rsp, %rdi			/* pt_regs pointer */
1039 1040

	.if \has_error_code
1041 1042
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
1043
	.else
1044
	xorl	%esi, %esi			/* no error code */
1045 1046
	.endif

1047
	.if \shift_ist != -1
1048
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
1049 1050
	.endif

1051
	call	\do_sym
1052

1053
	.if \shift_ist != -1
1054
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
1055 1056
	.endif

1057
	/* these procedures expect "no swapgs" flag in ebx */
1058
	.if \paranoid
1059
	jmp	paranoid_exit
1060
	.else
1061
	jmp	error_exit
1062 1063
	.endif

1064
	.if \paranoid < 2
1065
	/*
1066
	 * Entry from userspace.  Switch stacks and treat it
1067 1068 1069
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1070
.Lfrom_usermode_switch_stack_\@:
1071
	call	error_entry
1072

1073
	movq	%rsp, %rdi			/* pt_regs pointer */
1074 1075

	.if \has_error_code
1076 1077
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
1078
	.else
1079
	xorl	%esi, %esi			/* no error code */
1080 1081
	.endif

1082
	call	\do_sym
1083

1084
	jmp	error_exit			/* %ebx: no swapgs flag */
1085
	.endif
1086
END(\sym)
1087
.endm
1088

1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
1108
ENTRY(native_load_gs_index)
1109
	FRAME_BEGIN
1110
	pushfq
1111
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
1112
	SWAPGS
1113
.Lgs_change:
1114
	movl	%edi, %gs
1115
2:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
1116
	SWAPGS
1117
	popfq
1118
	FRAME_END
1119
	ret
1120
ENDPROC(native_load_gs_index)
1121
EXPORT_SYMBOL(native_load_gs_index)
1122

1123
	_ASM_EXTABLE(.Lgs_change, bad_gs)
1124
	.section .fixup, "ax"
L
Linus Torvalds 已提交
1125
	/* running with kernelgs */
1126
bad_gs:
1127
	SWAPGS					/* switch back to user gs */
1128 1129 1130 1131 1132 1133
.macro ZAP_GS
	/* This can't be a string because the preprocessor needs to see it. */
	movl $__USER_DS, %eax
	movl %eax, %gs
.endm
	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
1134 1135 1136
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
1137
	.previous
1138

1139
/* Call softirq on interrupt stack. Interrupts are off. */
1140
ENTRY(do_softirq_own_stack)
1141 1142
	pushq	%rbp
	mov	%rsp, %rbp
1143
	ENTER_IRQ_STACK regs=0 old_rsp=%r11
1144
	call	__do_softirq
1145
	LEAVE_IRQ_STACK regs=0
1146
	leaveq
1147
	ret
1148
ENDPROC(do_softirq_own_stack)
1149

1150
#ifdef CONFIG_XEN
1151
idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0
1152 1153

/*
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
1166 1167
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

1168 1169 1170 1171
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
1172
	UNWIND_HINT_FUNC
1173
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
1174
	UNWIND_HINT_REGS
1175 1176

	ENTER_IRQ_STACK old_rsp=%r10
1177
	call	xen_evtchn_do_upcall
1178 1179
	LEAVE_IRQ_STACK

1180
#ifndef CONFIG_PREEMPT
1181
	call	xen_maybe_preempt_hcall
1182
#endif
1183
	jmp	error_exit
1184
END(xen_do_hypervisor_callback)
1185 1186

/*
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1199
ENTRY(xen_failsafe_callback)
1200
	UNWIND_HINT_EMPTY
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
1213
	/* All segments match their saved values => Category 2 (Bad IRET). */
1214 1215 1216 1217
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
1218
	UNWIND_HINT_IRET_REGS offset=8
1219
	jmp	general_protection
1220
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1221 1222 1223
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
1224
	UNWIND_HINT_IRET_REGS
1225
	pushq	$-1 /* orig_ax = -1 => not a system call */
1226 1227 1228
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
1229
	ENCODE_FRAME_POINTER
1230
	jmp	error_exit
1231 1232
END(xen_failsafe_callback)

1233
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1234 1235
	xen_hvm_callback_vector xen_evtchn_do_upcall

1236
#endif /* CONFIG_XEN */
1237

1238
#if IS_ENABLED(CONFIG_HYPERV)
1239
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1240 1241 1242
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

1243 1244 1245 1246
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

1247
#ifdef CONFIG_XEN
1248
idtentry xennmi			do_nmi			has_error_code=0
1249 1250
idtentry xendebug		do_debug		has_error_code=0
idtentry xenint3		do_int3			has_error_code=0
1251
#endif
1252 1253

idtentry general_protection	do_general_protection	has_error_code=1
1254
idtentry page_fault		do_page_fault		has_error_code=1
1255

G
Gleb Natapov 已提交
1256
#ifdef CONFIG_KVM_GUEST
1257
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
1258
#endif
1259

1260
#ifdef CONFIG_X86_MCE
1261
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
1262 1263
#endif

1264 1265 1266 1267 1268 1269
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1270
	UNWIND_HINT_FUNC
1271 1272 1273
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1274
	ENCODE_FRAME_POINTER 8
1275 1276
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
1277
	rdmsr
1278 1279
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
1280
	SWAPGS
1281
	xorl	%ebx, %ebx
1282 1283 1284 1285 1286

1:
	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14

	ret
1287
END(paranoid_entry)
1288

1289 1290 1291 1292 1293 1294 1295 1296 1297
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1298 1299
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1300
 */
1301
ENTRY(paranoid_exit)
1302
	UNWIND_HINT_REGS
1303
	DISABLE_INTERRUPTS(CLBR_ANY)
1304
	TRACE_IRQS_OFF_DEBUG
1305
	testl	%ebx, %ebx			/* swapgs needed? */
1306
	jnz	.Lparanoid_exit_no_swapgs
1307
	TRACE_IRQS_IRETQ
P
Peter Zijlstra 已提交
1308
	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1309
	SWAPGS_UNSAFE_STACK
1310 1311
	jmp	.Lparanoid_exit_restore
.Lparanoid_exit_no_swapgs:
1312
	TRACE_IRQS_IRETQ_DEBUG
1313 1314
.Lparanoid_exit_restore:
	jmp restore_regs_and_return_to_kernel
1315 1316 1317
END(paranoid_exit)

/*
1318
 * Save all registers in pt_regs, and switch gs if needed.
1319
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1320 1321
 */
ENTRY(error_entry)
1322
	UNWIND_HINT_FUNC
1323
	cld
1324 1325
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1326
	ENCODE_FRAME_POINTER 8
1327
	xorl	%ebx, %ebx
1328
	testb	$3, CS+8(%rsp)
1329
	jz	.Lerror_kernelspace
1330

1331 1332 1333 1334
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
1335
	SWAPGS
1336 1337
	/* We have user CR3.  Change to kernel CR3. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1338

1339
.Lerror_entry_from_usermode_after_swapgs:
1340 1341 1342 1343 1344 1345 1346 1347
	/* Put us onto the real thread stack. */
	popq	%r12				/* save return addr in %12 */
	movq	%rsp, %rdi			/* arg0 = pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
	ENCODE_FRAME_POINTER
	pushq	%r12

1348 1349 1350 1351 1352 1353
	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).
	 */
	TRACE_IRQS_OFF
1354
	CALL_enter_from_user_mode
1355
	ret
1356

1357
.Lerror_entry_done:
1358 1359 1360
	TRACE_IRQS_OFF
	ret

1361 1362 1363 1364 1365 1366
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1367
.Lerror_kernelspace:
1368 1369 1370
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1371
	je	.Lerror_bad_iret
1372 1373
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1374
	je	.Lbstep_iret
1375
	cmpq	$.Lgs_change, RIP+8(%rsp)
1376
	jne	.Lerror_entry_done
1377 1378

	/*
1379
	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1380
	 * gsbase and proceed.  We'll fix up the exception and land in
1381
	 * .Lgs_change's error handler with kernel gsbase.
1382
	 */
1383
	SWAPGS
1384
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1385
	jmp .Lerror_entry_done
1386

1387
.Lbstep_iret:
1388
	/* Fix truncated RIP */
1389
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1390 1391
	/* fall through */

1392
.Lerror_bad_iret:
1393
	/*
1394 1395
	 * We came from an IRET to user mode, so we have user
	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
1396
	 */
A
Andy Lutomirski 已提交
1397
	SWAPGS
1398
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1399 1400 1401 1402 1403 1404

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1405 1406 1407
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1408
	decl	%ebx
1409
	jmp	.Lerror_entry_from_usermode_after_swapgs
1410 1411 1412
END(error_entry)


1413
/*
1414
 * On entry, EBX is a "return to kernel mode" flag:
1415 1416 1417
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1418
ENTRY(error_exit)
1419
	UNWIND_HINT_REGS
1420
	DISABLE_INTERRUPTS(CLBR_ANY)
1421
	TRACE_IRQS_OFF
1422
	testl	%ebx, %ebx
1423 1424
	jnz	retint_kernel
	jmp	retint_user
1425 1426
END(error_exit)

1427 1428 1429
/*
 * Runs on exception stack.  Xen PV does not go through this path at all,
 * so we can use real assembly here.
1430 1431 1432 1433
 *
 * Registers:
 *	%r14: Used to save/restore the CR3 of the interrupted context
 *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
1434
 */
1435
ENTRY(nmi)
1436
	UNWIND_HINT_IRET_REGS
1437

1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1455 1456 1457
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1458 1459
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1460
	 *    o Modify the "iret" location to jump to the repeat_nmi
1461 1462 1463 1464 1465 1466 1467 1468
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1469 1470 1471 1472 1473
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1474 1475
	 */

1476 1477
	ASM_CLAC

1478
	/* Use %rdx as our temp variable throughout */
1479
	pushq	%rdx
1480

1481 1482 1483 1484 1485 1486 1487 1488 1489
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
1490 1491 1492
	 *
	 * We also must not push anything to the stack before switching
	 * stacks lest we corrupt the "NMI executing" variable.
1493 1494
	 */

1495
	swapgs
1496
	cld
1497
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1498 1499
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1500
	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1501 1502 1503 1504 1505
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
1506
	UNWIND_HINT_IRET_REGS
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */
1523
	UNWIND_HINT_REGS
1524
	ENCODE_FRAME_POINTER
1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1536
	/*
1537
	 * Return back to user mode.  We must *not* do the normal exit
1538
	 * work, because we don't want to enable interrupts.
1539
	 */
1540
	jmp	swapgs_restore_regs_and_return_to_usermode
1541

1542
.Lnmi_from_kernel:
1543
	/*
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1584
	/*
1585 1586
	 * Determine whether we're a nested NMI.
	 *
1587 1588 1589 1590 1591 1592
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1593
	 */
1594 1595 1596 1597 1598 1599 1600 1601

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1602

1603
	/*
1604
	 * Now check "NMI executing".  If it's set, then we're nested.
1605 1606
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1607
	 */
1608 1609
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1610 1611

	/*
1612 1613
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1614 1615 1616 1617 1618 1619 1620 1621
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1622
	 */
1623 1624 1625 1626 1627
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1628

1629 1630 1631 1632
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1633 1634 1635 1636 1637 1638 1639

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1640

1641 1642
nested_nmi:
	/*
1643 1644
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1645
	 */
1646
	subq	$8, %rsp
1647 1648 1649
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1650
	pushfq
1651 1652
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1653 1654

	/* Put stack back */
1655
	addq	$(6*8), %rsp
1656 1657

nested_nmi_out:
1658
	popq	%rdx
1659

1660
	/* We are returning to kernel mode, so this cannot result in a fault. */
1661
	iretq
1662 1663

first_nmi:
1664
	/* Restore rdx. */
1665
	movq	(%rsp), %rdx
1666

1667 1668
	/* Make room for "NMI executing". */
	pushq	$0
1669

1670
	/* Leave room for the "iret" frame */
1671
	subq	$(5*8), %rsp
1672

1673
	/* Copy the "original" frame to the "outermost" frame */
1674
	.rept 5
1675
	pushq	11*8(%rsp)
1676
	.endr
1677
	UNWIND_HINT_IRET_REGS
1678

1679 1680
	/* Everything up to here is safe from nested NMIs */

1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
1692
	iretq			/* continues at repeat_nmi below */
1693
	UNWIND_HINT_IRET_REGS
1694 1695 1696
1:
#endif

1697
repeat_nmi:
1698 1699 1700 1701 1702 1703 1704 1705
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1706 1707 1708 1709
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1710 1711
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1712
	 */
1713
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1714

1715
	/*
1716 1717 1718
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1719
	 */
1720
	addq	$(10*8), %rsp
1721
	.rept 5
1722
	pushq	-6*8(%rsp)
1723
	.endr
1724
	subq	$(5*8), %rsp
1725
end_repeat_nmi:
1726 1727

	/*
1728 1729 1730
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1731
	 */
1732
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1733 1734
	ALLOC_PT_GPREGS_ON_STACK

1735
	/*
1736
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1737 1738 1739 1740 1741
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1742
	call	paranoid_entry
1743
	UNWIND_HINT_REGS
1744

1745
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1746 1747 1748
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1749

P
Peter Zijlstra 已提交
1750
	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1751

1752 1753
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1754 1755 1756
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1757 1758
	POP_EXTRA_REGS
	POP_C_REGS
1759

1760 1761 1762 1763 1764
	/*
	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
	 * at the "iret" frame.
	 */
	addq	$6*8, %rsp
1765

1766 1767 1768
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
1769 1770 1771 1772 1773
	 * the SYSCALL entry and exit paths.
	 *
	 * We arguably should just inspect RIP instead, but I (Andy) wrote
	 * this code when I had the misapprehension that Xen PV supported
	 * NMIs, and Xen PV would break that approach.
1774 1775 1776
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1777 1778

	/*
1779 1780 1781 1782
	 * iretq reads the "iret" frame and exits the NMI stack in a
	 * single instruction.  We are returning to kernel mode, so this
	 * cannot result in a fault.  Similarly, we don't need to worry
	 * about espfix64 on the way back to kernel mode.
1783
	 */
1784
	iretq
1785 1786 1787
END(nmi)

ENTRY(ignore_sysret)
1788
	UNWIND_HINT_EMPTY
1789
	mov	$-ENOSYS, %eax
1790 1791
	sysret
END(ignore_sysret)
1792 1793

ENTRY(rewind_stack_do_exit)
1794
	UNWIND_HINT_FUNC
1795 1796 1797 1798
	/* Prevent any naive code from trying to unwind to our caller. */
	xorl	%ebp, %ebp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
1799 1800
	leaq	-PTREGS_SIZE(%rax), %rsp
	UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
1801 1802 1803

	call	do_exit
END(rewind_stack_do_exit)