entry_64.S 44.0 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4 5 6 7
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
8
 *
L
Linus Torvalds 已提交
9 10
 * entry.S contains the system-call and fault low-level handling routines.
 *
11 12
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
13
 * A note on terminology:
14 15
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
16 17
 *
 * Some macro usage:
18 19 20
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
21 22 23 24 25
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
26
#include "calling.h"
27
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
28 29 30 31
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
32
#include <asm/page_types.h>
33
#include <asm/irqflags.h>
34
#include <asm/paravirt.h>
35
#include <asm/percpu.h>
36
#include <asm/asm.h>
37
#include <asm/smap.h>
38
#include <asm/pgtable_types.h>
39
#include <asm/export.h>
40
#include <asm/frame.h>
41
#include <linux/err.h>
L
Linus Torvalds 已提交
42

43 44
.code64
.section .entry.text, "ax"
45

46
#ifdef CONFIG_PARAVIRT
47
ENTRY(native_usergs_sysret64)
48
	UNWIND_HINT_EMPTY
49 50
	swapgs
	sysretq
51
END(native_usergs_sysret64)
52 53
#endif /* CONFIG_PARAVIRT */

54
.macro TRACE_IRQS_IRETQ
55
#ifdef CONFIG_TRACE_IRQFLAGS
56 57
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
58 59 60 61 62
	TRACE_IRQS_ON
1:
#endif
.endm

63 64 65 66 67 68 69 70 71 72 73 74 75 76
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
77
	call	debug_stack_set_zero
78
	TRACE_IRQS_OFF
79
	call	debug_stack_reset
80 81 82
.endm

.macro TRACE_IRQS_ON_DEBUG
83
	call	debug_stack_set_zero
84
	TRACE_IRQS_ON
85
	call	debug_stack_reset
86 87
.endm

88
.macro TRACE_IRQS_IRETQ_DEBUG
89 90
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
91 92 93 94 95
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
96 97 98
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
99 100
#endif

L
Linus Torvalds 已提交
101
/*
102
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
103
 *
104 105 106 107 108 109 110 111 112 113
 * This is the only entry point used for 64-bit system calls.  The
 * hardware interface is reasonably well designed and the register to
 * argument mapping Linux uses fits well with the registers that are
 * available when SYSCALL is used.
 *
 * SYSCALL instructions can be found inlined in libc implementations as
 * well as some other programs and libraries.  There are also a handful
 * of SYSCALL instructions in the vDSO used, for example, as a
 * clock_gettimeofday fallback.
 *
114
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
115 116 117 118 119 120
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
121
 * rax  system call number
122 123
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
124 125
 * rdi  arg0
 * rsi  arg1
126
 * rdx  arg2
127
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
128 129
 * r8   arg4
 * r9   arg5
130
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
131
 *
L
Linus Torvalds 已提交
132 133
 * Only called from user space.
 *
134
 * When user can change pt_regs->foo always force IRET. That is because
135 136
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
137
 */
L
Linus Torvalds 已提交
138

139
ENTRY(entry_SYSCALL_64)
140
	UNWIND_HINT_EMPTY
141 142 143 144 145
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
146

147
	swapgs
148 149
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
150

151 152
	TRACE_IRQS_OFF

153
	/* Construct struct pt_regs on stack */
154 155 156 157 158
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
159
GLOBAL(entry_SYSCALL_64_after_hwframe)
160 161 162 163 164 165 166 167 168 169 170
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
171
	UNWIND_HINT_REGS extra=0
172

173 174 175 176
	/*
	 * If we need to do entry work or if we guess we'll need to do
	 * exit work, go straight to the slow path.
	 */
177 178
	movq	PER_CPU_VAR(current_task), %r11
	testl	$_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
179 180
	jnz	entry_SYSCALL64_slow_path

181
entry_SYSCALL_64_fastpath:
182 183 184 185 186 187 188
	/*
	 * Easy case: enable interrupts and issue the syscall.  If the syscall
	 * needs pt_regs, we'll call a stub that disables interrupts again
	 * and jumps to the slow path.
	 */
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
189
#if __SYSCALL_MASK == ~0
190
	cmpq	$__NR_syscall_max, %rax
191
#else
192 193
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
194
#endif
195 196
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
197 198 199

	/*
	 * This call instruction is handled specially in stub_ptregs_64.
200 201
	 * It might end up jumping to the slow path.  If it jumps, RAX
	 * and all argument registers are clobbered.
202
	 */
203
	call	*sys_call_table(, %rax, 8)
204 205
.Lentry_SYSCALL_64_after_fastpath_call:

206
	movq	%rax, RAX(%rsp)
207
1:
208 209

	/*
210 211 212
	 * If we get here, then we know that pt_regs is clean for SYSRET64.
	 * If we see that no exit work is required (which we are required
	 * to check with IRQs off), then we can go straight to SYSRET64.
213
	 */
214
	DISABLE_INTERRUPTS(CLBR_ANY)
215
	TRACE_IRQS_OFF
216 217
	movq	PER_CPU_VAR(current_task), %r11
	testl	$_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
218
	jnz	1f
219

220 221
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON		/* user mode is traced as IRQs on */
222 223
	movq	RIP(%rsp), %rcx
	movq	EFLAGS(%rsp), %r11
224
	addq	$6*8, %rsp	/* skip extra regs -- they were preserved */
225
	UNWIND_HINT_EMPTY
226
	jmp	.Lpop_c_regs_except_rcx_r11_and_sysret
L
Linus Torvalds 已提交
227

228 229 230 231 232 233
1:
	/*
	 * The fast path looked good when we started, but something changed
	 * along the way and we need to switch to the slow path.  Calling
	 * raise(3) will trigger this, for example.  IRQs are off.
	 */
234
	TRACE_IRQS_ON
235
	ENABLE_INTERRUPTS(CLBR_ANY)
236
	SAVE_EXTRA_REGS
237
	movq	%rsp, %rdi
238 239
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	jmp	return_from_SYSCALL_64
240

241 242
entry_SYSCALL64_slow_path:
	/* IRQs are off. */
243
	SAVE_EXTRA_REGS
244
	movq	%rsp, %rdi
245 246 247
	call	do_syscall_64		/* returns with IRQs disabled */

return_from_SYSCALL_64:
248
	TRACE_IRQS_IRETQ		/* we're about to change IF */
249 250 251

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
252 253
	 * a completely clean 64-bit userspace context.  If we're not,
	 * go to the slow exit path.
254
	 */
255 256
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
257 258 259

	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
	jne	swapgs_restore_regs_and_return_to_usermode
260 261 262 263

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
264
	 * the kernel, since userspace controls RSP.
265
	 *
266
	 * If width of "canonical tail" ever becomes variable, this will need
267
	 * to be updated to remain correct on both old and new CPUs.
268
	 *
269 270
	 * Change top bits to match most significant bit (47th or 56th bit
	 * depending on paging mode) in the address.
271
	 */
272 273
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
274

275 276
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
277
	jne	swapgs_restore_regs_and_return_to_usermode
278

279
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
280
	jne	swapgs_restore_regs_and_return_to_usermode
281

282 283
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
284
	jne	swapgs_restore_regs_and_return_to_usermode
285 286

	/*
287 288 289 290 291 292 293 294 295
	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
	 * restore RF properly. If the slowpath sets it for whatever reason, we
	 * need to restore it correctly.
	 *
	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
	 * trap from userspace immediately after SYSRET.  This would cause an
	 * infinite loop whenever #DB happens with register state that satisfies
	 * the opportunistic SYSRET conditions.  For example, single-stepping
	 * this user code:
296
	 *
297
	 *           movq	$stuck_here, %rcx
298 299 300 301 302 303
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
304
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
305
	jnz	swapgs_restore_regs_and_return_to_usermode
306 307 308

	/* nothing to check for RSP */

309
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
310
	jne	swapgs_restore_regs_and_return_to_usermode
311 312

	/*
313 314
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
315 316
	 */
syscall_return_via_sysret:
317
	/* rcx and r11 are already restored (see code above) */
318
	UNWIND_HINT_EMPTY
319
	POP_EXTRA_REGS
320
.Lpop_c_regs_except_rcx_r11_and_sysret:
321 322 323 324 325 326 327 328 329 330
	popq	%rsi	/* skip r11 */
	popq	%r10
	popq	%r9
	popq	%r8
	popq	%rax
	popq	%rsi	/* skip rcx */
	popq	%rdx
	popq	%rsi
	popq	%rdi
	movq	RSP-ORIG_RAX(%rsp), %rsp
331
	USERGS_SYSRET64
332
END(entry_SYSCALL_64)
333

334 335 336
ENTRY(stub_ptregs_64)
	/*
	 * Syscalls marked as needing ptregs land here.
337 338 339
	 * If we are on the fast path, we need to save the extra regs,
	 * which we achieve by trying again on the slow path.  If we are on
	 * the slow path, the extra regs are already saved.
340 341
	 *
	 * RAX stores a pointer to the C function implementing the syscall.
342
	 * IRQs are on.
343 344 345 346
	 */
	cmpq	$.Lentry_SYSCALL_64_after_fastpath_call, (%rsp)
	jne	1f

347 348 349 350
	/*
	 * Called from fast path -- disable IRQs again, pop return address
	 * and jump to slow path
	 */
351
	DISABLE_INTERRUPTS(CLBR_ANY)
352
	TRACE_IRQS_OFF
353
	popq	%rax
354
	UNWIND_HINT_REGS extra=0
355
	jmp	entry_SYSCALL64_slow_path
356 357

1:
358
	jmp	*%rax				/* Called from C */
359 360 361 362
END(stub_ptregs_64)

.macro ptregs_stub func
ENTRY(ptregs_\func)
363
	UNWIND_HINT_FUNC
364 365 366 367 368 369 370 371 372 373
	leaq	\func(%rip), %rax
	jmp	stub_ptregs_64
END(ptregs_\func)
.endm

/* Instantiate ptregs_stub for each ptregs-using syscall */
#define __SYSCALL_64_QUAL_(sym)
#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym
#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym)
#include <asm/syscalls_64.h>
374

375 376 377 378 379
/*
 * %rdi: prev task
 * %rsi: next task
 */
ENTRY(__switch_to_asm)
380
	UNWIND_HINT_FUNC
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
	/*
	 * Save callee-saved registers
	 * This must match the order in inactive_task_frame
	 */
	pushq	%rbp
	pushq	%rbx
	pushq	%r12
	pushq	%r13
	pushq	%r14
	pushq	%r15

	/* switch stack */
	movq	%rsp, TASK_threadsp(%rdi)
	movq	TASK_threadsp(%rsi), %rsp

#ifdef CONFIG_CC_STACKPROTECTOR
	movq	TASK_stack_canary(%rsi), %rbx
	movq	%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
#endif

	/* restore callee-saved registers */
	popq	%r15
	popq	%r14
	popq	%r13
	popq	%r12
	popq	%rbx
	popq	%rbp

	jmp	__switch_to
END(__switch_to_asm)

412 413 414
/*
 * A newly forked process directly context switches into this address.
 *
415
 * rax: prev task we switched from
416 417
 * rbx: kernel thread func (NULL for user thread)
 * r12: kernel thread arg
418 419
 */
ENTRY(ret_from_fork)
420
	UNWIND_HINT_EMPTY
421
	movq	%rax, %rdi
422
	call	schedule_tail			/* rdi: 'prev' task parameter */
423

424 425
	testq	%rbx, %rbx			/* from kernel_thread? */
	jnz	1f				/* kernel threads are uncommon */
426

427
2:
428
	UNWIND_HINT_REGS
429
	movq	%rsp, %rdi
430 431
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
432
	jmp	swapgs_restore_regs_and_return_to_usermode
433 434 435 436 437 438 439 440 441 442 443 444

1:
	/* kernel thread */
	movq	%r12, %rdi
	call	*%rbx
	/*
	 * A kernel thread is allowed to return here after successfully
	 * calling do_execve().  Exit to userspace to complete the execve()
	 * syscall.
	 */
	movq	$0, RAX(%rsp)
	jmp	2b
445 446
END(ret_from_fork)

447
/*
448 449
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
450
 */
451
	.align 8
452
ENTRY(irq_entries_start)
453 454
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
455
	UNWIND_HINT_IRET_REGS
456
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
457 458
	jmp	common_interrupt
	.align	8
459
	vector=vector+1
460
    .endr
461 462
END(irq_entries_start)

463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
#ifdef CONFIG_DEBUG_ENTRY
	pushfq
	testl $X86_EFLAGS_IF, (%rsp)
	jz .Lokay_\@
	ud2
.Lokay_\@:
	addq $8, %rsp
#endif
.endm

/*
 * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
 * Requires kernel GSBASE.
 *
 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
 */
481
.macro ENTER_IRQ_STACK regs=1 old_rsp
482 483
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	movq	%rsp, \old_rsp
484 485 486 487 488

	.if \regs
	UNWIND_HINT_REGS base=\old_rsp
	.endif

489
	incl	PER_CPU_VAR(irq_count)
490
	jnz	.Lirq_stack_push_old_rsp_\@
491 492 493 494 495 496 497 498 499

	/*
	 * Right now, if we just incremented irq_count to zero, we've
	 * claimed the IRQ stack but we haven't switched to it yet.
	 *
	 * If anything is added that can interrupt us here without using IST,
	 * it must be *extremely* careful to limit its stack usage.  This
	 * could include kprobes and a hypothetical future IST-less #DB
	 * handler.
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
	 *
	 * The OOPS unwinder relies on the word at the top of the IRQ
	 * stack linking back to the previous RSP for the entire time we're
	 * on the IRQ stack.  For this to work reliably, we need to write
	 * it before we actually move ourselves to the IRQ stack.
	 */

	movq	\old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
	movq	PER_CPU_VAR(irq_stack_ptr), %rsp

#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * If the first movq above becomes wrong due to IRQ stack layout
	 * changes, the only way we'll notice is if we try to unwind right
	 * here.  Assert that we set up the stack right to catch this type
	 * of bug quickly.
516
	 */
517 518 519 520 521
	cmpq	-8(%rsp), \old_rsp
	je	.Lirq_stack_okay\@
	ud2
	.Lirq_stack_okay\@:
#endif
522

523
.Lirq_stack_push_old_rsp_\@:
524
	pushq	\old_rsp
525 526 527 528

	.if \regs
	UNWIND_HINT_REGS indirect=1
	.endif
529 530 531 532 533
.endm

/*
 * Undoes ENTER_IRQ_STACK.
 */
534
.macro LEAVE_IRQ_STACK regs=1
535 536 537 538
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	/* We need to be off the IRQ stack before decrementing irq_count. */
	popq	%rsp

539 540 541 542
	.if \regs
	UNWIND_HINT_REGS
	.endif

543 544 545 546 547 548 549 550
	/*
	 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
	 * the irq stack but we're not on it.
	 */

	decl	PER_CPU_VAR(irq_count)
.endm

551
/*
L
Linus Torvalds 已提交
552 553 554
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
555 556 557
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
558

559
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
560
	.macro interrupt func
561
	cld
562 563 564
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
565
	ENCODE_FRAME_POINTER
566

567
	testb	$3, CS(%rsp)
568
	jz	1f
569 570 571 572 573

	/*
	 * IRQ from user mode.  Switch to kernel gsbase and inform context
	 * tracking that we're in kernel mode.
	 */
574
	SWAPGS
575 576 577 578 579 580 581 582 583 584 585

	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
	 * the simplest way to handle it is to just call it twice if
	 * we enter from user mode.  There's no reason to optimize this since
	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
	 */
	TRACE_IRQS_OFF

586
	CALL_enter_from_user_mode
587

588
1:
589
	ENTER_IRQ_STACK old_rsp=%rdi
590 591 592
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

593
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
594 595
	.endm

596 597 598 599
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
600 601
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
602
	ASM_CLAC
603
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
604
	interrupt do_IRQ
605
	/* 0(%rsp): old RSP */
606
ret_from_intr:
607
	DISABLE_INTERRUPTS(CLBR_ANY)
608
	TRACE_IRQS_OFF
609

610
	LEAVE_IRQ_STACK
611

612
	testb	$3, CS(%rsp)
613
	jz	retint_kernel
614

615 616 617 618
	/* Interrupt came from user space */
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
619
	TRACE_IRQS_IRETQ
620

621
GLOBAL(swapgs_restore_regs_and_return_to_usermode)
622 623
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates user mode. */
624
	testb	$3, CS(%rsp)
625 626 627 628
	jnz	1f
	ud2
1:
#endif
629
	SWAPGS
630 631 632
	POP_EXTRA_REGS
	POP_C_REGS
	addq	$8, %rsp	/* skip regs->orig_ax */
633 634
	INTERRUPT_RETURN

635

636
/* Returning to kernel space */
637
retint_kernel:
638 639 640
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
641
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
642
	jnc	1f
643
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
644
	jnz	1f
645
	call	preempt_schedule_irq
646
	jmp	0b
647
1:
648
#endif
649 650 651 652
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
653

654 655 656
GLOBAL(restore_regs_and_return_to_kernel)
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates kernel mode. */
657
	testb	$3, CS(%rsp)
658 659 660 661
	jz	1f
	ud2
1:
#endif
662 663 664
	POP_EXTRA_REGS
	POP_C_REGS
	addq	$8, %rsp	/* skip regs->orig_ax */
665 666 667
	INTERRUPT_RETURN

ENTRY(native_iret)
668
	UNWIND_HINT_IRET_REGS
669 670 671 672
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
673
#ifdef CONFIG_X86_ESPFIX64
674 675
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
676
#endif
677

678
.global native_irq_return_iret
679
native_irq_return_iret:
A
Andy Lutomirski 已提交
680 681 682 683 684 685
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
686
	iretq
I
Ingo Molnar 已提交
687

688
#ifdef CONFIG_X86_ESPFIX64
689
native_irq_return_ldt:
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
	/*
	 * We are running with user GSBASE.  All GPRs contain their user
	 * values.  We have a percpu ESPFIX stack that is eight slots
	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
	 * of the ESPFIX stack.
	 *
	 * We clobber RAX and RDI in this code.  We stash RDI on the
	 * normal stack and RAX on the ESPFIX stack.
	 *
	 * The ESPFIX stack layout we set up looks like this:
	 *
	 * --- top of ESPFIX stack ---
	 * SS
	 * RSP
	 * RFLAGS
	 * CS
	 * RIP  <-- RSP points here when we're done
	 * RAX  <-- espfix_waddr points here
	 * --- bottom of ESPFIX stack ---
	 */

	pushq	%rdi				/* Stash user RDI */
712
	SWAPGS
713
	movq	PER_CPU_VAR(espfix_waddr), %rdi
714 715
	movq	%rax, (0*8)(%rdi)		/* user RAX */
	movq	(1*8)(%rsp), %rax		/* user RIP */
716
	movq	%rax, (1*8)(%rdi)
717
	movq	(2*8)(%rsp), %rax		/* user CS */
718
	movq	%rax, (2*8)(%rdi)
719
	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
720
	movq	%rax, (3*8)(%rdi)
721
	movq	(5*8)(%rsp), %rax		/* user SS */
722
	movq	%rax, (5*8)(%rdi)
723
	movq	(4*8)(%rsp), %rax		/* user RSP */
724
	movq	%rax, (4*8)(%rdi)
725 726 727 728 729 730 731 732 733 734 735 736 737
	/* Now RAX == RSP. */

	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */
	popq	%rdi				/* Restore user RDI */

	/*
	 * espfix_stack[31:16] == 0.  The page tables are set up such that
	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
	 * the same page.  Set up RSP so that RSP[31:16] contains the
	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
	 * still points to an RO alias of the ESPFIX stack.
	 */
738
	orq	PER_CPU_VAR(espfix_stack), %rax
739
	SWAPGS
740
	movq	%rax, %rsp
741
	UNWIND_HINT_IRET_REGS offset=8
742 743 744 745 746 747 748 749 750 751 752 753

	/*
	 * At this point, we cannot write to the stack any more, but we can
	 * still read.
	 */
	popq	%rax				/* Restore user RAX */

	/*
	 * RSP now points to an ordinary IRET frame, except that the page
	 * is read-only and RSP[31:16] are preloaded with the userspace
	 * values.  We can now IRET back to userspace.
	 */
754
	jmp	native_irq_return_iret
755
#endif
756
END(common_interrupt)
757

L
Linus Torvalds 已提交
758 759
/*
 * APIC interrupts.
760
 */
761
.macro apicinterrupt3 num sym do_sym
762
ENTRY(\sym)
763
	UNWIND_HINT_IRET_REGS
764
	ASM_CLAC
765
	pushq	$~(\num)
766
.Lcommon_\sym:
767
	interrupt \do_sym
768
	jmp	ret_from_intr
769 770
END(\sym)
.endm
L
Linus Torvalds 已提交
771

772
/* Make sure APIC interrupt handlers end up in the irqentry section: */
773 774
#define PUSH_SECTION_IRQENTRY	.pushsection .irqentry.text, "ax"
#define POP_SECTION_IRQENTRY	.popsection
775

776
.macro apicinterrupt num sym do_sym
777
PUSH_SECTION_IRQENTRY
778
apicinterrupt3 \num \sym \do_sym
779
POP_SECTION_IRQENTRY
780 781
.endm

782
#ifdef CONFIG_SMP
783 784
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
785
#endif
L
Linus Torvalds 已提交
786

N
Nick Piggin 已提交
787
#ifdef CONFIG_X86_UV
788
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
789
#endif
790 791 792

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
793

794
#ifdef CONFIG_HAVE_KVM
795 796
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
797
apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
798 799
#endif

800
#ifdef CONFIG_X86_MCE_THRESHOLD
801
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
802 803
#endif

804
#ifdef CONFIG_X86_MCE_AMD
805
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
806 807
#endif

808
#ifdef CONFIG_X86_THERMAL_VECTOR
809
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
810
#endif
811

812
#ifdef CONFIG_SMP
813 814 815
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
816
#endif
L
Linus Torvalds 已提交
817

818 819
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
820

821
#ifdef CONFIG_IRQ_WORK
822
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
823 824
#endif

L
Linus Torvalds 已提交
825 826
/*
 * Exception entry points.
827
 */
828
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
829 830

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
831
ENTRY(\sym)
832
	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
833

834 835 836 837 838
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

839
	ASM_CLAC
840

841
	.if \has_error_code == 0
842
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
843 844
	.endif

845
	ALLOC_PT_GPREGS_ON_STACK
846 847

	.if \paranoid
848
	.if \paranoid == 1
849 850
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
	jnz	1f
851
	.endif
852
	call	paranoid_entry
853
	.else
854
	call	error_entry
855
	.endif
856
	UNWIND_HINT_REGS
857
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
858 859

	.if \paranoid
860
	.if \shift_ist != -1
861
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
862
	.else
863
	TRACE_IRQS_OFF
864
	.endif
865
	.endif
866

867
	movq	%rsp, %rdi			/* pt_regs pointer */
868 869

	.if \has_error_code
870 871
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
872
	.else
873
	xorl	%esi, %esi			/* no error code */
874 875
	.endif

876
	.if \shift_ist != -1
877
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
878 879
	.endif

880
	call	\do_sym
881

882
	.if \shift_ist != -1
883
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
884 885
	.endif

886
	/* these procedures expect "no swapgs" flag in ebx */
887
	.if \paranoid
888
	jmp	paranoid_exit
889
	.else
890
	jmp	error_exit
891 892
	.endif

893 894 895 896 897 898 899
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
900
	call	error_entry
901 902


903 904 905
	movq	%rsp, %rdi			/* pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
906

907
	movq	%rsp, %rdi			/* pt_regs pointer */
908 909

	.if \has_error_code
910 911
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
912
	.else
913
	xorl	%esi, %esi			/* no error code */
914 915
	.endif

916
	call	\do_sym
917

918
	jmp	error_exit			/* %ebx: no swapgs flag */
919
	.endif
920
END(\sym)
921
.endm
922

923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
942
ENTRY(native_load_gs_index)
943
	FRAME_BEGIN
944
	pushfq
945
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
946
	SWAPGS
947
.Lgs_change:
948
	movl	%edi, %gs
949
2:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
950
	SWAPGS
951
	popfq
952
	FRAME_END
953
	ret
954
ENDPROC(native_load_gs_index)
955
EXPORT_SYMBOL(native_load_gs_index)
956

957
	_ASM_EXTABLE(.Lgs_change, bad_gs)
958
	.section .fixup, "ax"
L
Linus Torvalds 已提交
959
	/* running with kernelgs */
960
bad_gs:
961
	SWAPGS					/* switch back to user gs */
962 963 964 965 966 967
.macro ZAP_GS
	/* This can't be a string because the preprocessor needs to see it. */
	movl $__USER_DS, %eax
	movl %eax, %gs
.endm
	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
968 969 970
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
971
	.previous
972

973
/* Call softirq on interrupt stack. Interrupts are off. */
974
ENTRY(do_softirq_own_stack)
975 976
	pushq	%rbp
	mov	%rsp, %rbp
977
	ENTER_IRQ_STACK regs=0 old_rsp=%r11
978
	call	__do_softirq
979
	LEAVE_IRQ_STACK regs=0
980
	leaveq
981
	ret
982
ENDPROC(do_softirq_own_stack)
983

984
#ifdef CONFIG_XEN
985
idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0
986 987

/*
988 989 990 991 992 993 994 995 996 997 998 999
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
1000 1001
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

1002 1003 1004 1005
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
1006
	UNWIND_HINT_FUNC
1007
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
1008
	UNWIND_HINT_REGS
1009 1010

	ENTER_IRQ_STACK old_rsp=%r10
1011
	call	xen_evtchn_do_upcall
1012 1013
	LEAVE_IRQ_STACK

1014
#ifndef CONFIG_PREEMPT
1015
	call	xen_maybe_preempt_hcall
1016
#endif
1017
	jmp	error_exit
1018
END(xen_do_hypervisor_callback)
1019 1020

/*
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1033
ENTRY(xen_failsafe_callback)
1034
	UNWIND_HINT_EMPTY
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
1047
	/* All segments match their saved values => Category 2 (Bad IRET). */
1048 1049 1050 1051
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
1052
	UNWIND_HINT_IRET_REGS offset=8
1053
	jmp	general_protection
1054
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1055 1056 1057
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
1058
	UNWIND_HINT_IRET_REGS
1059
	pushq	$-1 /* orig_ax = -1 => not a system call */
1060 1061 1062
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
1063
	ENCODE_FRAME_POINTER
1064
	jmp	error_exit
1065 1066
END(xen_failsafe_callback)

1067
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1068 1069
	xen_hvm_callback_vector xen_evtchn_do_upcall

1070
#endif /* CONFIG_XEN */
1071

1072
#if IS_ENABLED(CONFIG_HYPERV)
1073
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1074 1075 1076
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

1077 1078 1079 1080
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

1081
#ifdef CONFIG_XEN
1082
idtentry xennmi			do_nmi			has_error_code=0
1083 1084
idtentry xendebug		do_debug		has_error_code=0
idtentry xenint3		do_int3			has_error_code=0
1085
#endif
1086 1087

idtentry general_protection	do_general_protection	has_error_code=1
1088
idtentry page_fault		do_page_fault		has_error_code=1
1089

G
Gleb Natapov 已提交
1090
#ifdef CONFIG_KVM_GUEST
1091
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
1092
#endif
1093

1094
#ifdef CONFIG_X86_MCE
1095
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
1096 1097
#endif

1098 1099 1100 1101 1102 1103
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1104
	UNWIND_HINT_FUNC
1105 1106 1107
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1108
	ENCODE_FRAME_POINTER 8
1109 1110
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
1111
	rdmsr
1112 1113
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
1114
	SWAPGS
1115
	xorl	%ebx, %ebx
1116
1:	ret
1117
END(paranoid_entry)
1118

1119 1120 1121 1122 1123 1124 1125 1126 1127
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1128 1129
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1130
 */
1131
ENTRY(paranoid_exit)
1132
	UNWIND_HINT_REGS
1133
	DISABLE_INTERRUPTS(CLBR_ANY)
1134
	TRACE_IRQS_OFF_DEBUG
1135
	testl	%ebx, %ebx			/* swapgs needed? */
1136
	jnz	.Lparanoid_exit_no_swapgs
1137
	TRACE_IRQS_IRETQ
1138
	SWAPGS_UNSAFE_STACK
1139 1140
	jmp	.Lparanoid_exit_restore
.Lparanoid_exit_no_swapgs:
1141
	TRACE_IRQS_IRETQ_DEBUG
1142 1143
.Lparanoid_exit_restore:
	jmp restore_regs_and_return_to_kernel
1144 1145 1146
END(paranoid_exit)

/*
1147
 * Save all registers in pt_regs, and switch gs if needed.
1148
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1149 1150
 */
ENTRY(error_entry)
1151
	UNWIND_HINT_FUNC
1152
	cld
1153 1154
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1155
	ENCODE_FRAME_POINTER 8
1156
	xorl	%ebx, %ebx
1157
	testb	$3, CS+8(%rsp)
1158
	jz	.Lerror_kernelspace
1159

1160 1161 1162 1163
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
1164
	SWAPGS
1165

1166
.Lerror_entry_from_usermode_after_swapgs:
1167 1168 1169 1170 1171 1172
	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).
	 */
	TRACE_IRQS_OFF
1173
	CALL_enter_from_user_mode
1174
	ret
1175

1176
.Lerror_entry_done:
1177 1178 1179
	TRACE_IRQS_OFF
	ret

1180 1181 1182 1183 1184 1185
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1186
.Lerror_kernelspace:
1187 1188 1189
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1190
	je	.Lerror_bad_iret
1191 1192
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1193
	je	.Lbstep_iret
1194
	cmpq	$.Lgs_change, RIP+8(%rsp)
1195
	jne	.Lerror_entry_done
1196 1197

	/*
1198
	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1199
	 * gsbase and proceed.  We'll fix up the exception and land in
1200
	 * .Lgs_change's error handler with kernel gsbase.
1201
	 */
1202 1203
	SWAPGS
	jmp .Lerror_entry_done
1204

1205
.Lbstep_iret:
1206
	/* Fix truncated RIP */
1207
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1208 1209
	/* fall through */

1210
.Lerror_bad_iret:
1211 1212 1213 1214
	/*
	 * We came from an IRET to user mode, so we have user gsbase.
	 * Switch to kernel gsbase:
	 */
A
Andy Lutomirski 已提交
1215
	SWAPGS
1216 1217 1218 1219 1220 1221

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1222 1223 1224
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1225
	decl	%ebx
1226
	jmp	.Lerror_entry_from_usermode_after_swapgs
1227 1228 1229
END(error_entry)


1230
/*
1231
 * On entry, EBX is a "return to kernel mode" flag:
1232 1233 1234
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1235
ENTRY(error_exit)
1236
	UNWIND_HINT_REGS
1237
	DISABLE_INTERRUPTS(CLBR_ANY)
1238
	TRACE_IRQS_OFF
1239
	testl	%ebx, %ebx
1240 1241
	jnz	retint_kernel
	jmp	retint_user
1242 1243
END(error_exit)

1244 1245 1246 1247
/*
 * Runs on exception stack.  Xen PV does not go through this path at all,
 * so we can use real assembly here.
 */
1248
ENTRY(nmi)
1249
	UNWIND_HINT_IRET_REGS
1250

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1268 1269 1270
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1271 1272
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1273
	 *    o Modify the "iret" location to jump to the repeat_nmi
1274 1275 1276 1277 1278 1279 1280 1281
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1282 1283 1284 1285 1286
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1287 1288
	 */

1289 1290
	ASM_CLAC

1291
	/* Use %rdx as our temp variable throughout */
1292
	pushq	%rdx
1293

1294 1295 1296 1297 1298 1299 1300 1301 1302
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
1303 1304 1305
	 *
	 * We also must not push anything to the stack before switching
	 * stacks lest we corrupt the "NMI executing" variable.
1306 1307
	 */

1308
	swapgs
1309 1310 1311
	cld
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1312
	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1313 1314 1315 1316 1317
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
1318
	UNWIND_HINT_IRET_REGS
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */
1335
	UNWIND_HINT_REGS
1336
	ENCODE_FRAME_POINTER
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1348
	/*
1349
	 * Return back to user mode.  We must *not* do the normal exit
1350
	 * work, because we don't want to enable interrupts.
1351
	 */
1352
	jmp	swapgs_restore_regs_and_return_to_usermode
1353

1354
.Lnmi_from_kernel:
1355
	/*
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1396
	/*
1397 1398
	 * Determine whether we're a nested NMI.
	 *
1399 1400 1401 1402 1403 1404
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1405
	 */
1406 1407 1408 1409 1410 1411 1412 1413

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1414

1415
	/*
1416
	 * Now check "NMI executing".  If it's set, then we're nested.
1417 1418
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1419
	 */
1420 1421
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1422 1423

	/*
1424 1425
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1426 1427 1428 1429 1430 1431 1432 1433
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1434
	 */
1435 1436 1437 1438 1439
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1440

1441 1442 1443 1444
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1445 1446 1447 1448 1449 1450 1451

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1452

1453 1454
nested_nmi:
	/*
1455 1456
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1457
	 */
1458
	subq	$8, %rsp
1459 1460 1461
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1462
	pushfq
1463 1464
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1465 1466

	/* Put stack back */
1467
	addq	$(6*8), %rsp
1468 1469

nested_nmi_out:
1470
	popq	%rdx
1471

1472
	/* We are returning to kernel mode, so this cannot result in a fault. */
1473
	iretq
1474 1475

first_nmi:
1476
	/* Restore rdx. */
1477
	movq	(%rsp), %rdx
1478

1479 1480
	/* Make room for "NMI executing". */
	pushq	$0
1481

1482
	/* Leave room for the "iret" frame */
1483
	subq	$(5*8), %rsp
1484

1485
	/* Copy the "original" frame to the "outermost" frame */
1486
	.rept 5
1487
	pushq	11*8(%rsp)
1488
	.endr
1489
	UNWIND_HINT_IRET_REGS
1490

1491 1492
	/* Everything up to here is safe from nested NMIs */

1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
1504
	iretq			/* continues at repeat_nmi below */
1505
	UNWIND_HINT_IRET_REGS
1506 1507 1508
1:
#endif

1509
repeat_nmi:
1510 1511 1512 1513 1514 1515 1516 1517
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1518 1519 1520 1521
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1522 1523
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1524
	 */
1525
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1526

1527
	/*
1528 1529 1530
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1531
	 */
1532
	addq	$(10*8), %rsp
1533
	.rept 5
1534
	pushq	-6*8(%rsp)
1535
	.endr
1536
	subq	$(5*8), %rsp
1537
end_repeat_nmi:
1538 1539

	/*
1540 1541 1542
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1543
	 */
1544
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1545 1546
	ALLOC_PT_GPREGS_ON_STACK

1547
	/*
1548
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1549 1550 1551 1552 1553
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1554
	call	paranoid_entry
1555
	UNWIND_HINT_REGS
1556

1557
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1558 1559 1560
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1561

1562 1563
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1564 1565 1566
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1567 1568
	POP_EXTRA_REGS
	POP_C_REGS
1569

1570 1571 1572 1573 1574
	/*
	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
	 * at the "iret" frame.
	 */
	addq	$6*8, %rsp
1575

1576 1577 1578
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
1579 1580 1581 1582 1583
	 * the SYSCALL entry and exit paths.
	 *
	 * We arguably should just inspect RIP instead, but I (Andy) wrote
	 * this code when I had the misapprehension that Xen PV supported
	 * NMIs, and Xen PV would break that approach.
1584 1585 1586
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1587 1588

	/*
1589 1590 1591 1592
	 * iretq reads the "iret" frame and exits the NMI stack in a
	 * single instruction.  We are returning to kernel mode, so this
	 * cannot result in a fault.  Similarly, we don't need to worry
	 * about espfix64 on the way back to kernel mode.
1593
	 */
1594
	iretq
1595 1596 1597
END(nmi)

ENTRY(ignore_sysret)
1598
	UNWIND_HINT_EMPTY
1599
	mov	$-ENOSYS, %eax
1600 1601
	sysret
END(ignore_sysret)
1602 1603

ENTRY(rewind_stack_do_exit)
1604
	UNWIND_HINT_FUNC
1605 1606 1607 1608
	/* Prevent any naive code from trying to unwind to our caller. */
	xorl	%ebp, %ebp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
1609 1610
	leaq	-PTREGS_SIZE(%rax), %rsp
	UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
1611 1612 1613

	call	do_exit
END(rewind_stack_do_exit)