entry_64.S 45.3 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4 5 6 7
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
8
 *
L
Linus Torvalds 已提交
9 10
 * entry.S contains the system-call and fault low-level handling routines.
 *
11 12
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
13
 * A note on terminology:
14 15
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
16 17
 *
 * Some macro usage:
18 19 20
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
21 22 23 24 25
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/smap.h>
37
#include <asm/pgtable_types.h>
38
#include <asm/export.h>
39
#include <asm/frame.h>
40
#include <asm/nospec-branch.h>
41
#include <linux/err.h>
L
Linus Torvalds 已提交
42

43 44
#include "calling.h"

45 46
.code64
.section .entry.text, "ax"
47

48
#ifdef CONFIG_PARAVIRT
49
ENTRY(native_usergs_sysret64)
50
	UNWIND_HINT_EMPTY
51 52
	swapgs
	sysretq
53
END(native_usergs_sysret64)
54 55
#endif /* CONFIG_PARAVIRT */

56
.macro TRACE_IRQS_FLAGS flags:req
57
#ifdef CONFIG_TRACE_IRQFLAGS
58
	bt	$9, \flags		/* interrupts off? */
59
	jnc	1f
60 61 62 63 64
	TRACE_IRQS_ON
1:
#endif
.endm

65 66 67 68
.macro TRACE_IRQS_IRETQ
	TRACE_IRQS_FLAGS EFLAGS(%rsp)
.endm

69 70 71 72 73 74 75 76 77 78 79 80 81 82
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
83
	call	debug_stack_set_zero
84
	TRACE_IRQS_OFF
85
	call	debug_stack_reset
86 87 88
.endm

.macro TRACE_IRQS_ON_DEBUG
89
	call	debug_stack_set_zero
90
	TRACE_IRQS_ON
91
	call	debug_stack_reset
92 93
.endm

94
.macro TRACE_IRQS_IRETQ_DEBUG
95 96
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
97 98 99 100 101
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
102 103 104
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
105 106
#endif

L
Linus Torvalds 已提交
107
/*
108
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
109
 *
110 111 112 113 114 115 116 117 118 119
 * This is the only entry point used for 64-bit system calls.  The
 * hardware interface is reasonably well designed and the register to
 * argument mapping Linux uses fits well with the registers that are
 * available when SYSCALL is used.
 *
 * SYSCALL instructions can be found inlined in libc implementations as
 * well as some other programs and libraries.  There are also a handful
 * of SYSCALL instructions in the vDSO used, for example, as a
 * clock_gettimeofday fallback.
 *
120
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
121 122 123 124 125 126
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
127
 * rax  system call number
128 129
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
130 131
 * rdi  arg0
 * rsi  arg1
132
 * rdx  arg2
133
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
134 135
 * r8   arg4
 * r9   arg5
136
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
137
 *
L
Linus Torvalds 已提交
138 139
 * Only called from user space.
 *
140
 * When user can change pt_regs->foo always force IRET. That is because
141 142
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
143
 */
L
Linus Torvalds 已提交
144

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
	.pushsection .entry_trampoline, "ax"

/*
 * The code in here gets remapped into cpu_entry_area's trampoline.  This means
 * that the assembler and linker have the wrong idea as to where this code
 * lives (and, in fact, it's mapped more than once, so it's not even at a
 * fixed address).  So we can't reference any symbols outside the entry
 * trampoline and expect it to work.
 *
 * Instead, we carefully abuse %rip-relative addressing.
 * _entry_trampoline(%rip) refers to the start of the remapped) entry
 * trampoline.  We can thus find cpu_entry_area with this macro:
 */

#define CPU_ENTRY_AREA \
	_entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip)

/* The top word of the SYSENTER stack is hot and is usable as scratch space. */
163 164
#define RSP_SCRATCH	CPU_ENTRY_AREA_entry_stack + \
			SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA
165 166 167 168 169 170 171 172

ENTRY(entry_SYSCALL_64_trampoline)
	UNWIND_HINT_EMPTY
	swapgs

	/* Stash the user RSP. */
	movq	%rsp, RSP_SCRATCH

173 174 175
	/* Note: using %rsp as a scratch reg. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp

176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	/* Load the top of the task stack into RSP */
	movq	CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp

	/* Start building the simulated IRET frame. */
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	RSP_SCRATCH			/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */

	/*
	 * x86 lacks a near absolute jump, and we can't jump to the real
	 * entry text with a relative jump.  We could push the target
	 * address and then use retq, but this destroys the pipeline on
	 * many CPUs (wasting over 20 cycles on Sandy Bridge).  Instead,
	 * spill RDI and restore it in a second-stage trampoline.
	 */
	pushq	%rdi
	movq	$entry_SYSCALL_64_stage2, %rdi
195
	JMP_NOSPEC %rdi
196 197 198 199 200 201 202 203 204 205
END(entry_SYSCALL_64_trampoline)

	.popsection

ENTRY(entry_SYSCALL_64_stage2)
	UNWIND_HINT_EMPTY
	popq	%rdi
	jmp	entry_SYSCALL_64_after_hwframe
END(entry_SYSCALL_64_stage2)

206
ENTRY(entry_SYSCALL_64)
207
	UNWIND_HINT_EMPTY
208 209 210 211 212
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
213

214
	swapgs
215
	/*
216
	 * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it
217 218
	 * is not required to switch CR3.
	 */
219 220
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
221 222

	/* Construct struct pt_regs on stack */
223 224 225 226 227
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
228
GLOBAL(entry_SYSCALL_64_after_hwframe)
229
	pushq	%rax				/* pt_regs->orig_ax */
230 231

	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
232

233 234
	TRACE_IRQS_OFF

235
	/* IRQs are off. */
236
	movq	%rsp, %rdi
237 238
	call	do_syscall_64		/* returns with IRQs disabled */

239
	TRACE_IRQS_IRETQ		/* we're about to change IF */
240 241 242

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
243 244
	 * a completely clean 64-bit userspace context.  If we're not,
	 * go to the slow exit path.
245
	 */
246 247
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
248 249 250

	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
	jne	swapgs_restore_regs_and_return_to_usermode
251 252 253 254

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
255
	 * the kernel, since userspace controls RSP.
256
	 *
257
	 * If width of "canonical tail" ever becomes variable, this will need
258
	 * to be updated to remain correct on both old and new CPUs.
259
	 *
260 261
	 * Change top bits to match most significant bit (47th or 56th bit
	 * depending on paging mode) in the address.
262
	 */
263 264
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
265

266 267
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
268
	jne	swapgs_restore_regs_and_return_to_usermode
269

270
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
271
	jne	swapgs_restore_regs_and_return_to_usermode
272

273 274
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
275
	jne	swapgs_restore_regs_and_return_to_usermode
276 277

	/*
278 279 280 281 282 283 284 285 286
	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
	 * restore RF properly. If the slowpath sets it for whatever reason, we
	 * need to restore it correctly.
	 *
	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
	 * trap from userspace immediately after SYSRET.  This would cause an
	 * infinite loop whenever #DB happens with register state that satisfies
	 * the opportunistic SYSRET conditions.  For example, single-stepping
	 * this user code:
287
	 *
288
	 *           movq	$stuck_here, %rcx
289 290 291 292 293 294
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
295
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
296
	jnz	swapgs_restore_regs_and_return_to_usermode
297 298 299

	/* nothing to check for RSP */

300
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
301
	jne	swapgs_restore_regs_and_return_to_usermode
302 303

	/*
304 305
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
306 307
	 */
syscall_return_via_sysret:
308
	/* rcx and r11 are already restored (see code above) */
309
	UNWIND_HINT_EMPTY
310
	POP_REGS pop_rdi=0 skip_r11rcx=1
311 312 313 314 315 316

	/*
	 * Now all regs are restored except RSP and RDI.
	 * Save old stack pointer and switch to trampoline stack.
	 */
	movq	%rsp, %rdi
317
	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
318 319 320 321 322 323 324 325

	pushq	RSP-RDI(%rdi)	/* RSP */
	pushq	(%rdi)		/* RDI */

	/*
	 * We are on the trampoline stack.  All regs except RDI are live.
	 * We can do future final exit work right here.
	 */
326
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
327

328
	popq	%rdi
329
	popq	%rsp
330
	USERGS_SYSRET64
331
END(entry_SYSCALL_64)
332

333 334 335 336 337
/*
 * %rdi: prev task
 * %rsi: next task
 */
ENTRY(__switch_to_asm)
338
	UNWIND_HINT_FUNC
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
	/*
	 * Save callee-saved registers
	 * This must match the order in inactive_task_frame
	 */
	pushq	%rbp
	pushq	%rbx
	pushq	%r12
	pushq	%r13
	pushq	%r14
	pushq	%r15

	/* switch stack */
	movq	%rsp, TASK_threadsp(%rdi)
	movq	TASK_threadsp(%rsi), %rsp

#ifdef CONFIG_CC_STACKPROTECTOR
	movq	TASK_stack_canary(%rsi), %rbx
	movq	%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
#endif

359 360 361 362 363 364 365 366
#ifdef CONFIG_RETPOLINE
	/*
	 * When switching from a shallower to a deeper call stack
	 * the RSB may either underflow or use entries populated
	 * with userspace addresses. On CPUs where those concerns
	 * exist, overwrite the RSB with entries which capture
	 * speculative execution to prevent attack.
	 */
367
	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
368 369
#endif

370 371 372 373 374 375 376 377 378 379 380
	/* restore callee-saved registers */
	popq	%r15
	popq	%r14
	popq	%r13
	popq	%r12
	popq	%rbx
	popq	%rbp

	jmp	__switch_to
END(__switch_to_asm)

381 382 383
/*
 * A newly forked process directly context switches into this address.
 *
384
 * rax: prev task we switched from
385 386
 * rbx: kernel thread func (NULL for user thread)
 * r12: kernel thread arg
387 388
 */
ENTRY(ret_from_fork)
389
	UNWIND_HINT_EMPTY
390
	movq	%rax, %rdi
391
	call	schedule_tail			/* rdi: 'prev' task parameter */
392

393 394
	testq	%rbx, %rbx			/* from kernel_thread? */
	jnz	1f				/* kernel threads are uncommon */
395

396
2:
397
	UNWIND_HINT_REGS
398
	movq	%rsp, %rdi
399 400
	call	syscall_return_slowpath	/* returns with IRQs disabled */
	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
401
	jmp	swapgs_restore_regs_and_return_to_usermode
402 403 404 405

1:
	/* kernel thread */
	movq	%r12, %rdi
406
	CALL_NOSPEC %rbx
407 408 409 410 411 412 413
	/*
	 * A kernel thread is allowed to return here after successfully
	 * calling do_execve().  Exit to userspace to complete the execve()
	 * syscall.
	 */
	movq	$0, RAX(%rsp)
	jmp	2b
414 415
END(ret_from_fork)

416
/*
417 418
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
419
 */
420
	.align 8
421
ENTRY(irq_entries_start)
422 423
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
424
	UNWIND_HINT_IRET_REGS
425
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
426 427
	jmp	common_interrupt
	.align	8
428
	vector=vector+1
429
    .endr
430 431
END(irq_entries_start)

432 433
.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
#ifdef CONFIG_DEBUG_ENTRY
434 435 436
	pushq %rax
	SAVE_FLAGS(CLBR_RAX)
	testl $X86_EFLAGS_IF, %eax
437 438 439
	jz .Lokay_\@
	ud2
.Lokay_\@:
440
	popq %rax
441 442 443 444 445 446 447 448 449 450
#endif
.endm

/*
 * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
 * Requires kernel GSBASE.
 *
 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
 */
451
.macro ENTER_IRQ_STACK regs=1 old_rsp
452 453
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	movq	%rsp, \old_rsp
454 455 456 457 458

	.if \regs
	UNWIND_HINT_REGS base=\old_rsp
	.endif

459
	incl	PER_CPU_VAR(irq_count)
460
	jnz	.Lirq_stack_push_old_rsp_\@
461 462 463 464 465 466 467 468 469

	/*
	 * Right now, if we just incremented irq_count to zero, we've
	 * claimed the IRQ stack but we haven't switched to it yet.
	 *
	 * If anything is added that can interrupt us here without using IST,
	 * it must be *extremely* careful to limit its stack usage.  This
	 * could include kprobes and a hypothetical future IST-less #DB
	 * handler.
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
	 *
	 * The OOPS unwinder relies on the word at the top of the IRQ
	 * stack linking back to the previous RSP for the entire time we're
	 * on the IRQ stack.  For this to work reliably, we need to write
	 * it before we actually move ourselves to the IRQ stack.
	 */

	movq	\old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
	movq	PER_CPU_VAR(irq_stack_ptr), %rsp

#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * If the first movq above becomes wrong due to IRQ stack layout
	 * changes, the only way we'll notice is if we try to unwind right
	 * here.  Assert that we set up the stack right to catch this type
	 * of bug quickly.
486
	 */
487 488 489 490 491
	cmpq	-8(%rsp), \old_rsp
	je	.Lirq_stack_okay\@
	ud2
	.Lirq_stack_okay\@:
#endif
492

493
.Lirq_stack_push_old_rsp_\@:
494
	pushq	\old_rsp
495 496 497 498

	.if \regs
	UNWIND_HINT_REGS indirect=1
	.endif
499 500 501 502 503
.endm

/*
 * Undoes ENTER_IRQ_STACK.
 */
504
.macro LEAVE_IRQ_STACK regs=1
505 506 507 508
	DEBUG_ENTRY_ASSERT_IRQS_OFF
	/* We need to be off the IRQ stack before decrementing irq_count. */
	popq	%rsp

509 510 511 512
	.if \regs
	UNWIND_HINT_REGS
	.endif

513 514 515 516 517 518 519 520
	/*
	 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
	 * the irq stack but we're not on it.
	 */

	decl	PER_CPU_VAR(irq_count)
.endm

521
/*
L
Linus Torvalds 已提交
522 523 524
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
525 526 527
 *
 * Entry runs with interrupts off.
 */
528 529 530 531 532 533 534 535
ENTRY(interrupt_entry)
	UNWIND_HINT_FUNC

	PUSH_AND_CLEAR_REGS save_ret=1
	ENCODE_FRAME_POINTER 8

	ret
END(interrupt_entry)
L
Linus Torvalds 已提交
536

537
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
538
	.macro interrupt func
539
	cld
540 541 542 543 544 545 546

	testb	$3, CS-ORIG_RAX(%rsp)
	jz	1f
	SWAPGS
	call	switch_to_thread_stack
1:

547
	call	interrupt_entry
548

549
	testb	$3, CS(%rsp)
550
	jz	1f
551 552

	/*
553 554
	 * IRQ from user mode.
	 *
555 556 557 558 559 560 561 562 563
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
	 * the simplest way to handle it is to just call it twice if
	 * we enter from user mode.  There's no reason to optimize this since
	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
	 */
	TRACE_IRQS_OFF

564
	CALL_enter_from_user_mode
565

566
1:
567
	ENTER_IRQ_STACK old_rsp=%rdi
568 569 570
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

571
	call	\func	/* rdi points to pt_regs */
L
Linus Torvalds 已提交
572 573
	.endm

574 575 576 577
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
578 579
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
580
	ASM_CLAC
581
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
582
	interrupt do_IRQ
583
	/* 0(%rsp): old RSP */
584
ret_from_intr:
585
	DISABLE_INTERRUPTS(CLBR_ANY)
586
	TRACE_IRQS_OFF
587

588
	LEAVE_IRQ_STACK
589

590
	testb	$3, CS(%rsp)
591
	jz	retint_kernel
592

593 594 595 596
	/* Interrupt came from user space */
GLOBAL(retint_user)
	mov	%rsp,%rdi
	call	prepare_exit_to_usermode
597
	TRACE_IRQS_IRETQ
598

599
GLOBAL(swapgs_restore_regs_and_return_to_usermode)
600 601
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates user mode. */
602
	testb	$3, CS(%rsp)
603 604 605 606
	jnz	1f
	ud2
1:
#endif
607
	POP_REGS pop_rdi=0
608 609 610 611 612 613

	/*
	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
	 * Save old stack pointer and switch to trampoline stack.
	 */
	movq	%rsp, %rdi
614
	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630

	/* Copy the IRET frame to the trampoline stack. */
	pushq	6*8(%rdi)	/* SS */
	pushq	5*8(%rdi)	/* RSP */
	pushq	4*8(%rdi)	/* EFLAGS */
	pushq	3*8(%rdi)	/* CS */
	pushq	2*8(%rdi)	/* RIP */

	/* Push user RDI on the trampoline stack. */
	pushq	(%rdi)

	/*
	 * We are on the trampoline stack.  All regs except RDI are live.
	 * We can do future final exit work right here.
	 */

631
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
632

633 634 635
	/* Restore RDI. */
	popq	%rdi
	SWAPGS
636 637
	INTERRUPT_RETURN

638

639
/* Returning to kernel space */
640
retint_kernel:
641 642 643
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
644
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
645
	jnc	1f
646
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
647
	jnz	1f
648
	call	preempt_schedule_irq
649
	jmp	0b
650
1:
651
#endif
652 653 654 655
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
656

657 658 659
GLOBAL(restore_regs_and_return_to_kernel)
#ifdef CONFIG_DEBUG_ENTRY
	/* Assert that pt_regs indicates kernel mode. */
660
	testb	$3, CS(%rsp)
661 662 663 664
	jz	1f
	ud2
1:
#endif
665
	POP_REGS
666
	addq	$8, %rsp	/* skip regs->orig_ax */
667 668 669 670
	/*
	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
	 * when returning from IPI handler.
	 */
671 672 673
	INTERRUPT_RETURN

ENTRY(native_iret)
674
	UNWIND_HINT_IRET_REGS
675 676 677 678
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
679
#ifdef CONFIG_X86_ESPFIX64
680 681
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
682
#endif
683

684
.global native_irq_return_iret
685
native_irq_return_iret:
A
Andy Lutomirski 已提交
686 687 688 689 690 691
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
692
	iretq
I
Ingo Molnar 已提交
693

694
#ifdef CONFIG_X86_ESPFIX64
695
native_irq_return_ldt:
696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
	/*
	 * We are running with user GSBASE.  All GPRs contain their user
	 * values.  We have a percpu ESPFIX stack that is eight slots
	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
	 * of the ESPFIX stack.
	 *
	 * We clobber RAX and RDI in this code.  We stash RDI on the
	 * normal stack and RAX on the ESPFIX stack.
	 *
	 * The ESPFIX stack layout we set up looks like this:
	 *
	 * --- top of ESPFIX stack ---
	 * SS
	 * RSP
	 * RFLAGS
	 * CS
	 * RIP  <-- RSP points here when we're done
	 * RAX  <-- espfix_waddr points here
	 * --- bottom of ESPFIX stack ---
	 */

	pushq	%rdi				/* Stash user RDI */
718 719 720
	SWAPGS					/* to kernel GS */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */

721
	movq	PER_CPU_VAR(espfix_waddr), %rdi
722 723
	movq	%rax, (0*8)(%rdi)		/* user RAX */
	movq	(1*8)(%rsp), %rax		/* user RIP */
724
	movq	%rax, (1*8)(%rdi)
725
	movq	(2*8)(%rsp), %rax		/* user CS */
726
	movq	%rax, (2*8)(%rdi)
727
	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
728
	movq	%rax, (3*8)(%rdi)
729
	movq	(5*8)(%rsp), %rax		/* user SS */
730
	movq	%rax, (5*8)(%rdi)
731
	movq	(4*8)(%rsp), %rax		/* user RSP */
732
	movq	%rax, (4*8)(%rdi)
733 734 735 736 737 738 739 740 741 742 743 744
	/* Now RAX == RSP. */

	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */

	/*
	 * espfix_stack[31:16] == 0.  The page tables are set up such that
	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
	 * the same page.  Set up RSP so that RSP[31:16] contains the
	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
	 * still points to an RO alias of the ESPFIX stack.
	 */
745
	orq	PER_CPU_VAR(espfix_stack), %rax
746

747
	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
748 749 750
	SWAPGS					/* to user GS */
	popq	%rdi				/* Restore user RDI */

751
	movq	%rax, %rsp
752
	UNWIND_HINT_IRET_REGS offset=8
753 754 755 756 757 758 759 760 761 762 763 764

	/*
	 * At this point, we cannot write to the stack any more, but we can
	 * still read.
	 */
	popq	%rax				/* Restore user RAX */

	/*
	 * RSP now points to an ordinary IRET frame, except that the page
	 * is read-only and RSP[31:16] are preloaded with the userspace
	 * values.  We can now IRET back to userspace.
	 */
765
	jmp	native_irq_return_iret
766
#endif
767
END(common_interrupt)
768

L
Linus Torvalds 已提交
769 770
/*
 * APIC interrupts.
771
 */
772
.macro apicinterrupt3 num sym do_sym
773
ENTRY(\sym)
774
	UNWIND_HINT_IRET_REGS
775
	ASM_CLAC
776
	pushq	$~(\num)
777
.Lcommon_\sym:
778
	interrupt \do_sym
779
	jmp	ret_from_intr
780 781
END(\sym)
.endm
L
Linus Torvalds 已提交
782

783
/* Make sure APIC interrupt handlers end up in the irqentry section: */
784 785
#define PUSH_SECTION_IRQENTRY	.pushsection .irqentry.text, "ax"
#define POP_SECTION_IRQENTRY	.popsection
786

787
.macro apicinterrupt num sym do_sym
788
PUSH_SECTION_IRQENTRY
789
apicinterrupt3 \num \sym \do_sym
790
POP_SECTION_IRQENTRY
791 792
.endm

793
#ifdef CONFIG_SMP
794 795
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
796
#endif
L
Linus Torvalds 已提交
797

N
Nick Piggin 已提交
798
#ifdef CONFIG_X86_UV
799
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
800
#endif
801 802 803

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
804

805
#ifdef CONFIG_HAVE_KVM
806 807
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
808
apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
809 810
#endif

811
#ifdef CONFIG_X86_MCE_THRESHOLD
812
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
813 814
#endif

815
#ifdef CONFIG_X86_MCE_AMD
816
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
817 818
#endif

819
#ifdef CONFIG_X86_THERMAL_VECTOR
820
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
821
#endif
822

823
#ifdef CONFIG_SMP
824 825 826
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
827
#endif
L
Linus Torvalds 已提交
828

829 830
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
831

832
#ifdef CONFIG_IRQ_WORK
833
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
834 835
#endif

L
Linus Torvalds 已提交
836 837
/*
 * Exception entry points.
838
 */
839
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8)
840

841 842 843 844 845 846 847 848 849
/*
 * Switch to the thread stack.  This is called with the IRET frame and
 * orig_ax on the stack.  (That is, RDI..R12 are not on the stack and
 * space has not been allocated for them.)
 */
ENTRY(switch_to_thread_stack)
	UNWIND_HINT_FUNC

	pushq	%rdi
850 851
	/* Need to switch before accessing the thread stack. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
	movq	%rsp, %rdi
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI

	pushq	7*8(%rdi)		/* regs->ss */
	pushq	6*8(%rdi)		/* regs->rsp */
	pushq	5*8(%rdi)		/* regs->eflags */
	pushq	4*8(%rdi)		/* regs->cs */
	pushq	3*8(%rdi)		/* regs->ip */
	pushq	2*8(%rdi)		/* regs->orig_ax */
	pushq	8(%rdi)			/* return address */
	UNWIND_HINT_FUNC

	movq	(%rdi), %rdi
	ret
END(switch_to_thread_stack)
868 869

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
870
ENTRY(\sym)
871
	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
872

873 874 875 876 877
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

878
	ASM_CLAC
879

880
	.if \has_error_code == 0
881
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
882 883
	.endif

884
	.if \paranoid < 2
885
	testb	$3, CS-ORIG_RAX(%rsp)		/* If coming from userspace, switch stacks */
886
	jnz	.Lfrom_usermode_switch_stack_\@
887
	.endif
888 889

	.if \paranoid
890
	call	paranoid_entry
891
	.else
892
	call	error_entry
893
	.endif
894
	UNWIND_HINT_REGS
895
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
896 897

	.if \paranoid
898
	.if \shift_ist != -1
899
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
900
	.else
901
	TRACE_IRQS_OFF
902
	.endif
903
	.endif
904

905
	movq	%rsp, %rdi			/* pt_regs pointer */
906 907

	.if \has_error_code
908 909
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
910
	.else
911
	xorl	%esi, %esi			/* no error code */
912 913
	.endif

914
	.if \shift_ist != -1
915
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
916 917
	.endif

918
	call	\do_sym
919

920
	.if \shift_ist != -1
921
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
922 923
	.endif

924
	/* these procedures expect "no swapgs" flag in ebx */
925
	.if \paranoid
926
	jmp	paranoid_exit
927
	.else
928
	jmp	error_exit
929 930
	.endif

931
	.if \paranoid < 2
932
	/*
933
	 * Entry from userspace.  Switch stacks and treat it
934 935 936
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
937
.Lfrom_usermode_switch_stack_\@:
938
	call	error_entry
939

940
	movq	%rsp, %rdi			/* pt_regs pointer */
941 942

	.if \has_error_code
943 944
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
945
	.else
946
	xorl	%esi, %esi			/* no error code */
947 948
	.endif

949
	call	\do_sym
950

951
	jmp	error_exit			/* %ebx: no swapgs flag */
952
	.endif
953
END(\sym)
954
.endm
955

956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
975
ENTRY(native_load_gs_index)
976
	FRAME_BEGIN
977
	pushfq
978
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
979
	TRACE_IRQS_OFF
980
	SWAPGS
981
.Lgs_change:
982
	movl	%edi, %gs
983
2:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
984
	SWAPGS
985
	TRACE_IRQS_FLAGS (%rsp)
986
	popfq
987
	FRAME_END
988
	ret
989
ENDPROC(native_load_gs_index)
990
EXPORT_SYMBOL(native_load_gs_index)
991

992
	_ASM_EXTABLE(.Lgs_change, bad_gs)
993
	.section .fixup, "ax"
L
Linus Torvalds 已提交
994
	/* running with kernelgs */
995
bad_gs:
996
	SWAPGS					/* switch back to user gs */
997 998 999 1000 1001 1002
.macro ZAP_GS
	/* This can't be a string because the preprocessor needs to see it. */
	movl $__USER_DS, %eax
	movl %eax, %gs
.endm
	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
1003 1004 1005
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
1006
	.previous
1007

1008
/* Call softirq on interrupt stack. Interrupts are off. */
1009
ENTRY(do_softirq_own_stack)
1010 1011
	pushq	%rbp
	mov	%rsp, %rbp
1012
	ENTER_IRQ_STACK regs=0 old_rsp=%r11
1013
	call	__do_softirq
1014
	LEAVE_IRQ_STACK regs=0
1015
	leaveq
1016
	ret
1017
ENDPROC(do_softirq_own_stack)
1018

1019
#ifdef CONFIG_XEN
1020
idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0
1021 1022

/*
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
1035 1036
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

1037 1038 1039 1040
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
1041
	UNWIND_HINT_FUNC
1042
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
1043
	UNWIND_HINT_REGS
1044 1045

	ENTER_IRQ_STACK old_rsp=%r10
1046
	call	xen_evtchn_do_upcall
1047 1048
	LEAVE_IRQ_STACK

1049
#ifndef CONFIG_PREEMPT
1050
	call	xen_maybe_preempt_hcall
1051
#endif
1052
	jmp	error_exit
1053
END(xen_do_hypervisor_callback)
1054 1055

/*
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1068
ENTRY(xen_failsafe_callback)
1069
	UNWIND_HINT_EMPTY
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
1082
	/* All segments match their saved values => Category 2 (Bad IRET). */
1083 1084 1085 1086
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
1087
	UNWIND_HINT_IRET_REGS offset=8
1088
	jmp	general_protection
1089
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1090 1091 1092
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
1093
	UNWIND_HINT_IRET_REGS
1094
	pushq	$-1 /* orig_ax = -1 => not a system call */
1095
	PUSH_AND_CLEAR_REGS
1096
	ENCODE_FRAME_POINTER
1097
	jmp	error_exit
1098 1099
END(xen_failsafe_callback)

1100
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1101 1102
	xen_hvm_callback_vector xen_evtchn_do_upcall

1103
#endif /* CONFIG_XEN */
1104

1105
#if IS_ENABLED(CONFIG_HYPERV)
1106
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1107
	hyperv_callback_vector hyperv_vector_handler
1108 1109 1110

apicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \
	hyperv_reenlightenment_vector hyperv_reenlightenment_intr
1111 1112
#endif /* CONFIG_HYPERV */

1113 1114 1115 1116
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

1117
#ifdef CONFIG_XEN
1118
idtentry xennmi			do_nmi			has_error_code=0
1119 1120
idtentry xendebug		do_debug		has_error_code=0
idtentry xenint3		do_int3			has_error_code=0
1121
#endif
1122 1123

idtentry general_protection	do_general_protection	has_error_code=1
1124
idtentry page_fault		do_page_fault		has_error_code=1
1125

G
Gleb Natapov 已提交
1126
#ifdef CONFIG_KVM_GUEST
1127
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
1128
#endif
1129

1130
#ifdef CONFIG_X86_MCE
1131
idtentry machine_check		do_mce			has_error_code=0	paranoid=1
1132 1133
#endif

1134
/*
1135
 * Save all registers in pt_regs, and switch gs if needed.
1136 1137 1138 1139
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1140
	UNWIND_HINT_FUNC
1141
	cld
1142 1143
	PUSH_AND_CLEAR_REGS save_ret=1
	ENCODE_FRAME_POINTER 8
1144 1145
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
1146
	rdmsr
1147 1148
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
1149
	SWAPGS
1150
	xorl	%ebx, %ebx
1151 1152 1153 1154 1155

1:
	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14

	ret
1156
END(paranoid_entry)
1157

1158 1159 1160 1161 1162 1163 1164 1165 1166
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1167 1168
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1169
 */
1170
ENTRY(paranoid_exit)
1171
	UNWIND_HINT_REGS
1172
	DISABLE_INTERRUPTS(CLBR_ANY)
1173
	TRACE_IRQS_OFF_DEBUG
1174
	testl	%ebx, %ebx			/* swapgs needed? */
1175
	jnz	.Lparanoid_exit_no_swapgs
1176
	TRACE_IRQS_IRETQ
P
Peter Zijlstra 已提交
1177
	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1178
	SWAPGS_UNSAFE_STACK
1179 1180
	jmp	.Lparanoid_exit_restore
.Lparanoid_exit_no_swapgs:
1181
	TRACE_IRQS_IRETQ_DEBUG
1182
	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1183 1184
.Lparanoid_exit_restore:
	jmp restore_regs_and_return_to_kernel
1185 1186 1187
END(paranoid_exit)

/*
1188
 * Save all registers in pt_regs, and switch GS if needed.
1189
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1190 1191
 */
ENTRY(error_entry)
1192
	UNWIND_HINT_FUNC
1193
	cld
1194 1195
	PUSH_AND_CLEAR_REGS save_ret=1
	ENCODE_FRAME_POINTER 8
1196
	testb	$3, CS+8(%rsp)
1197
	jz	.Lerror_kernelspace
1198

1199 1200 1201 1202
	/*
	 * We entered from user mode or we're pretending to have entered
	 * from user mode due to an IRET fault.
	 */
1203
	SWAPGS
1204 1205
	/* We have user CR3.  Change to kernel CR3. */
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1206

1207
.Lerror_entry_from_usermode_after_swapgs:
1208 1209 1210 1211 1212 1213 1214 1215
	/* Put us onto the real thread stack. */
	popq	%r12				/* save return addr in %12 */
	movq	%rsp, %rdi			/* arg0 = pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
	ENCODE_FRAME_POINTER
	pushq	%r12

1216 1217 1218 1219 1220 1221
	/*
	 * We need to tell lockdep that IRQs are off.  We can't do this until
	 * we fix gsbase, and we should do it before enter_from_user_mode
	 * (which can take locks).
	 */
	TRACE_IRQS_OFF
1222
	CALL_enter_from_user_mode
1223
	ret
1224

1225
.Lerror_entry_done:
1226 1227 1228
	TRACE_IRQS_OFF
	ret

1229 1230 1231 1232 1233 1234
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1235
.Lerror_kernelspace:
1236 1237 1238
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
1239
	je	.Lerror_bad_iret
1240 1241
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
1242
	je	.Lbstep_iret
1243
	cmpq	$.Lgs_change, RIP+8(%rsp)
1244
	jne	.Lerror_entry_done
1245 1246

	/*
1247
	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1248
	 * gsbase and proceed.  We'll fix up the exception and land in
1249
	 * .Lgs_change's error handler with kernel gsbase.
1250
	 */
1251
	SWAPGS
1252
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1253
	jmp .Lerror_entry_done
1254

1255
.Lbstep_iret:
1256
	/* Fix truncated RIP */
1257
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1258 1259
	/* fall through */

1260
.Lerror_bad_iret:
1261
	/*
1262 1263
	 * We came from an IRET to user mode, so we have user
	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
1264
	 */
A
Andy Lutomirski 已提交
1265
	SWAPGS
1266
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1267 1268 1269 1270 1271 1272

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1273 1274 1275
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1276
	decl	%ebx
1277
	jmp	.Lerror_entry_from_usermode_after_swapgs
1278 1279 1280
END(error_entry)


1281
/*
1282
 * On entry, EBX is a "return to kernel mode" flag:
1283 1284 1285
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1286
ENTRY(error_exit)
1287
	UNWIND_HINT_REGS
1288
	DISABLE_INTERRUPTS(CLBR_ANY)
1289
	TRACE_IRQS_OFF
1290
	testl	%ebx, %ebx
1291 1292
	jnz	retint_kernel
	jmp	retint_user
1293 1294
END(error_exit)

1295 1296 1297
/*
 * Runs on exception stack.  Xen PV does not go through this path at all,
 * so we can use real assembly here.
1298 1299 1300 1301
 *
 * Registers:
 *	%r14: Used to save/restore the CR3 of the interrupted context
 *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
1302
 */
1303
ENTRY(nmi)
1304
	UNWIND_HINT_IRET_REGS
1305

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1323 1324 1325
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1326 1327
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1328
	 *    o Modify the "iret" location to jump to the repeat_nmi
1329 1330 1331 1332 1333 1334 1335 1336
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1337 1338 1339 1340 1341
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1342 1343
	 */

1344 1345
	ASM_CLAC

1346
	/* Use %rdx as our temp variable throughout */
1347
	pushq	%rdx
1348

1349 1350 1351 1352 1353 1354 1355 1356 1357
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
1358 1359 1360
	 *
	 * We also must not push anything to the stack before switching
	 * stacks lest we corrupt the "NMI executing" variable.
1361 1362
	 */

1363
	swapgs
1364
	cld
1365
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1366 1367
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1368
	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1369 1370 1371 1372 1373
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
1374
	UNWIND_HINT_IRET_REGS
1375
	pushq   $-1		/* pt_regs->orig_ax */
1376
	PUSH_AND_CLEAR_REGS rdx=(%rdx)
1377
	ENCODE_FRAME_POINTER
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1389
	/*
1390
	 * Return back to user mode.  We must *not* do the normal exit
1391
	 * work, because we don't want to enable interrupts.
1392
	 */
1393
	jmp	swapgs_restore_regs_and_return_to_usermode
1394

1395
.Lnmi_from_kernel:
1396
	/*
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

1437
	/*
1438 1439
	 * Determine whether we're a nested NMI.
	 *
1440 1441 1442 1443 1444 1445
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
1446
	 */
1447 1448 1449 1450 1451 1452 1453 1454

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:
1455

1456
	/*
1457
	 * Now check "NMI executing".  If it's set, then we're nested.
1458 1459
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1460
	 */
1461 1462
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1463 1464

	/*
1465 1466
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
1467 1468 1469 1470 1471 1472 1473 1474
	 * "NMI executing" but before IRET.  We need to be careful, though:
	 * there is one case in which RSP could point to the NMI stack
	 * despite there being no NMI active: naughty userspace controls
	 * RSP at the very beginning of the SYSCALL targets.  We can
	 * pull a fast one on naughty userspace, though: we program
	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
	 * if it controls the kernel's RSP.  We set DF before we clear
	 * "NMI executing".
1475
	 */
1476 1477 1478 1479 1480
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1481

1482 1483 1484 1485
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
1486 1487 1488 1489 1490 1491 1492

	/* Ah, it is within the NMI stack. */

	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
	jz	first_nmi	/* RSP was user controlled. */

	/* This is a nested NMI. */
1493

1494 1495
nested_nmi:
	/*
1496 1497
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
1498
	 */
1499
	subq	$8, %rsp
1500 1501 1502
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1503
	pushfq
1504 1505
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1506 1507

	/* Put stack back */
1508
	addq	$(6*8), %rsp
1509 1510

nested_nmi_out:
1511
	popq	%rdx
1512

1513
	/* We are returning to kernel mode, so this cannot result in a fault. */
1514
	iretq
1515 1516

first_nmi:
1517
	/* Restore rdx. */
1518
	movq	(%rsp), %rdx
1519

1520 1521
	/* Make room for "NMI executing". */
	pushq	$0
1522

1523
	/* Leave room for the "iret" frame */
1524
	subq	$(5*8), %rsp
1525

1526
	/* Copy the "original" frame to the "outermost" frame */
1527
	.rept 5
1528
	pushq	11*8(%rsp)
1529
	.endr
1530
	UNWIND_HINT_IRET_REGS
1531

1532 1533
	/* Everything up to here is safe from nested NMIs */

1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
#ifdef CONFIG_DEBUG_ENTRY
	/*
	 * For ease of testing, unmask NMIs right away.  Disabled by
	 * default because IRET is very expensive.
	 */
	pushq	$0		/* SS */
	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
	addq	$8, (%rsp)	/* Fix up RSP */
	pushfq			/* RFLAGS */
	pushq	$__KERNEL_CS	/* CS */
	pushq	$1f		/* RIP */
1545
	iretq			/* continues at repeat_nmi below */
1546
	UNWIND_HINT_IRET_REGS
1547 1548 1549
1:
#endif

1550
repeat_nmi:
1551 1552 1553 1554 1555 1556 1557 1558
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1559 1560 1561 1562
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
1563 1564
	 * gsbase if needed before we call do_nmi.  "NMI executing"
	 * is zero.
1565
	 */
1566
	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1567

1568
	/*
1569 1570 1571
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
1572
	 */
1573
	addq	$(10*8), %rsp
1574
	.rept 5
1575
	pushq	-6*8(%rsp)
1576
	.endr
1577
	subq	$(5*8), %rsp
1578
end_repeat_nmi:
1579 1580

	/*
1581 1582 1583
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1584
	 */
1585
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1586

1587
	/*
1588
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1589 1590 1591 1592 1593
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1594
	call	paranoid_entry
1595
	UNWIND_HINT_REGS
1596

1597
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1598 1599 1600
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1601

P
Peter Zijlstra 已提交
1602
	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1603

1604 1605
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1606 1607 1608
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1609
	POP_REGS
1610

1611 1612 1613 1614 1615
	/*
	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
	 * at the "iret" frame.
	 */
	addq	$6*8, %rsp
1616

1617 1618 1619
	/*
	 * Clear "NMI executing".  Set DF first so that we can easily
	 * distinguish the remaining code between here and IRET from
1620 1621 1622 1623 1624
	 * the SYSCALL entry and exit paths.
	 *
	 * We arguably should just inspect RIP instead, but I (Andy) wrote
	 * this code when I had the misapprehension that Xen PV supported
	 * NMIs, and Xen PV would break that approach.
1625 1626 1627
	 */
	std
	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1628 1629

	/*
1630 1631 1632 1633
	 * iretq reads the "iret" frame and exits the NMI stack in a
	 * single instruction.  We are returning to kernel mode, so this
	 * cannot result in a fault.  Similarly, we don't need to worry
	 * about espfix64 on the way back to kernel mode.
1634
	 */
1635
	iretq
1636 1637 1638
END(nmi)

ENTRY(ignore_sysret)
1639
	UNWIND_HINT_EMPTY
1640
	mov	$-ENOSYS, %eax
1641 1642
	sysret
END(ignore_sysret)
1643 1644

ENTRY(rewind_stack_do_exit)
1645
	UNWIND_HINT_FUNC
1646 1647 1648 1649
	/* Prevent any naive code from trying to unwind to our caller. */
	xorl	%ebp, %ebp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
1650 1651
	leaq	-PTREGS_SIZE(%rax), %rsp
	UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
1652 1653 1654

	call	do_exit
END(rewind_stack_do_exit)