entry_64.S 41.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
7
 *
L
Linus Torvalds 已提交
8 9
 * entry.S contains the system-call and fault low-level handling routines.
 *
10 11
 * Some of this is documented in Documentation/x86/entry_64.txt
 *
12
 * A note on terminology:
13 14
 * - iret frame:	Architecture defined interrupt frame from SS to RIP
 *			at the top of the kernel process stack.
15 16
 *
 * Some macro usage:
17 18 19
 * - ENTRY/END:		Define functions in the symbol table.
 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
 * - idtentry:		Define exception entry points.
L
Linus Torvalds 已提交
20 21 22 23 24
 */
#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
25
#include "calling.h"
26
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
31
#include <asm/page_types.h>
32
#include <asm/irqflags.h>
33
#include <asm/paravirt.h>
34
#include <asm/percpu.h>
35
#include <asm/asm.h>
36
#include <asm/context_tracking.h>
37
#include <asm/smap.h>
38
#include <asm/pgtable_types.h>
39
#include <linux/err.h>
L
Linus Torvalds 已提交
40

R
Roland McGrath 已提交
41 42
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
43 44 45
#define AUDIT_ARCH_X86_64			(EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT			0x80000000
#define __AUDIT_ARCH_LE				0x40000000
J
Jiri Olsa 已提交
46

47 48
.code64
.section .entry.text, "ax"
49

50
#ifdef CONFIG_PARAVIRT
51
ENTRY(native_usergs_sysret64)
52 53
	swapgs
	sysretq
54
ENDPROC(native_usergs_sysret64)
55 56
#endif /* CONFIG_PARAVIRT */

57
.macro TRACE_IRQS_IRETQ
58
#ifdef CONFIG_TRACE_IRQFLAGS
59 60
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
61 62 63 64 65
	TRACE_IRQS_ON
1:
#endif
.endm

66 67 68 69 70 71 72 73 74 75 76 77 78 79
/*
 * When dynamic function tracer is enabled it will add a breakpoint
 * to all locations that it is about to modify, sync CPUs, update
 * all the code, sync CPUs, then remove the breakpoints. In this time
 * if lockdep is enabled, it might jump back into the debug handler
 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
 *
 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
 * make sure the stack pointer does not get reset back to the top
 * of the debug stack, and instead just reuses the current stack.
 */
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)

.macro TRACE_IRQS_OFF_DEBUG
80
	call	debug_stack_set_zero
81
	TRACE_IRQS_OFF
82
	call	debug_stack_reset
83 84 85
.endm

.macro TRACE_IRQS_ON_DEBUG
86
	call	debug_stack_set_zero
87
	TRACE_IRQS_ON
88
	call	debug_stack_reset
89 90
.endm

91
.macro TRACE_IRQS_IRETQ_DEBUG
92 93
	bt	$9, EFLAGS(%rsp)		/* interrupts off? */
	jnc	1f
94 95 96 97 98
	TRACE_IRQS_ON_DEBUG
1:
.endm

#else
99 100 101
# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
102 103
#endif

L
Linus Torvalds 已提交
104
/*
105
 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
L
Linus Torvalds 已提交
106
 *
107
 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
108 109 110 111 112 113
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Registers on entry:
L
Linus Torvalds 已提交
114
 * rax  system call number
115 116
 * rcx  return address
 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
L
Linus Torvalds 已提交
117 118
 * rdi  arg0
 * rsi  arg1
119
 * rdx  arg2
120
 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
L
Linus Torvalds 已提交
121 122
 * r8   arg4
 * r9   arg5
123
 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
124
 *
L
Linus Torvalds 已提交
125 126
 * Only called from user space.
 *
127
 * When user can change pt_regs->foo always force IRET. That is because
128 129
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
130
 */
L
Linus Torvalds 已提交
131

132
ENTRY(entry_SYSCALL_64)
133 134 135 136 137
	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
138 139 140 141 142 143
	SWAPGS_UNSAFE_STACK
	/*
	 * A hypervisor implementation might want to use a label
	 * after the swapgs, so that it can do the swapgs
	 * for the guest and jump here on syscall.
	 */
144
GLOBAL(entry_SYSCALL_64_after_swapgs)
145

146 147
	movq	%rsp, PER_CPU_VAR(rsp_scratch)
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
148 149

	/* Construct struct pt_regs on stack */
150 151
	pushq	$__USER_DS			/* pt_regs->ss */
	pushq	PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
152
	/*
153 154 155 156 157
	 * Re-enable interrupts.
	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
	 * must execute atomically in the face of possible interrupt-driven
	 * task preemption. We must enable interrupts only after we're done
	 * with using rsp_scratch:
158 159
	 */
	ENABLE_INTERRUPTS(CLBR_NONE)
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
	pushq	%r11				/* pt_regs->flags */
	pushq	$__USER_CS			/* pt_regs->cs */
	pushq	%rcx				/* pt_regs->ip */
	pushq	%rax				/* pt_regs->orig_ax */
	pushq	%rdi				/* pt_regs->di */
	pushq	%rsi				/* pt_regs->si */
	pushq	%rdx				/* pt_regs->dx */
	pushq	%rcx				/* pt_regs->cx */
	pushq	$-ENOSYS			/* pt_regs->ax */
	pushq	%r8				/* pt_regs->r8 */
	pushq	%r9				/* pt_regs->r9 */
	pushq	%r10				/* pt_regs->r10 */
	pushq	%r11				/* pt_regs->r11 */
	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */

	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	tracesys
177
entry_SYSCALL_64_fastpath:
178
#if __SYSCALL_MASK == ~0
179
	cmpq	$__NR_syscall_max, %rax
180
#else
181 182
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
183
#endif
184 185 186 187
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
188
1:
L
Linus Torvalds 已提交
189
/*
190 191
 * Syscall return path ending with SYSRET (fast path).
 * Has incompletely filled pt_regs.
192
 */
193
	LOCKDEP_SYS_EXIT
194 195 196 197
	/*
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
198
	DISABLE_INTERRUPTS(CLBR_NONE)
199 200 201 202 203 204 205 206 207

	/*
	 * We must check ti flags with interrupts (or at least preemption)
	 * off because we must *never* return to userspace without
	 * processing exit work that is enqueued if we're preempted here.
	 * In particular, returning to userspace with any of the one-shot
	 * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
	 * very bad.
	 */
208 209
	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	int_ret_from_sys_call_irqs_off	/* Go to the slow path */
210

211
	RESTORE_C_REGS_EXCEPT_RCX_R11
212 213 214
	movq	RIP(%rsp), %rcx
	movq	EFLAGS(%rsp), %r11
	movq	RSP(%rsp), %rsp
215
	/*
216
	 * 64-bit SYSRET restores rip from rcx,
217 218
	 * rflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
219
	 * Restoration of rflags re-enables interrupts.
220 221 222 223 224 225 226 227 228
	 *
	 * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
	 * descriptor is not reinitialized.  This means that we should
	 * avoid SYSRET with SS == NULL, which could happen if we schedule,
	 * exit the kernel, and re-enter using an interrupt vector.  (All
	 * interrupt entries on x86_64 set SS to NULL.)  We prevent that
	 * from happening by reloading SS in __switch_to.  (Actually
	 * detecting the failure in 64-bit userspace is tricky but can be
	 * done.)
229
	 */
230
	USERGS_SYSRET64
L
Linus Torvalds 已提交
231

232
	/* Do syscall entry tracing */
233
tracesys:
234 235 236 237 238 239 240 241
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	call	syscall_trace_enter_phase1
	test	%rax, %rax
	jnz	tracesys_phase2			/* if needed, run the slow path */
	RESTORE_C_REGS_EXCEPT_RAX		/* else restore clobbered regs */
	movq	ORIG_RAX(%rsp), %rax
	jmp	entry_SYSCALL_64_fastpath	/* and return to the fast path */
242 243

tracesys_phase2:
244
	SAVE_EXTRA_REGS
245 246 247 248
	movq	%rsp, %rdi
	movl	$AUDIT_ARCH_X86_64, %esi
	movq	%rax, %rdx
	call	syscall_trace_enter_phase2
249

250
	/*
D
Denys Vlasenko 已提交
251
	 * Reload registers from stack in case ptrace changed them.
252
	 * We don't reload %rax because syscall_trace_entry_phase2() returned
253 254
	 * the value it wants us to use in the table lookup.
	 */
255 256
	RESTORE_C_REGS_EXCEPT_RAX
	RESTORE_EXTRA_REGS
257
#if __SYSCALL_MASK == ~0
258
	cmpq	$__NR_syscall_max, %rax
259
#else
260 261
	andl	$__SYSCALL_MASK, %eax
	cmpl	$__NR_syscall_max, %eax
262
#endif
263 264 265 266
	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
	movq	%r10, %rcx			/* fixup for C */
	call	*sys_call_table(, %rax, 8)
	movq	%rax, RAX(%rsp)
267
1:
268
	/* Use IRET because user could have changed pt_regs->foo */
269 270

/*
L
Linus Torvalds 已提交
271
 * Syscall return path ending with IRET.
272
 * Has correct iret frame.
273
 */
274
GLOBAL(int_ret_from_sys_call)
275
	DISABLE_INTERRUPTS(CLBR_NONE)
276
int_ret_from_sys_call_irqs_off: /* jumps come here from the irqs-off SYSRET path */
277
	TRACE_IRQS_OFF
278
	movl	$_TIF_ALLWORK_MASK, %edi
L
Linus Torvalds 已提交
279
	/* edi:	mask to check */
280
GLOBAL(int_with_check)
281
	LOCKDEP_SYS_EXIT_IRQ
L
Linus Torvalds 已提交
282
	GET_THREAD_INFO(%rcx)
283 284 285 286
	movl	TI_flags(%rcx), %edx
	andl	%edi, %edx
	jnz	int_careful
	andl	$~TS_COMPAT, TI_status(%rcx)
287
	jmp	syscall_return
L
Linus Torvalds 已提交
288

289 290 291 292 293
	/*
	 * Either reschedule or signal or syscall exit tracking needed.
	 * First do a reschedule test.
	 * edx:	work, edi: workmask
	 */
L
Linus Torvalds 已提交
294
int_careful:
295 296
	bt	$TIF_NEED_RESCHED, %edx
	jnc	int_very_careful
297
	TRACE_IRQS_ON
298
	ENABLE_INTERRUPTS(CLBR_NONE)
299
	pushq	%rdi
300
	SCHEDULE_USER
301
	popq	%rdi
302
	DISABLE_INTERRUPTS(CLBR_NONE)
303
	TRACE_IRQS_OFF
304
	jmp	int_with_check
L
Linus Torvalds 已提交
305

306
	/* handle signals and tracing -- both require a full pt_regs */
L
Linus Torvalds 已提交
307
int_very_careful:
308
	TRACE_IRQS_ON
309
	ENABLE_INTERRUPTS(CLBR_NONE)
310
	SAVE_EXTRA_REGS
311
	/* Check for syscall exit trace */
312 313 314 315 316 317 318 319
	testl	$_TIF_WORK_SYSCALL_EXIT, %edx
	jz	int_signal
	pushq	%rdi
	leaq	8(%rsp), %rdi			/* &ptregs -> arg1 */
	call	syscall_trace_leave
	popq	%rdi
	andl	$~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU), %edi
	jmp	int_restore_rest
320

L
Linus Torvalds 已提交
321
int_signal:
322 323 324 325 326 327
	testl	$_TIF_DO_NOTIFY_MASK, %edx
	jz	1f
	movq	%rsp, %rdi			/* &ptregs -> arg1 */
	xorl	%esi, %esi			/* oldset -> arg2 */
	call	do_notify_resume
1:	movl	$_TIF_WORK_MASK, %edi
L
Linus Torvalds 已提交
328
int_restore_rest:
329
	RESTORE_EXTRA_REGS
330
	DISABLE_INTERRUPTS(CLBR_NONE)
331
	TRACE_IRQS_OFF
332
	jmp	int_with_check
333 334 335 336 337 338 339 340 341 342

syscall_return:
	/* The IRETQ could re-enable interrupts: */
	DISABLE_INTERRUPTS(CLBR_ANY)
	TRACE_IRQS_IRETQ

	/*
	 * Try to use SYSRET instead of IRET if we're returning to
	 * a completely clean 64-bit userspace context.
	 */
343 344 345 346
	movq	RCX(%rsp), %rcx
	movq	RIP(%rsp), %r11
	cmpq	%rcx, %r11			/* RCX == RIP */
	jne	opportunistic_sysret_failed
347 348 349 350

	/*
	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
	 * in kernel space.  This essentially lets the user take over
351
	 * the kernel, since userspace controls RSP.
352
	 *
353
	 * If width of "canonical tail" ever becomes variable, this will need
354 355 356 357 358
	 * to be updated to remain correct on both old and new CPUs.
	 */
	.ifne __VIRTUAL_MASK_SHIFT - 47
	.error "virtual address width changed -- SYSRET checks need update"
	.endif
359

360 361 362
	/* Change top 16 bits to be the sign-extension of 47th bit */
	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
363

364 365 366
	/* If this changed %rcx, it was not canonical */
	cmpq	%rcx, %r11
	jne	opportunistic_sysret_failed
367

368 369
	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
	jne	opportunistic_sysret_failed
370

371 372 373
	movq	R11(%rsp), %r11
	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
	jne	opportunistic_sysret_failed
374 375 376 377 378 379 380 381

	/*
	 * SYSRET can't restore RF.  SYSRET can restore TF, but unlike IRET,
	 * restoring TF results in a trap from userspace immediately after
	 * SYSRET.  This would cause an infinite loop whenever #DB happens
	 * with register state that satisfies the opportunistic SYSRET
	 * conditions.  For example, single-stepping this user code:
	 *
382
	 *           movq	$stuck_here, %rcx
383 384 385 386 387 388
	 *           pushfq
	 *           popq %r11
	 *   stuck_here:
	 *
	 * would never get past 'stuck_here'.
	 */
389 390
	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
	jnz	opportunistic_sysret_failed
391 392 393

	/* nothing to check for RSP */

394 395
	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
	jne	opportunistic_sysret_failed
396 397

	/*
398 399
	 * We win! This label is here just for ease of understanding
	 * perf profiles. Nothing jumps here.
400 401
	 */
syscall_return_via_sysret:
402 403
	/* rcx and r11 are already restored (see code above) */
	RESTORE_C_REGS_EXCEPT_RCX_R11
404
	movq	RSP(%rsp), %rsp
405 406 407 408 409
	USERGS_SYSRET64

opportunistic_sysret_failed:
	SWAPGS
	jmp	restore_c_regs_and_iret
410
END(entry_SYSCALL_64)
411

412

413 414
	.macro FORK_LIKE func
ENTRY(stub_\func)
415
	SAVE_EXTRA_REGS 8
416
	jmp	sys_\func
417 418 419 420 421 422
END(stub_\func)
	.endm

	FORK_LIKE  clone
	FORK_LIKE  fork
	FORK_LIKE  vfork
L
Linus Torvalds 已提交
423 424

ENTRY(stub_execve)
425 426 427 428 429 430 431 432 433 434
	call	sys_execve
return_from_execve:
	testl	%eax, %eax
	jz	1f
	/* exec failed, can use fast SYSRET code path in this case */
	ret
1:
	/* must use IRET code path (pt_regs->cs may have changed) */
	addq	$8, %rsp
	ZERO_EXTRA_REGS
435
	movq	%rax, RAX(%rsp)
436
	jmp	int_ret_from_sys_call
437
END(stub_execve)
438 439 440 441 442 443
/*
 * Remaining execve stubs are only 7 bytes long.
 * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
 */
	.align	8
GLOBAL(stub_execveat)
444 445
	call	sys_execveat
	jmp	return_from_execve
D
David Drysdale 已提交
446 447
END(stub_execveat)

448
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
449 450
	.align	8
GLOBAL(stub_x32_execve)
451
GLOBAL(stub32_execve)
452 453
	call	compat_sys_execve
	jmp	return_from_execve
454
END(stub32_execve)
455
END(stub_x32_execve)
456 457 458
	.align	8
GLOBAL(stub_x32_execveat)
GLOBAL(stub32_execveat)
459 460 461
	call	compat_sys_execveat
	jmp	return_from_execve
END(stub32_execveat)
462
END(stub_x32_execveat)
463 464
#endif

L
Linus Torvalds 已提交
465 466 467
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
468
 */
L
Linus Torvalds 已提交
469
ENTRY(stub_rt_sigreturn)
470 471 472 473 474 475 476 477
	/*
	 * SAVE_EXTRA_REGS result is not normally needed:
	 * sigreturn overwrites all pt_regs->GPREGS.
	 * But sigreturn can fail (!), and there is no easy way to detect that.
	 * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
	 * we SAVE_EXTRA_REGS here.
	 */
	SAVE_EXTRA_REGS 8
478
	call	sys_rt_sigreturn
479 480
return_from_stub:
	addq	$8, %rsp
481
	RESTORE_EXTRA_REGS
482 483
	movq	%rax, RAX(%rsp)
	jmp	int_ret_from_sys_call
484
END(stub_rt_sigreturn)
L
Linus Torvalds 已提交
485

486 487
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
488
	SAVE_EXTRA_REGS 8
489 490
	call	sys32_x32_rt_sigreturn
	jmp	return_from_stub
491 492 493
END(stub_x32_rt_sigreturn)
#endif

494 495 496 497 498 499 500
/*
 * A newly forked process directly context switches into this address.
 *
 * rdi: prev task we switched from
 */
ENTRY(ret_from_fork)

501
	LOCK ; btr $TIF_FORK, TI_flags(%r8)
502

503 504
	pushq	$0x0002
	popfq					/* reset kernel eflags */
505

506
	call	schedule_tail			/* rdi: 'prev' task parameter */
507 508 509

	RESTORE_EXTRA_REGS

510
	testb	$3, CS(%rsp)			/* from kernel_thread? */
511

512 513 514
	/*
	 * By the time we get here, we have no idea whether our pt_regs,
	 * ti flags, and ti status came from the 64-bit SYSCALL fast path,
515
	 * the slow path, or one of the 32-bit compat paths.
516
	 * Use IRET code path to return, since it can safely handle
517 518
	 * all of the above.
	 */
519
	jnz	int_ret_from_sys_call
520

521 522 523 524 525 526 527
	/*
	 * We came from kernel_thread
	 * nb: we depend on RESTORE_EXTRA_REGS above
	 */
	movq	%rbp, %rdi
	call	*%rbx
	movl	$0, RAX(%rsp)
528
	RESTORE_EXTRA_REGS
529
	jmp	int_ret_from_sys_call
530 531
END(ret_from_fork)

532
/*
533 534
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
535
 */
536
	.align 8
537
ENTRY(irq_entries_start)
538 539
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
540
	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
541 542 543 544
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
545 546
END(irq_entries_start)

547
/*
L
Linus Torvalds 已提交
548 549 550
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
551 552 553
 *
 * Entry runs with interrupts off.
 */
L
Linus Torvalds 已提交
554

555
/* 0(%rsp): ~(interrupt number) */
L
Linus Torvalds 已提交
556
	.macro interrupt func
557
	cld
D
Denys Vlasenko 已提交
558 559 560 561 562 563 564
	/*
	 * Since nothing in interrupt handling code touches r12...r15 members
	 * of "struct pt_regs", and since interrupts can nest, we can save
	 * four stack slots and simultaneously provide
	 * an unwind-friendly stack layout by saving "truncated" pt_regs
	 * exactly up to rbp slot, without these members.
	 */
565 566 567 568 569
	ALLOC_PT_GPREGS_ON_STACK -RBP
	SAVE_C_REGS -RBP
	/* this goes to 0(%rsp) for unwinder, not for saving the value: */
	SAVE_EXTRA_REGS_RBP -RBP

570
	leaq	-RBP(%rsp), %rdi		/* arg1 for \func (pointer to pt_regs) */
571

572
	testb	$3, CS-RBP(%rsp)
573
	jz	1f
574
	SWAPGS
575
1:
576
	/*
D
Denys Vlasenko 已提交
577
	 * Save previous stack pointer, optionally switch to interrupt stack.
578 579 580 581 582
	 * irq_count is used to check if a CPU is already on an interrupt stack
	 * or not. While this is essentially redundant with preempt_count it is
	 * a little cheaper to use a separate counter in the PDA (short of
	 * moving irq_enter into assembly, which would be too much work)
	 */
583 584 585 586
	movq	%rsp, %rsi
	incl	PER_CPU_VAR(irq_count)
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
	pushq	%rsi
587 588 589
	/* We entered an interrupt context - irqs are off: */
	TRACE_IRQS_OFF

590
	call	\func
L
Linus Torvalds 已提交
591 592
	.endm

593 594 595 596
	/*
	 * The interrupt stubs push (~vector+0x80) onto the stack and
	 * then jump to common_interrupt.
	 */
597 598
	.p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt:
599
	ASM_CLAC
600
	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
L
Linus Torvalds 已提交
601
	interrupt do_IRQ
602
	/* 0(%rsp): old RSP */
603
ret_from_intr:
604
	DISABLE_INTERRUPTS(CLBR_NONE)
605
	TRACE_IRQS_OFF
606
	decl	PER_CPU_VAR(irq_count)
607

608
	/* Restore saved previous stack */
609
	popq	%rsi
D
Denys Vlasenko 已提交
610
	/* return code expects complete pt_regs - adjust rsp accordingly: */
611
	leaq	-RBP(%rsi), %rsp
612

613
	testb	$3, CS(%rsp)
614
	jz	retint_kernel
L
Linus Torvalds 已提交
615
	/* Interrupt came from user space */
616
retint_user:
617
	GET_THREAD_INFO(%rcx)
618 619

	/* %rcx: thread info. Interrupts are off. */
L
Linus Torvalds 已提交
620
retint_with_reschedule:
621
	movl	$_TIF_WORK_MASK, %edi
622
retint_check:
623
	LOCKDEP_SYS_EXIT_IRQ
624 625 626
	movl	TI_flags(%rcx), %edx
	andl	%edi, %edx
	jnz	retint_careful
627

628
retint_swapgs:					/* return to user-space */
629 630 631
	/*
	 * The iretq could re-enable interrupts:
	 */
632
	DISABLE_INTERRUPTS(CLBR_ANY)
633
	TRACE_IRQS_IRETQ
634

635
	SWAPGS
636
	jmp	restore_c_regs_and_iret
637

638
/* Returning to kernel space */
639
retint_kernel:
640 641 642
#ifdef CONFIG_PREEMPT
	/* Interrupts are off */
	/* Check if we need preemption */
643
	bt	$9, EFLAGS(%rsp)		/* were interrupts off? */
644
	jnc	1f
645
0:	cmpl	$0, PER_CPU_VAR(__preempt_count)
646
	jnz	1f
647
	call	preempt_schedule_irq
648
	jmp	0b
649
1:
650
#endif
651 652 653 654
	/*
	 * The iretq could re-enable interrupts:
	 */
	TRACE_IRQS_IRETQ
655 656 657 658 659 660

/*
 * At this label, code paths which return to kernel and to user,
 * which come from interrupts/exception and from syscalls, merge.
 */
restore_c_regs_and_iret:
661 662
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
663 664 665
	INTERRUPT_RETURN

ENTRY(native_iret)
666 667 668 669
	/*
	 * Are we returning to a stack segment from the LDT?  Note: in
	 * 64-bit mode SS:RSP on the exception stack is always valid.
	 */
670
#ifdef CONFIG_X86_ESPFIX64
671 672
	testb	$4, (SS-RIP)(%rsp)
	jnz	native_irq_return_ldt
673
#endif
674

675
.global native_irq_return_iret
676
native_irq_return_iret:
A
Andy Lutomirski 已提交
677 678 679 680 681 682
	/*
	 * This may fault.  Non-paranoid faults on return to userspace are
	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
	 * Double-faults due to espfix64 are handled in do_double_fault.
	 * Other faults here are fatal.
	 */
L
Linus Torvalds 已提交
683
	iretq
I
Ingo Molnar 已提交
684

685
#ifdef CONFIG_X86_ESPFIX64
686
native_irq_return_ldt:
687 688
	pushq	%rax
	pushq	%rdi
689
	SWAPGS
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
	movq	PER_CPU_VAR(espfix_waddr), %rdi
	movq	%rax, (0*8)(%rdi)		/* RAX */
	movq	(2*8)(%rsp), %rax		/* RIP */
	movq	%rax, (1*8)(%rdi)
	movq	(3*8)(%rsp), %rax		/* CS */
	movq	%rax, (2*8)(%rdi)
	movq	(4*8)(%rsp), %rax		/* RFLAGS */
	movq	%rax, (3*8)(%rdi)
	movq	(6*8)(%rsp), %rax		/* SS */
	movq	%rax, (5*8)(%rdi)
	movq	(5*8)(%rsp), %rax		/* RSP */
	movq	%rax, (4*8)(%rdi)
	andl	$0xffff0000, %eax
	popq	%rdi
	orq	PER_CPU_VAR(espfix_stack), %rax
705
	SWAPGS
706 707 708
	movq	%rax, %rsp
	popq	%rax
	jmp	native_irq_return_iret
709
#endif
710

711
	/* edi: workmask, edx: work */
L
Linus Torvalds 已提交
712
retint_careful:
713 714
	bt	$TIF_NEED_RESCHED, %edx
	jnc	retint_signal
715
	TRACE_IRQS_ON
716
	ENABLE_INTERRUPTS(CLBR_NONE)
717
	pushq	%rdi
718
	SCHEDULE_USER
719
	popq	%rdi
L
Linus Torvalds 已提交
720
	GET_THREAD_INFO(%rcx)
721
	DISABLE_INTERRUPTS(CLBR_NONE)
722
	TRACE_IRQS_OFF
723
	jmp	retint_check
724

L
Linus Torvalds 已提交
725
retint_signal:
726 727
	testl	$_TIF_DO_NOTIFY_MASK, %edx
	jz	retint_swapgs
728
	TRACE_IRQS_ON
729
	ENABLE_INTERRUPTS(CLBR_NONE)
730
	SAVE_EXTRA_REGS
731 732 733 734
	movq	$-1, ORIG_RAX(%rsp)
	xorl	%esi, %esi			/* oldset */
	movq	%rsp, %rdi			/* &pt_regs */
	call	do_notify_resume
735
	RESTORE_EXTRA_REGS
736
	DISABLE_INTERRUPTS(CLBR_NONE)
737
	TRACE_IRQS_OFF
738
	GET_THREAD_INFO(%rcx)
739
	jmp	retint_with_reschedule
L
Linus Torvalds 已提交
740

741
END(common_interrupt)
742

L
Linus Torvalds 已提交
743 744
/*
 * APIC interrupts.
745
 */
746
.macro apicinterrupt3 num sym do_sym
747
ENTRY(\sym)
748
	ASM_CLAC
749
	pushq	$~(\num)
750
.Lcommon_\sym:
751
	interrupt \do_sym
752
	jmp	ret_from_intr
753 754
END(\sym)
.endm
L
Linus Torvalds 已提交
755

756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
#ifdef CONFIG_TRACING
#define trace(sym) trace_##sym
#define smp_trace(sym) smp_trace_##sym

.macro trace_apicinterrupt num sym
apicinterrupt3 \num trace(\sym) smp_trace(\sym)
.endm
#else
.macro trace_apicinterrupt num sym do_sym
.endm
#endif

.macro apicinterrupt num sym do_sym
apicinterrupt3 \num \sym \do_sym
trace_apicinterrupt \num \sym
.endm

773
#ifdef CONFIG_SMP
774 775
apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
776
#endif
L
Linus Torvalds 已提交
777

N
Nick Piggin 已提交
778
#ifdef CONFIG_X86_UV
779
apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
N
Nick Piggin 已提交
780
#endif
781 782 783

apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
784

785
#ifdef CONFIG_HAVE_KVM
786 787
apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
788 789
#endif

790
#ifdef CONFIG_X86_MCE_THRESHOLD
791
apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
792 793
#endif

794
#ifdef CONFIG_X86_MCE_AMD
795
apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
796 797
#endif

798
#ifdef CONFIG_X86_THERMAL_VECTOR
799
apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
800
#endif
801

802
#ifdef CONFIG_SMP
803 804 805
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
806
#endif
L
Linus Torvalds 已提交
807

808 809
apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
810

811
#ifdef CONFIG_IRQ_WORK
812
apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
I
Ingo Molnar 已提交
813 814
#endif

L
Linus Torvalds 已提交
815 816
/*
 * Exception entry points.
817
 */
818
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
819 820

.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
821
ENTRY(\sym)
822 823 824 825 826
	/* Sanity check */
	.if \shift_ist != -1 && \paranoid == 0
	.error "using shift_ist requires paranoid=1"
	.endif

827
	ASM_CLAC
828
	PARAVIRT_ADJUST_EXCEPTION_FRAME
829 830

	.ifeq \has_error_code
831
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
832 833
	.endif

834
	ALLOC_PT_GPREGS_ON_STACK
835 836

	.if \paranoid
837
	.if \paranoid == 1
838 839
	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
	jnz	1f
840
	.endif
841
	call	paranoid_entry
842
	.else
843
	call	error_entry
844
	.endif
845
	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
846 847

	.if \paranoid
848
	.if \shift_ist != -1
849
	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
850
	.else
851
	TRACE_IRQS_OFF
852
	.endif
853
	.endif
854

855
	movq	%rsp, %rdi			/* pt_regs pointer */
856 857

	.if \has_error_code
858 859
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
860
	.else
861
	xorl	%esi, %esi			/* no error code */
862 863
	.endif

864
	.if \shift_ist != -1
865
	subq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
866 867
	.endif

868
	call	\do_sym
869

870
	.if \shift_ist != -1
871
	addq	$EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
872 873
	.endif

874
	/* these procedures expect "no swapgs" flag in ebx */
875
	.if \paranoid
876
	jmp	paranoid_exit
877
	.else
878
	jmp	error_exit
879 880
	.endif

881 882 883 884 885 886 887
	.if \paranoid == 1
	/*
	 * Paranoid entry from userspace.  Switch stacks and treat it
	 * as a normal entry.  This means that paranoid handlers
	 * run in real process context if user_mode(regs).
	 */
1:
888
	call	error_entry
889 890


891 892 893
	movq	%rsp, %rdi			/* pt_regs pointer */
	call	sync_regs
	movq	%rax, %rsp			/* switch stack */
894

895
	movq	%rsp, %rdi			/* pt_regs pointer */
896 897

	.if \has_error_code
898 899
	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
900
	.else
901
	xorl	%esi, %esi			/* no error code */
902 903
	.endif

904
	call	\do_sym
905

906
	jmp	error_exit			/* %ebx: no swapgs flag */
907
	.endif
908
END(\sym)
909
.endm
910

911
#ifdef CONFIG_TRACING
912 913 914
.macro trace_idtentry sym do_sym has_error_code:req
idtentry trace(\sym) trace(\do_sym) has_error_code=\has_error_code
idtentry \sym \do_sym has_error_code=\has_error_code
915 916
.endm
#else
917 918
.macro trace_idtentry sym do_sym has_error_code:req
idtentry \sym \do_sym has_error_code=\has_error_code
919 920 921
.endm
#endif

922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
idtentry divide_error			do_divide_error			has_error_code=0
idtentry overflow			do_overflow			has_error_code=0
idtentry bounds				do_bounds			has_error_code=0
idtentry invalid_op			do_invalid_op			has_error_code=0
idtentry device_not_available		do_device_not_available		has_error_code=0
idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
idtentry segment_not_present		do_segment_not_present		has_error_code=1
idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
idtentry alignment_check		do_alignment_check		has_error_code=1
idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0


	/*
	 * Reload gs selector with exception handling
	 * edi:  new selector
	 */
941
ENTRY(native_load_gs_index)
942
	pushfq
943
	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
944
	SWAPGS
945
gs_change:
946 947
	movl	%edi, %gs
2:	mfence					/* workaround */
948
	SWAPGS
949
	popfq
950
	ret
951
END(native_load_gs_index)
952

953 954
	_ASM_EXTABLE(gs_change, bad_gs)
	.section .fixup, "ax"
L
Linus Torvalds 已提交
955
	/* running with kernelgs */
956
bad_gs:
957 958 959 960
	SWAPGS					/* switch back to user gs */
	xorl	%eax, %eax
	movl	%eax, %gs
	jmp	2b
961
	.previous
962

963
/* Call softirq on interrupt stack. Interrupts are off. */
964
ENTRY(do_softirq_own_stack)
965 966 967 968 969 970
	pushq	%rbp
	mov	%rsp, %rbp
	incl	PER_CPU_VAR(irq_count)
	cmove	PER_CPU_VAR(irq_stack_ptr), %rsp
	push	%rbp				/* frame pointer backlink */
	call	__do_softirq
971
	leaveq
972
	decl	PER_CPU_VAR(irq_count)
973
	ret
974
END(do_softirq_own_stack)
975

976
#ifdef CONFIG_XEN
977
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
978 979

/*
980 981 982 983 984 985 986 987 988 989 990 991
 * A note on the "critical region" in our callback handler.
 * We want to avoid stacking callback handlers due to events occurring
 * during handling of the last event. To do this, we keep events disabled
 * until we've done all processing. HOWEVER, we must enable events before
 * popping the stack frame (can't be done atomically) and so it would still
 * be possible to get enough handler activations to overflow the stack.
 * Although unlikely, bugs of that kind are hard to track down, so we'd
 * like to avoid the possibility.
 * So, on entry to the handler we detect whether we interrupted an
 * existing activation in its critical region -- if so, we pop the current
 * activation and restart the handler using the previous one.
 */
992 993
ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */

994 995 996 997
/*
 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
 * see the correct pointer to the pt_regs
 */
998 999 1000 1001 1002 1003 1004 1005
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
11:	incl	PER_CPU_VAR(irq_count)
	movq	%rsp, %rbp
	cmovzq	PER_CPU_VAR(irq_stack_ptr), %rsp
	pushq	%rbp				/* frame pointer backlink */
	call	xen_evtchn_do_upcall
	popq	%rsp
	decl	PER_CPU_VAR(irq_count)
1006
#ifndef CONFIG_PREEMPT
1007
	call	xen_maybe_preempt_hcall
1008
#endif
1009
	jmp	error_exit
1010
END(xen_do_hypervisor_callback)
1011 1012

/*
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we do not need to fix up as Xen has already reloaded all segment
 * registers that could be reloaded and zeroed the others.
 * Category 2 we fix up by killing the current process. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by comparing each saved segment register
 * with its current contents: any discrepancy means we in category 1.
 */
1025
ENTRY(xen_failsafe_callback)
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	movl	%ds, %ecx
	cmpw	%cx, 0x10(%rsp)
	jne	1f
	movl	%es, %ecx
	cmpw	%cx, 0x18(%rsp)
	jne	1f
	movl	%fs, %ecx
	cmpw	%cx, 0x20(%rsp)
	jne	1f
	movl	%gs, %ecx
	cmpw	%cx, 0x28(%rsp)
	jne	1f
1038
	/* All segments match their saved values => Category 2 (Bad IRET). */
1039 1040 1041 1042 1043 1044 1045
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$0				/* RIP */
	pushq	%r11
	pushq	%rcx
	jmp	general_protection
1046
1:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1047 1048 1049 1050
	movq	(%rsp), %rcx
	movq	8(%rsp), %r11
	addq	$0x30, %rsp
	pushq	$-1 /* orig_ax = -1 => not a system call */
1051 1052 1053
	ALLOC_PT_GPREGS_ON_STACK
	SAVE_C_REGS
	SAVE_EXTRA_REGS
1054
	jmp	error_exit
1055 1056
END(xen_failsafe_callback)

1057
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1058 1059
	xen_hvm_callback_vector xen_evtchn_do_upcall

1060
#endif /* CONFIG_XEN */
1061

1062
#if IS_ENABLED(CONFIG_HYPERV)
1063
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1064 1065 1066
	hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */

1067 1068 1069 1070
idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
idtentry stack_segment		do_stack_segment	has_error_code=1

1071
#ifdef CONFIG_XEN
1072 1073 1074
idtentry xen_debug		do_debug		has_error_code=0
idtentry xen_int3		do_int3			has_error_code=0
idtentry xen_stack_segment	do_stack_segment	has_error_code=1
1075
#endif
1076 1077 1078 1079

idtentry general_protection	do_general_protection	has_error_code=1
trace_idtentry page_fault	do_page_fault		has_error_code=1

G
Gleb Natapov 已提交
1080
#ifdef CONFIG_KVM_GUEST
1081
idtentry async_page_fault	do_async_page_fault	has_error_code=1
G
Gleb Natapov 已提交
1082
#endif
1083

1084
#ifdef CONFIG_X86_MCE
1085
idtentry machine_check					has_error_code=0	paranoid=1 do_sym=*machine_check_vector(%rip)
1086 1087
#endif

1088 1089 1090 1091 1092 1093
/*
 * Save all registers in pt_regs, and switch gs if needed.
 * Use slow, but surefire "are we in kernel?" check.
 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 */
ENTRY(paranoid_entry)
1094 1095 1096
	cld
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1097 1098
	movl	$1, %ebx
	movl	$MSR_GS_BASE, %ecx
1099
	rdmsr
1100 1101
	testl	%edx, %edx
	js	1f				/* negative -> in kernel */
1102
	SWAPGS
1103
	xorl	%ebx, %ebx
1104
1:	ret
1105
END(paranoid_entry)
1106

1107 1108 1109 1110 1111 1112 1113 1114 1115
/*
 * "Paranoid" exit path from exception stack.  This is invoked
 * only on return from non-NMI IST interrupts that came
 * from kernel space.
 *
 * We may be returning to very strange contexts (e.g. very early
 * in syscall entry), so checking for preemption here would
 * be complicated.  Fortunately, we there's no good reason
 * to try to handle preemption here.
1116 1117
 *
 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1118
 */
1119 1120
ENTRY(paranoid_exit)
	DISABLE_INTERRUPTS(CLBR_NONE)
1121
	TRACE_IRQS_OFF_DEBUG
1122 1123
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	paranoid_exit_no_swapgs
1124
	TRACE_IRQS_IRETQ
1125
	SWAPGS_UNSAFE_STACK
1126
	jmp	paranoid_exit_restore
1127
paranoid_exit_no_swapgs:
1128
	TRACE_IRQS_IRETQ_DEBUG
1129
paranoid_exit_restore:
1130 1131 1132
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
	REMOVE_PT_GPREGS_FROM_STACK 8
1133
	INTERRUPT_RETURN
1134 1135 1136
END(paranoid_exit)

/*
1137
 * Save all registers in pt_regs, and switch gs if needed.
1138
 * Return: EBX=0: came from user mode; EBX=1: otherwise
1139 1140 1141
 */
ENTRY(error_entry)
	cld
1142 1143
	SAVE_C_REGS 8
	SAVE_EXTRA_REGS 8
1144
	xorl	%ebx, %ebx
1145
	testb	$3, CS+8(%rsp)
1146
	jz	error_kernelspace
1147 1148

	/* We entered from user mode */
1149
	SWAPGS
1150 1151

error_entry_done:
1152 1153 1154
	TRACE_IRQS_OFF
	ret

1155 1156 1157 1158 1159 1160
	/*
	 * There are two places in the kernel that can potentially fault with
	 * usergs. Handle them here.  B stepping K8s sometimes report a
	 * truncated RIP for IRET exceptions returning to compat mode. Check
	 * for these here too.
	 */
1161
error_kernelspace:
1162 1163 1164 1165 1166 1167 1168 1169
	incl	%ebx
	leaq	native_irq_return_iret(%rip), %rcx
	cmpq	%rcx, RIP+8(%rsp)
	je	error_bad_iret
	movl	%ecx, %eax			/* zero extend */
	cmpq	%rax, RIP+8(%rsp)
	je	bstep_iret
	cmpq	$gs_change, RIP+8(%rsp)
1170 1171 1172 1173 1174 1175 1176 1177 1178
	jne	error_entry_done

	/*
	 * hack: gs_change can fail with user gsbase.  If this happens, fix up
	 * gsbase and proceed.  We'll fix up the exception and land in
	 * gs_change's error handler with kernel gsbase.
	 */
	SWAPGS
	jmp	error_entry_done
1179 1180 1181

bstep_iret:
	/* Fix truncated RIP */
1182
	movq	%rcx, RIP+8(%rsp)
A
Andy Lutomirski 已提交
1183 1184 1185
	/* fall through */

error_bad_iret:
1186 1187 1188 1189
	/*
	 * We came from an IRET to user mode, so we have user gsbase.
	 * Switch to kernel gsbase:
	 */
A
Andy Lutomirski 已提交
1190
	SWAPGS
1191 1192 1193 1194 1195 1196

	/*
	 * Pretend that the exception came from user mode: set up pt_regs
	 * as if we faulted immediately after IRET and clear EBX so that
	 * error_exit knows that we will be returning to user mode.
	 */
1197 1198 1199
	mov	%rsp, %rdi
	call	fixup_bad_iret
	mov	%rax, %rsp
1200 1201
	decl	%ebx
	jmp	error_entry_done
1202 1203 1204
END(error_entry)


1205 1206 1207 1208 1209
/*
 * On entry, EBS is a "return to kernel mode" flag:
 *   1: already in kernel mode, don't need SWAPGS
 *   0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
 */
1210
ENTRY(error_exit)
1211
	movl	%ebx, %eax
1212
	RESTORE_EXTRA_REGS
1213 1214
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
1215 1216 1217
	testl	%eax, %eax
	jnz	retint_kernel
	jmp	retint_user
1218 1219
END(error_exit)

1220
/* Runs on exception stack */
1221 1222
ENTRY(nmi)
	PARAVIRT_ADJUST_EXCEPTION_FRAME
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
	/*
	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
	 * the iretq it performs will take us out of NMI context.
	 * This means that we can have nested NMIs where the next
	 * NMI is using the top of the stack of the previous NMI. We
	 * can't let it execute because the nested NMI will corrupt the
	 * stack of the previous NMI. NMI handlers are not re-entrant
	 * anyway.
	 *
	 * To handle this case we do the following:
	 *  Check the a special location on the stack that contains
	 *  a variable that is set when NMIs are executing.
	 *  The interrupted task's stack is also checked to see if it
	 *  is an NMI stack.
	 *  If the variable is not set and the stack is not the NMI
	 *  stack then:
	 *    o Set the special variable on the stack
1240 1241 1242
	 *    o Copy the interrupt frame into an "outermost" location on the
	 *      stack
	 *    o Copy the interrupt frame into an "iret" location on the stack
1243 1244
	 *    o Continue processing the NMI
	 *  If the variable is set or the previous stack is the NMI stack:
1245
	 *    o Modify the "iret" location to jump to the repeat_nmi
1246 1247 1248 1249 1250 1251 1252 1253
	 *    o return back to the first NMI
	 *
	 * Now on exit of the first NMI, we first clear the stack variable
	 * The NMI stack will tell any nested NMIs at that point that it is
	 * nested. Then we pop the stack normally with iret, and if there was
	 * a nested NMI that updated the copy interrupt stack frame, a
	 * jump will be made to the repeat_nmi code that will handle the second
	 * NMI.
1254 1255 1256 1257 1258
	 *
	 * However, espfix prevents us from directly returning to userspace
	 * with a single IRET instruction.  Similarly, IRET to user mode
	 * can fault.  We therefore handle NMIs from user space like
	 * other IST entries.
1259 1260
	 */

1261
	/* Use %rdx as our temp variable throughout */
1262
	pushq	%rdx
1263

1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
	testb	$3, CS-RIP+8(%rsp)
	jz	.Lnmi_from_kernel

	/*
	 * NMI from user mode.  We need to run on the thread stack, but we
	 * can't go through the normal entry paths: NMIs are masked, and
	 * we don't want to enable interrupts, because then we'll end
	 * up in an awkward situation in which IRQs are on but NMIs
	 * are off.
	 */

	SWAPGS
	cld
	movq	%rsp, %rdx
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	pushq	5*8(%rdx)	/* pt_regs->ss */
	pushq	4*8(%rdx)	/* pt_regs->rsp */
	pushq	3*8(%rdx)	/* pt_regs->flags */
	pushq	2*8(%rdx)	/* pt_regs->cs */
	pushq	1*8(%rdx)	/* pt_regs->rip */
	pushq   $-1		/* pt_regs->orig_ax */
	pushq   %rdi		/* pt_regs->di */
	pushq   %rsi		/* pt_regs->si */
	pushq   (%rdx)		/* pt_regs->dx */
	pushq   %rcx		/* pt_regs->cx */
	pushq   %rax		/* pt_regs->ax */
	pushq   %r8		/* pt_regs->r8 */
	pushq   %r9		/* pt_regs->r9 */
	pushq   %r10		/* pt_regs->r10 */
	pushq   %r11		/* pt_regs->r11 */
	pushq	%rbx		/* pt_regs->rbx */
	pushq	%rbp		/* pt_regs->rbp */
	pushq	%r12		/* pt_regs->r12 */
	pushq	%r13		/* pt_regs->r13 */
	pushq	%r14		/* pt_regs->r14 */
	pushq	%r15		/* pt_regs->r15 */

	/*
	 * At this point we no longer need to worry about stack damage
	 * due to nesting -- we're on the normal thread stack and we're
	 * done with the NMI stack.
	 */

	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi

1311
	/*
1312 1313 1314
	 * Return back to user mode.  We must *not* do the normal exit
	 * work, because we don't want to enable interrupts.  Fortunately,
	 * do_nmi doesn't modify pt_regs.
1315
	 */
1316 1317
	SWAPGS
	jmp	restore_c_regs_and_iret
1318

1319
.Lnmi_from_kernel:
1320
	/*
1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
	 * Here's what our stack frame will look like:
	 * +---------------------------------------------------------+
	 * | original SS                                             |
	 * | original Return RSP                                     |
	 * | original RFLAGS                                         |
	 * | original CS                                             |
	 * | original RIP                                            |
	 * +---------------------------------------------------------+
	 * | temp storage for rdx                                    |
	 * +---------------------------------------------------------+
	 * | "NMI executing" variable                                |
	 * +---------------------------------------------------------+
	 * | iret SS          } Copied from "outermost" frame        |
	 * | iret Return RSP  } on each loop iteration; overwritten  |
	 * | iret RFLAGS      } by a nested NMI to force another     |
	 * | iret CS          } iteration if needed.                 |
	 * | iret RIP         }                                      |
	 * +---------------------------------------------------------+
	 * | outermost SS          } initialized in first_nmi;       |
	 * | outermost Return RSP  } will not be changed before      |
	 * | outermost RFLAGS      } NMI processing is done.         |
	 * | outermost CS          } Copied to "iret" frame on each  |
	 * | outermost RIP         } iteration.                      |
	 * +---------------------------------------------------------+
	 * | pt_regs                                                 |
	 * +---------------------------------------------------------+
	 *
	 * The "original" frame is used by hardware.  Before re-enabling
	 * NMIs, we need to be done with it, and we need to leave enough
	 * space for the asm code here.
	 *
	 * We return by executing IRET while RSP points to the "iret" frame.
	 * That will either return for real or it will loop back into NMI
	 * processing.
	 *
	 * The "outermost" frame is copied to the "iret" frame on each
	 * iteration of the loop, so each iteration starts with the "iret"
	 * frame pointing to the final return target.
	 */

	/*
	 * Determine whether we're a nested NMI.
	 *
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
	 * If we interrupted kernel code between repeat_nmi and
	 * end_repeat_nmi, then we are a nested NMI.  We must not
	 * modify the "iret" frame because it's being written by
	 * the outer NMI.  That's okay; the outer NMI handler is
	 * about to about to call do_nmi anyway, so we can just
	 * resume the outer NMI.
	 */

	movq	$repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	1f
	movq	$end_repeat_nmi, %rdx
	cmpq	8(%rsp), %rdx
	ja	nested_nmi_out
1:

	/*
	 * Now check "NMI executing".  If it's set, then we're nested.
1382 1383
	 * This will not detect if we interrupted an outer NMI just
	 * before IRET.
1384
	 */
1385 1386
	cmpl	$1, -8(%rsp)
	je	nested_nmi
1387 1388

	/*
1389 1390 1391
	 * Now test if the previous stack was an NMI stack.  This covers
	 * the case where we interrupt an outer NMI after it clears
	 * "NMI executing" but before IRET.
1392
	 */
1393 1394 1395 1396 1397
	lea	6*8(%rsp), %rdx
	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
	cmpq	%rdx, 4*8(%rsp)
	/* If the stack pointer is above the NMI stack, this is a normal NMI */
	ja	first_nmi
1398

1399 1400 1401 1402 1403 1404
	subq	$EXCEPTION_STKSZ, %rdx
	cmpq	%rdx, 4*8(%rsp)
	/* If it is below the NMI stack, it is a normal NMI */
	jb	first_nmi
	/* Ah, it is within the NMI stack, treat it as nested */

1405
nested_nmi:
1406 1407 1408 1409
	/*
	 * Modify the "iret" frame to point to repeat_nmi, forcing another
	 * iteration of NMI handling.
	 */
1410 1411 1412 1413 1414
	leaq	-1*8(%rsp), %rdx
	movq	%rdx, %rsp
	leaq	-10*8(%rsp), %rdx
	pushq	$__KERNEL_DS
	pushq	%rdx
1415
	pushfq
1416 1417
	pushq	$__KERNEL_CS
	pushq	$repeat_nmi
1418 1419

	/* Put stack back */
1420
	addq	$(6*8), %rsp
1421 1422

nested_nmi_out:
1423
	popq	%rdx
1424

1425
	/* We are returning to kernel mode, so this cannot result in a fault. */
1426 1427 1428
	INTERRUPT_RETURN

first_nmi:
1429
	/* Restore rdx. */
1430
	movq	(%rsp), %rdx
1431

1432
	/* Set "NMI executing" on the stack. */
1433
	pushq	$1
1434

1435
	/* Leave room for the "iret" frame */
1436
	subq	$(5*8), %rsp
1437

1438
	/* Copy the "original" frame to the "outermost" frame */
1439
	.rept 5
1440
	pushq	11*8(%rsp)
1441
	.endr
1442

1443 1444
	/* Everything up to here is safe from nested NMIs */

1445
repeat_nmi:
1446 1447 1448 1449 1450 1451 1452 1453
	/*
	 * If there was a nested NMI, the first NMI's iret will return
	 * here. But NMIs are still enabled and we can take another
	 * nested NMI. The nested NMI checks the interrupted RIP to see
	 * if it is between repeat_nmi and end_repeat_nmi, and if so
	 * it will just return, as we are about to repeat an NMI anyway.
	 * This makes it safe to copy to the stack frame that a nested
	 * NMI will update.
1454 1455 1456 1457 1458 1459 1460
	 *
	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
	 * we're repeating an NMI, gsbase has the same value that it had on
	 * the first iteration.  paranoid_entry will load the kernel
	 * gsbase if needed before we call do_nmi.
	 *
	 * Set "NMI executing" in case we came back here via IRET.
1461
	 */
1462
	movq	$1, 10*8(%rsp)
1463

1464 1465 1466 1467 1468
	/*
	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
	 * here must not modify the "iret" frame while we're writing to
	 * it or it will end up containing garbage.
	 */
1469
	addq	$(10*8), %rsp
1470
	.rept 5
1471
	pushq	-6*8(%rsp)
1472
	.endr
1473
	subq	$(5*8), %rsp
1474
end_repeat_nmi:
1475 1476

	/*
1477 1478 1479
	 * Everything below this point can be preempted by a nested NMI.
	 * If this happens, then the inner NMI will change the "iret"
	 * frame to point back to repeat_nmi.
1480
	 */
1481
	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1482 1483
	ALLOC_PT_GPREGS_ON_STACK

1484
	/*
1485
	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1486 1487 1488 1489 1490
	 * as we should not be calling schedule in NMI context.
	 * Even with normal interrupts enabled. An NMI should not be
	 * setting NEED_RESCHED or anything that normal interrupts and
	 * exceptions might do.
	 */
1491
	call	paranoid_entry
1492

1493
	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1494 1495 1496
	movq	%rsp, %rdi
	movq	$-1, %rsi
	call	do_nmi
1497

1498 1499
	testl	%ebx, %ebx			/* swapgs needed? */
	jnz	nmi_restore
1500 1501 1502
nmi_swapgs:
	SWAPGS_UNSAFE_STACK
nmi_restore:
1503 1504
	RESTORE_EXTRA_REGS
	RESTORE_C_REGS
1505 1506

	/* Point RSP at the "iret" frame. */
1507
	REMOVE_PT_GPREGS_FROM_STACK 6*8
1508

1509
	/* Clear "NMI executing". */
1510
	movq	$0, 5*8(%rsp)
1511 1512 1513 1514 1515 1516

	/*
	 * INTERRUPT_RETURN reads the "iret" frame and exits the NMI
	 * stack in a single instruction.  We are returning to kernel
	 * mode, so this cannot result in a fault.
	 */
1517
	INTERRUPT_RETURN
1518 1519 1520
END(nmi)

ENTRY(ignore_sysret)
1521
	mov	$-ENOSYS, %eax
1522 1523
	sysret
END(ignore_sysret)