entry_32.S 27.5 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  Copyright (C) 1991,1992  Linus Torvalds
L
Linus Torvalds 已提交
3
 *
4
 * entry_32.S contains the system-call and low-level fault and trap handling routines.
L
Linus Torvalds 已提交
5
 *
6
 * Stack layout while running C code:
7 8 9
 *	ptrace needs to have all registers on the stack.
 *	If the order here is changed, it needs to be
 *	updated in fork.c:copy_process(), signal.c:do_signal(),
L
Linus Torvalds 已提交
10 11 12 13 14
 *	ptrace.c and ptrace.h
 *
 *	 0(%esp) - %ebx
 *	 4(%esp) - %ecx
 *	 8(%esp) - %edx
15
 *	 C(%esp) - %esi
L
Linus Torvalds 已提交
16 17 18 19 20
 *	10(%esp) - %edi
 *	14(%esp) - %ebp
 *	18(%esp) - %eax
 *	1C(%esp) - %ds
 *	20(%esp) - %es
21
 *	24(%esp) - %fs
22 23 24 25 26 27 28
 *	28(%esp) - %gs		saved iff !CONFIG_X86_32_LAZY_GS
 *	2C(%esp) - orig_eax
 *	30(%esp) - %eip
 *	34(%esp) - %cs
 *	38(%esp) - %eflags
 *	3C(%esp) - %oldesp
 *	40(%esp) - %oldss
L
Linus Torvalds 已提交
29 30 31
 */

#include <linux/linkage.h>
32
#include <linux/err.h>
L
Linus Torvalds 已提交
33
#include <asm/thread_info.h>
34
#include <asm/irqflags.h>
L
Linus Torvalds 已提交
35 36 37
#include <asm/errno.h>
#include <asm/segment.h>
#include <asm/smp.h>
38
#include <asm/page_types.h>
S
Stas Sergeev 已提交
39
#include <asm/percpu.h>
40
#include <asm/processor-flags.h>
41
#include <asm/ftrace.h>
42
#include <asm/irq_vectors.h>
43
#include <asm/cpufeatures.h>
44
#include <asm/alternative-asm.h>
45
#include <asm/asm.h>
46
#include <asm/smap.h>
47
#include <asm/export.h>
L
Linus Torvalds 已提交
48

J
Jiri Olsa 已提交
49 50
	.section .entry.text, "ax"

51 52 53 54 55
/*
 * We use macros for low-level operations which need to be overridden
 * for paravirtualization.  The following will never clobber any registers:
 *   INTERRUPT_RETURN (aka. "iret")
 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
56
 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
57 58 59 60 61 62 63
 *
 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
 * Allowing a register to be clobbered can shrink the paravirt replacement
 * enough to patch inline, increasing performance.
 */

L
Linus Torvalds 已提交
64
#ifdef CONFIG_PREEMPT
65
# define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
L
Linus Torvalds 已提交
66
#else
67 68
# define preempt_stop(clobbers)
# define resume_kernel		restore_all
L
Linus Torvalds 已提交
69 70
#endif

71 72
.macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS
73 74
	testl	$X86_EFLAGS_IF, PT_EFLAGS(%esp)     # interrupts off?
	jz	1f
75 76 77 78 79
	TRACE_IRQS_ON
1:
#endif
.endm

80 81 82 83 84 85 86 87 88 89 90 91 92
/*
 * User gs save/restore
 *
 * %gs is used for userland TLS and kernel only uses it for stack
 * canary which is required to be at %gs:20 by gcc.  Read the comment
 * at the top of stackprotector.h for more info.
 *
 * Local labels 98 and 99 are used.
 */
#ifdef CONFIG_X86_32_LAZY_GS

 /* unfortunately push/pop can't be no-op */
.macro PUSH_GS
93
	pushl	$0
94 95
.endm
.macro POP_GS pop=0
96
	addl	$(4 + \pop), %esp
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
.endm
.macro POP_GS_EX
.endm

 /* all the rest are no-op */
.macro PTGS_TO_GS
.endm
.macro PTGS_TO_GS_EX
.endm
.macro GS_TO_REG reg
.endm
.macro REG_TO_PTGS reg
.endm
.macro SET_KERNEL_GS reg
.endm

#else	/* CONFIG_X86_32_LAZY_GS */

.macro PUSH_GS
116
	pushl	%gs
117 118 119
.endm

.macro POP_GS pop=0
120
98:	popl	%gs
121
  .if \pop <> 0
122
	add	$\pop, %esp
123 124 125 126
  .endif
.endm
.macro POP_GS_EX
.pushsection .fixup, "ax"
127 128
99:	movl	$0, (%esp)
	jmp	98b
129
.popsection
130
	_ASM_EXTABLE(98b, 99b)
131 132 133
.endm

.macro PTGS_TO_GS
134
98:	mov	PT_GS(%esp), %gs
135 136 137
.endm
.macro PTGS_TO_GS_EX
.pushsection .fixup, "ax"
138 139
99:	movl	$0, PT_GS(%esp)
	jmp	98b
140
.popsection
141
	_ASM_EXTABLE(98b, 99b)
142 143 144
.endm

.macro GS_TO_REG reg
145
	movl	%gs, \reg
146 147
.endm
.macro REG_TO_PTGS reg
148
	movl	\reg, PT_GS(%esp)
149 150
.endm
.macro SET_KERNEL_GS reg
151 152
	movl	$(__KERNEL_STACK_CANARY), \reg
	movl	\reg, %gs
153 154
.endm

155
#endif /* CONFIG_X86_32_LAZY_GS */
156

157
.macro SAVE_ALL pt_regs_ax=%eax
158
	cld
159
	PUSH_GS
160 161 162
	pushl	%fs
	pushl	%es
	pushl	%ds
163
	pushl	\pt_regs_ax
164 165 166 167 168 169 170 171 172 173 174
	pushl	%ebp
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx
	movl	$(__USER_DS), %edx
	movl	%edx, %ds
	movl	%edx, %es
	movl	$(__KERNEL_PERCPU), %edx
	movl	%edx, %fs
175
	SET_KERNEL_GS %edx
176
.endm
L
Linus Torvalds 已提交
177

178
.macro RESTORE_INT_REGS
179 180 181 182 183 184 185
	popl	%ebx
	popl	%ecx
	popl	%edx
	popl	%esi
	popl	%edi
	popl	%ebp
	popl	%eax
186
.endm
L
Linus Torvalds 已提交
187

188
.macro RESTORE_REGS pop=0
189
	RESTORE_INT_REGS
190 191 192
1:	popl	%ds
2:	popl	%es
3:	popl	%fs
193
	POP_GS \pop
194
.pushsection .fixup, "ax"
195 196 197 198 199 200
4:	movl	$0, (%esp)
	jmp	1b
5:	movl	$0, (%esp)
	jmp	2b
6:	movl	$0, (%esp)
	jmp	3b
201
.popsection
202 203 204
	_ASM_EXTABLE(1b, 4b)
	_ASM_EXTABLE(2b, 5b)
	_ASM_EXTABLE(3b, 6b)
205
	POP_GS_EX
206
.endm
L
Linus Torvalds 已提交
207 208

ENTRY(ret_from_fork)
209 210 211
	pushl	%eax
	call	schedule_tail
	popl	%eax
212 213 214 215 216

	/* When we fork, we trace the syscall return in the child, too. */
	movl    %esp, %eax
	call    syscall_return_slowpath
	jmp     restore_all
217
END(ret_from_fork)
L
Linus Torvalds 已提交
218

219
ENTRY(ret_from_kernel_thread)
220 221 222 223 224 225
	pushl	%eax
	call	schedule_tail
	popl	%eax
	movl	PT_EBP(%esp), %eax
	call	*PT_EBX(%esp)
	movl	$0, PT_EAX(%esp)
226 227 228 229 230 231 232 233 234

	/*
	 * Kernel threads return to userspace as if returning from a syscall.
	 * We should check whether anything actually uses this path and, if so,
	 * consider switching it over to ret_from_fork.
	 */
	movl    %esp, %eax
	call    syscall_return_slowpath
	jmp     restore_all
235
ENDPROC(ret_from_kernel_thread)
236

L
Linus Torvalds 已提交
237 238 239 240 241 242 243 244 245 246
/*
 * Return to user mode is not as complex as all this looks,
 * but we want the default path for a system call return to
 * go as quickly as possible which is why some of this is
 * less clear than it otherwise should be.
 */

	# userspace resumption stub bypassing syscall exit tracing
	ALIGN
ret_from_exception:
247
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
248
ret_from_intr:
249
#ifdef CONFIG_VM86
250 251 252
	movl	PT_EFLAGS(%esp), %eax		# mix EFLAGS and CS
	movb	PT_CS(%esp), %al
	andl	$(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
253 254
#else
	/*
255
	 * We can be coming here from child spawned by kernel_thread().
256
	 */
257 258
	movl	PT_CS(%esp), %eax
	andl	$SEGMENT_RPL_MASK, %eax
259
#endif
260 261
	cmpl	$USER_RPL, %eax
	jb	resume_kernel			# not returning to v8086 or userspace
262

L
Linus Torvalds 已提交
263
ENTRY(resume_userspace)
264
	DISABLE_INTERRUPTS(CLBR_ANY)
265
	TRACE_IRQS_OFF
266 267
	movl	%esp, %eax
	call	prepare_exit_to_usermode
268
	jmp	restore_all
269
END(ret_from_exception)
L
Linus Torvalds 已提交
270 271 272

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
273
	DISABLE_INTERRUPTS(CLBR_ANY)
L
Linus Torvalds 已提交
274
need_resched:
275 276 277 278 279 280
	cmpl	$0, PER_CPU_VAR(__preempt_count)
	jnz	restore_all
	testl	$X86_EFLAGS_IF, PT_EFLAGS(%esp)	# interrupts off (exception path) ?
	jz	restore_all
	call	preempt_schedule_irq
	jmp	need_resched
281
END(resume_kernel)
L
Linus Torvalds 已提交
282 283
#endif

284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
GLOBAL(__begin_SYSENTER_singlestep_region)
/*
 * All code from here through __end_SYSENTER_singlestep_region is subject
 * to being single-stepped if a user program sets TF and executes SYSENTER.
 * There is absolutely nothing that we can do to prevent this from happening
 * (thanks Intel!).  To keep our handling of this situation as simple as
 * possible, we handle TF just like AC and NT, except that our #DB handler
 * will ignore all of the single-step traps generated in this range.
 */

#ifdef CONFIG_XEN
/*
 * Xen doesn't set %esp to be precisely what the normal SYSENTER
 * entry point expects, so fix it up before using the normal path.
 */
ENTRY(xen_sysenter_target)
	addl	$5*4, %esp			/* remove xen-provided frame */
	jmp	sysenter_past_esp
#endif

304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
/*
 * 32-bit SYSENTER entry.
 *
 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
 * if X86_FEATURE_SEP is available.  This is the preferred system call
 * entry on 32-bit systems.
 *
 * The SYSENTER instruction, in principle, should *only* occur in the
 * vDSO.  In practice, a small number of Android devices were shipped
 * with a copy of Bionic that inlined a SYSENTER instruction.  This
 * never happened in any of Google's Bionic versions -- it only happened
 * in a narrow range of Intel-provided versions.
 *
 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
 * SYSENTER does not save anything on the stack,
 * and does not save old EIP (!!!), ESP, or EFLAGS.
 *
 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
 * user and/or vm86 state), we explicitly disable the SYSENTER
 * instruction in vm86 mode by reprogramming the MSRs.
 *
 * Arguments:
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * ebp  user stack
 * 0(%ebp) arg6
 */
336
ENTRY(entry_SYSENTER_32)
337
	movl	TSS_sysenter_sp0(%esp), %esp
L
Linus Torvalds 已提交
338
sysenter_past_esp:
339
	pushl	$__USER_DS		/* pt_regs->ss */
340
	pushl	%ebp			/* pt_regs->sp (stashed in bp) */
341 342 343 344 345 346 347
	pushfl				/* pt_regs->flags (except IF = 0) */
	orl	$X86_EFLAGS_IF, (%esp)	/* Fix IF */
	pushl	$__USER_CS		/* pt_regs->cs */
	pushl	$0			/* pt_regs->ip = 0 (placeholder) */
	pushl	%eax			/* pt_regs->orig_ax */
	SAVE_ALL pt_regs_ax=$-ENOSYS	/* save rest */

348
	/*
349 350
	 * SYSENTER doesn't filter flags, so we need to clear NT, AC
	 * and TF ourselves.  To save a few cycles, we can check whether
351 352 353 354
	 * either was set instead of doing an unconditional popfq.
	 * This needs to happen before enabling interrupts so that
	 * we don't get preempted with NT set.
	 *
355 356 357 358 359 360
	 * If TF is set, we will single-step all the way to here -- do_debug
	 * will ignore all the traps.  (Yes, this is slow, but so is
	 * single-stepping in general.  This allows us to avoid having
	 * a more complicated code to handle the case where a user program
	 * forces us to single-step through the SYSENTER entry code.)
	 *
361 362 363 364 365 366
	 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
	 * out-of-line as an optimization: NT is unlikely to be set in the
	 * majority of the cases and instead of polluting the I$ unnecessarily,
	 * we're keeping that code behind a branch which will predict as
	 * not-taken and therefore its instructions won't be fetched.
	 */
367
	testl	$X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
368 369 370
	jnz	.Lsysenter_fix_flags
.Lsysenter_flags_fixed:

371
	/*
372 373
	 * User mode is traced as though IRQs are on, and SYSENTER
	 * turned them off.
374
	 */
375
	TRACE_IRQS_OFF
376 377 378

	movl	%esp, %eax
	call	do_fast_syscall_32
379 380 381
	/* XEN PV guests always use IRET path */
	ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
		    "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
382 383 384 385 386

/* Opportunistic SYSEXIT */
	TRACE_IRQS_ON			/* User mode traces as IRQs on. */
	movl	PT_EIP(%esp), %edx	/* pt_regs->ip */
	movl	PT_OLDESP(%esp), %ecx	/* pt_regs->sp */
387 388
1:	mov	PT_FS(%esp), %fs
	PTGS_TO_GS
389 390 391 392 393 394 395
	popl	%ebx			/* pt_regs->bx */
	addl	$2*4, %esp		/* skip pt_regs->cx and pt_regs->dx */
	popl	%esi			/* pt_regs->si */
	popl	%edi			/* pt_regs->di */
	popl	%ebp			/* pt_regs->bp */
	popl	%eax			/* pt_regs->ax */

396 397 398 399 400 401 402 403 404
	/*
	 * Restore all flags except IF. (We restore IF separately because
	 * STI gives a one-instruction window in which we won't be interrupted,
	 * whereas POPF does not.)
	 */
	addl	$PT_EFLAGS-PT_DS, %esp	/* point esp at pt_regs->flags */
	btr	$X86_EFLAGS_IF_BIT, (%esp)
	popfl

405 406 407 408
	/*
	 * Return back to the vDSO, which will pop ecx and edx.
	 * Don't bother with DS and ES (they already contain __USER_DS).
	 */
409 410
	sti
	sysexit
R
Roland McGrath 已提交
411

412 413 414
.pushsection .fixup, "ax"
2:	movl	$0, PT_FS(%esp)
	jmp	1b
415
.popsection
416
	_ASM_EXTABLE(1b, 2b)
417
	PTGS_TO_GS_EX
418 419 420 421 422

.Lsysenter_fix_flags:
	pushl	$X86_EFLAGS_FIXED
	popfl
	jmp	.Lsysenter_flags_fixed
423
GLOBAL(__end_SYSENTER_singlestep_region)
424
ENDPROC(entry_SYSENTER_32)
L
Linus Torvalds 已提交
425

426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
/*
 * 32-bit legacy system call entry.
 *
 * 32-bit x86 Linux system calls traditionally used the INT $0x80
 * instruction.  INT $0x80 lands here.
 *
 * This entry point can be used by any 32-bit perform system calls.
 * Instances of INT $0x80 can be found inline in various programs and
 * libraries.  It is also used by the vDSO's __kernel_vsyscall
 * fallback for hardware that doesn't support a faster entry method.
 * Restarted 32-bit system calls also fall back to INT $0x80
 * regardless of what instruction was originally used to do the system
 * call.  (64-bit programs can use INT $0x80 as well, but they can
 * only run on 64-bit kernels and therefore land in
 * entry_INT80_compat.)
 *
 * This is considered a slow path.  It is not used by most libc
 * implementations on modern hardware except during process startup.
 *
 * Arguments:
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * ebp  arg6
 */
454
ENTRY(entry_INT80_32)
455
	ASM_CLAC
456
	pushl	%eax			/* pt_regs->orig_ax */
457
	SAVE_ALL pt_regs_ax=$-ENOSYS	/* save rest */
458 459

	/*
460 461
	 * User mode is traced as though IRQs are on, and the interrupt gate
	 * turned them off.
462
	 */
463
	TRACE_IRQS_OFF
464 465

	movl	%esp, %eax
466
	call	do_int80_syscall_32
467
.Lsyscall_32_done:
L
Linus Torvalds 已提交
468 469

restore_all:
470 471
	TRACE_IRQS_IRET
restore_all_notrace:
472
#ifdef CONFIG_X86_ESPFIX32
473 474
	ALTERNATIVE	"jmp restore_nocheck", "", X86_BUG_ESPFIX

475 476 477 478 479 480 481 482 483 484 485
	movl	PT_EFLAGS(%esp), %eax		# mix EFLAGS, SS and CS
	/*
	 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
	 * are returning to the kernel.
	 * See comments in process.c:copy_thread() for details.
	 */
	movb	PT_OLDSS(%esp), %ah
	movb	PT_CS(%esp), %al
	andl	$(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
	cmpl	$((SEGMENT_LDT << 8) | USER_RPL), %eax
	je ldt_ss				# returning to user-space with LDT SS
486
#endif
L
Linus Torvalds 已提交
487
restore_nocheck:
488
	RESTORE_REGS 4				# skip orig_eax/error_code
A
Adrian Bunk 已提交
489
irq_return:
I
Ingo Molnar 已提交
490
	INTERRUPT_RETURN
491 492 493 494 495
.section .fixup, "ax"
ENTRY(iret_exc	)
	pushl	$0				# no error code
	pushl	$do_iret_error
	jmp	error_code
L
Linus Torvalds 已提交
496
.previous
497
	_ASM_EXTABLE(irq_return, iret_exc)
L
Linus Torvalds 已提交
498

499
#ifdef CONFIG_X86_ESPFIX32
L
Linus Torvalds 已提交
500
ldt_ss:
501 502 503 504 505 506 507 508 509 510 511
/*
 * Setup and switch to ESPFIX stack
 *
 * We're returning to userspace with a 16 bit stack. The CPU will not
 * restore the high word of ESP for us on executing iret... This is an
 * "official" bug of all the x86-compatible CPUs, which we can work
 * around to make dosemu and wine happy. We do this by preloading the
 * high word of ESP with the high word of the userspace ESP while
 * compensating for the offset by changing to the ESPFIX segment with
 * a base address that matches for the difference.
 */
512
#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
513 514 515
	mov	%esp, %edx			/* load kernel esp */
	mov	PT_OLDESP(%esp), %eax		/* load userspace esp */
	mov	%dx, %ax			/* eax: new kernel esp */
516 517
	sub	%eax, %edx			/* offset (low word is 0) */
	shr	$16, %edx
518 519 520 521 522 523
	mov	%dl, GDT_ESPFIX_SS + 4		/* bits 16..23 */
	mov	%dh, GDT_ESPFIX_SS + 7		/* bits 24..31 */
	pushl	$__ESPFIX_SS
	pushl	%eax				/* new kernel esp */
	/*
	 * Disable interrupts, but do not irqtrace this section: we
524
	 * will soon execute iret and the tracer was already set to
525 526
	 * the irqstate after the IRET:
	 */
527
	DISABLE_INTERRUPTS(CLBR_EAX)
528 529
	lss	(%esp), %esp			/* switch to espfix segment */
	jmp	restore_nocheck
530
#endif
531
ENDPROC(entry_INT80_32)
L
Linus Torvalds 已提交
532

533
.macro FIXUP_ESPFIX_STACK
534 535 536 537 538 539 540
/*
 * Switch back for ESPFIX stack to the normal zerobased stack
 *
 * We can't call C functions using the ESPFIX stack. This code reads
 * the high word of the segment base from the GDT and swiches to the
 * normal stack and adjusts ESP with the matching offset.
 */
541
#ifdef CONFIG_X86_ESPFIX32
542
	/* fixup the stack */
543 544
	mov	GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
	mov	GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
545
	shl	$16, %eax
546 547 548 549
	addl	%esp, %eax			/* the adjusted stack pointer */
	pushl	$__KERNEL_DS
	pushl	%eax
	lss	(%esp), %esp			/* switch to the normal stack segment */
550
#endif
551 552
.endm
.macro UNWIND_ESPFIX_STACK
553
#ifdef CONFIG_X86_ESPFIX32
554
	movl	%ss, %eax
555
	/* see if on espfix stack */
556 557 558 559 560
	cmpw	$__ESPFIX_SS, %ax
	jne	27f
	movl	$__KERNEL_DS, %eax
	movl	%eax, %ds
	movl	%eax, %es
561 562 563
	/* switch to normal stack */
	FIXUP_ESPFIX_STACK
27:
564
#endif
565
.endm
L
Linus Torvalds 已提交
566 567

/*
568 569
 * Build the entry stubs with some assembler magic.
 * We pack 1 stub into every 8-byte block.
L
Linus Torvalds 已提交
570
 */
571
	.align 8
L
Linus Torvalds 已提交
572
ENTRY(irq_entries_start)
573 574
    vector=FIRST_EXTERNAL_VECTOR
    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
575
	pushl	$(~vector+0x80)			/* Note: always in signed byte range */
576 577 578 579
    vector=vector+1
	jmp	common_interrupt
	.align	8
    .endr
580 581
END(irq_entries_start)

582 583 584 585
/*
 * the CPU automatically disables interrupts when executing an IRQ vector,
 * so IRQ-flags tracing has to follow that:
 */
586
	.p2align CONFIG_X86_L1_CACHE_SHIFT
L
Linus Torvalds 已提交
587
common_interrupt:
588
	ASM_CLAC
589
	addl	$-0x80, (%esp)			/* Adjust vector into the [-256, -1] range */
L
Linus Torvalds 已提交
590
	SAVE_ALL
591
	TRACE_IRQS_OFF
592 593 594
	movl	%esp, %eax
	call	do_IRQ
	jmp	ret_from_intr
595
ENDPROC(common_interrupt)
L
Linus Torvalds 已提交
596

T
Tejun Heo 已提交
597
#define BUILD_INTERRUPT3(name, nr, fn)	\
L
Linus Torvalds 已提交
598
ENTRY(name)				\
599
	ASM_CLAC;			\
600
	pushl	$~(nr);			\
601
	SAVE_ALL;			\
602
	TRACE_IRQS_OFF			\
603 604 605
	movl	%esp, %eax;		\
	call	fn;			\
	jmp	ret_from_intr;		\
606
ENDPROC(name)
L
Linus Torvalds 已提交
607

608 609

#ifdef CONFIG_TRACING
610
# define TRACE_BUILD_INTERRUPT(name, nr)	BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name)
611
#else
612
# define TRACE_BUILD_INTERRUPT(name, nr)
613 614
#endif

615 616
#define BUILD_INTERRUPT(name, nr)		\
	BUILD_INTERRUPT3(name, nr, smp_##name);	\
617
	TRACE_BUILD_INTERRUPT(name, nr)
T
Tejun Heo 已提交
618

L
Linus Torvalds 已提交
619
/* The include is where all of the SMP etc. interrupts come from */
620
#include <asm/entry_arch.h>
L
Linus Torvalds 已提交
621 622

ENTRY(coprocessor_error)
623
	ASM_CLAC
624 625 626
	pushl	$0
	pushl	$do_coprocessor_error
	jmp	error_code
627
END(coprocessor_error)
L
Linus Torvalds 已提交
628 629

ENTRY(simd_coprocessor_error)
630
	ASM_CLAC
631
	pushl	$0
632 633
#ifdef CONFIG_X86_INVD_BUG
	/* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
634 635
	ALTERNATIVE "pushl	$do_general_protection",	\
		    "pushl	$do_simd_coprocessor_error",	\
636
		    X86_FEATURE_XMM
637
#else
638
	pushl	$do_simd_coprocessor_error
639
#endif
640
	jmp	error_code
641
END(simd_coprocessor_error)
L
Linus Torvalds 已提交
642 643

ENTRY(device_not_available)
644
	ASM_CLAC
645 646 647
	pushl	$-1				# mark this as an int
	pushl	$do_device_not_available
	jmp	error_code
648
END(device_not_available)
L
Linus Torvalds 已提交
649

650 651
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
I
Ingo Molnar 已提交
652
	iret
653
	_ASM_EXTABLE(native_iret, iret_exc)
654
END(native_iret)
655 656
#endif

L
Linus Torvalds 已提交
657
ENTRY(overflow)
658
	ASM_CLAC
659 660 661
	pushl	$0
	pushl	$do_overflow
	jmp	error_code
662
END(overflow)
L
Linus Torvalds 已提交
663 664

ENTRY(bounds)
665
	ASM_CLAC
666 667 668
	pushl	$0
	pushl	$do_bounds
	jmp	error_code
669
END(bounds)
L
Linus Torvalds 已提交
670 671

ENTRY(invalid_op)
672
	ASM_CLAC
673 674 675
	pushl	$0
	pushl	$do_invalid_op
	jmp	error_code
676
END(invalid_op)
L
Linus Torvalds 已提交
677 678

ENTRY(coprocessor_segment_overrun)
679
	ASM_CLAC
680 681 682
	pushl	$0
	pushl	$do_coprocessor_segment_overrun
	jmp	error_code
683
END(coprocessor_segment_overrun)
L
Linus Torvalds 已提交
684 685

ENTRY(invalid_TSS)
686
	ASM_CLAC
687 688
	pushl	$do_invalid_TSS
	jmp	error_code
689
END(invalid_TSS)
L
Linus Torvalds 已提交
690 691

ENTRY(segment_not_present)
692
	ASM_CLAC
693 694
	pushl	$do_segment_not_present
	jmp	error_code
695
END(segment_not_present)
L
Linus Torvalds 已提交
696 697

ENTRY(stack_segment)
698
	ASM_CLAC
699 700
	pushl	$do_stack_segment
	jmp	error_code
701
END(stack_segment)
L
Linus Torvalds 已提交
702 703

ENTRY(alignment_check)
704
	ASM_CLAC
705 706
	pushl	$do_alignment_check
	jmp	error_code
707
END(alignment_check)
L
Linus Torvalds 已提交
708

709
ENTRY(divide_error)
710
	ASM_CLAC
711 712 713
	pushl	$0				# no error code
	pushl	$do_divide_error
	jmp	error_code
714
END(divide_error)
L
Linus Torvalds 已提交
715 716 717

#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
718
	ASM_CLAC
719 720 721
	pushl	$0
	pushl	machine_check_vector
	jmp	error_code
722
END(machine_check)
L
Linus Torvalds 已提交
723 724 725
#endif

ENTRY(spurious_interrupt_bug)
726
	ASM_CLAC
727 728 729
	pushl	$0
	pushl	$do_spurious_interrupt_bug
	jmp	error_code
730
END(spurious_interrupt_bug)
L
Linus Torvalds 已提交
731

732 733
#ifdef CONFIG_XEN
ENTRY(xen_hypervisor_callback)
734
	pushl	$-1				/* orig_ax = -1 => not a system call */
735 736
	SAVE_ALL
	TRACE_IRQS_OFF
737

738 739 740 741 742 743 744 745 746 747 748 749
	/*
	 * Check to see if we got the event in the critical
	 * region in xen_iret_direct, after we've reenabled
	 * events and checked for pending events.  This simulates
	 * iret instruction's behaviour where it delivers a
	 * pending interrupt when enabling interrupts:
	 */
	movl	PT_EIP(%esp), %eax
	cmpl	$xen_iret_start_crit, %eax
	jb	1f
	cmpl	$xen_iret_end_crit, %eax
	jae	1f
750

751
	jmp	xen_iret_crit_fixup
752 753

ENTRY(xen_do_upcall)
754 755
1:	mov	%esp, %eax
	call	xen_evtchn_do_upcall
756
#ifndef CONFIG_PREEMPT
757
	call	xen_maybe_preempt_hcall
758
#endif
759
	jmp	ret_from_intr
760 761
ENDPROC(xen_hypervisor_callback)

762 763 764 765 766 767 768 769 770 771 772 773
/*
 * Hypervisor uses this for application faults while it executes.
 * We get here for two reasons:
 *  1. Fault while reloading DS, ES, FS or GS
 *  2. Fault while executing IRET
 * Category 1 we fix up by reattempting the load, and zeroing the segment
 * register if the load fails.
 * Category 2 we fix up by jumping to do_iret_error. We cannot use the
 * normal Linux return path in this case because if we use the IRET hypercall
 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 * We distinguish between categories by maintaining a status value in EAX.
 */
774
ENTRY(xen_failsafe_callback)
775 776 777 778 779 780
	pushl	%eax
	movl	$1, %eax
1:	mov	4(%esp), %ds
2:	mov	8(%esp), %es
3:	mov	12(%esp), %fs
4:	mov	16(%esp), %gs
781 782
	/* EAX == 0 => Category 1 (Bad segment)
	   EAX != 0 => Category 2 (Bad IRET) */
783 784 785 786 787 788
	testl	%eax, %eax
	popl	%eax
	lea	16(%esp), %esp
	jz	5f
	jmp	iret_exc
5:	pushl	$-1				/* orig_ax = -1 => not a system call */
789
	SAVE_ALL
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
	jmp	ret_from_exception

.section .fixup, "ax"
6:	xorl	%eax, %eax
	movl	%eax, 4(%esp)
	jmp	1b
7:	xorl	%eax, %eax
	movl	%eax, 8(%esp)
	jmp	2b
8:	xorl	%eax, %eax
	movl	%eax, 12(%esp)
	jmp	3b
9:	xorl	%eax, %eax
	movl	%eax, 16(%esp)
	jmp	4b
805
.previous
806 807 808 809
	_ASM_EXTABLE(1b, 6b)
	_ASM_EXTABLE(2b, 7b)
	_ASM_EXTABLE(3b, 8b)
	_ASM_EXTABLE(4b, 9b)
810 811
ENDPROC(xen_failsafe_callback)

812
BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
813 814
		xen_evtchn_do_upcall)

815
#endif /* CONFIG_XEN */
816 817 818 819 820 821 822

#if IS_ENABLED(CONFIG_HYPERV)

BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
	hyperv_vector_handler)

#endif /* CONFIG_HYPERV */
823

824
#ifdef CONFIG_FUNCTION_TRACER
825 826 827 828 829 830 831
#ifdef CONFIG_DYNAMIC_FTRACE

ENTRY(mcount)
	ret
END(mcount)

ENTRY(ftrace_caller)
832 833 834 835 836 837 838 839
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	pushl	$0				/* Pass NULL as regs pointer */
	movl	4*4(%esp), %eax
	movl	0x4(%ebp), %edx
	movl	function_trace_op, %ecx
	subl	$MCOUNT_INSN_SIZE, %eax
840 841 842

.globl ftrace_call
ftrace_call:
843
	call	ftrace_stub
844

845 846 847 848
	addl	$4, %esp			/* skip NULL pointer */
	popl	%edx
	popl	%ecx
	popl	%eax
849
ftrace_ret:
850 851 852
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call
ftrace_graph_call:
853
	jmp	ftrace_stub
854
#endif
855 856 857 858 859 860

.globl ftrace_stub
ftrace_stub:
	ret
END(ftrace_caller)

861 862 863 864 865 866 867 868 869 870
ENTRY(ftrace_regs_caller)
	pushf	/* push flags before compare (in cs location) */

	/*
	 * i386 does not save SS and ESP when coming from kernel.
	 * Instead, to get sp, &regs->sp is used (see ptrace.h).
	 * Unfortunately, that means eflags must be at the same location
	 * as the current return ip is. We move the return ip into the
	 * ip location, and move flags into the return ip location.
	 */
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
	pushl	4(%esp)				/* save return ip into ip slot */

	pushl	$0				/* Load 0 into orig_ax */
	pushl	%gs
	pushl	%fs
	pushl	%es
	pushl	%ds
	pushl	%eax
	pushl	%ebp
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx

	movl	13*4(%esp), %eax		/* Get the saved flags */
	movl	%eax, 14*4(%esp)		/* Move saved flags into regs->flags location */
						/* clobbering return ip */
	movl	$__KERNEL_CS, 13*4(%esp)

	movl	12*4(%esp), %eax		/* Load ip (1st parameter) */
	subl	$MCOUNT_INSN_SIZE, %eax		/* Adjust ip */
	movl	0x4(%ebp), %edx			/* Load parent ip (2nd parameter) */
	movl	function_trace_op, %ecx		/* Save ftrace_pos in 3rd parameter */
	pushl	%esp				/* Save pt_regs as 4th parameter */
896 897

GLOBAL(ftrace_regs_call)
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
	call	ftrace_stub

	addl	$4, %esp			/* Skip pt_regs */
	movl	14*4(%esp), %eax		/* Move flags back into cs */
	movl	%eax, 13*4(%esp)		/* Needed to keep addl	from modifying flags */
	movl	12*4(%esp), %eax		/* Get return ip from regs->ip */
	movl	%eax, 14*4(%esp)		/* Put return ip back for ret */

	popl	%ebx
	popl	%ecx
	popl	%edx
	popl	%esi
	popl	%edi
	popl	%ebp
	popl	%eax
	popl	%ds
	popl	%es
	popl	%fs
	popl	%gs
	addl	$8, %esp			/* Skip orig_ax and ip */
	popf					/* Pop flags at end (no addl to corrupt flags) */
	jmp	ftrace_ret
920 921

	popf
922
	jmp	ftrace_stub
923 924
#else /* ! CONFIG_DYNAMIC_FTRACE */

925
ENTRY(mcount)
926 927
	cmpl	$__PAGE_OFFSET, %esp
	jb	ftrace_stub			/* Paging not enabled yet? */
928

929 930
	cmpl	$ftrace_stub, ftrace_trace_function
	jnz	trace
931
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
932 933
	cmpl	$ftrace_stub, ftrace_graph_return
	jnz	ftrace_graph_caller
934

935 936
	cmpl	$ftrace_graph_entry_stub, ftrace_graph_entry
	jnz	ftrace_graph_caller
937
#endif
938 939 940 941 942 943
.globl ftrace_stub
ftrace_stub:
	ret

	/* taken from glibc */
trace:
944 945 946 947 948 949 950 951 952 953 954 955 956
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	movl	0xc(%esp), %eax
	movl	0x4(%ebp), %edx
	subl	$MCOUNT_INSN_SIZE, %eax

	call	*ftrace_trace_function

	popl	%edx
	popl	%ecx
	popl	%eax
	jmp	ftrace_stub
957
END(mcount)
958
#endif /* CONFIG_DYNAMIC_FTRACE */
959
EXPORT_SYMBOL(mcount)
960
#endif /* CONFIG_FUNCTION_TRACER */
961

962 963
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
ENTRY(ftrace_graph_caller)
964 965 966 967 968 969 970 971 972 973 974
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	movl	0xc(%esp), %eax
	lea	0x4(%ebp), %edx
	movl	(%ebp), %ecx
	subl	$MCOUNT_INSN_SIZE, %eax
	call	prepare_ftrace_return
	popl	%edx
	popl	%ecx
	popl	%eax
975
	ret
976
END(ftrace_graph_caller)
977 978 979

.globl return_to_handler
return_to_handler:
980 981 982 983 984 985 986 987
	pushl	%eax
	pushl	%edx
	movl	%ebp, %eax
	call	ftrace_return_to_handler
	movl	%eax, %ecx
	popl	%edx
	popl	%eax
	jmp	*%ecx
988
#endif
989

990 991 992
#ifdef CONFIG_TRACING
ENTRY(trace_page_fault)
	ASM_CLAC
993 994
	pushl	$trace_do_page_fault
	jmp	error_code
995 996 997
END(trace_page_fault)
#endif

998
ENTRY(page_fault)
999
	ASM_CLAC
1000
	pushl	$do_page_fault
1001 1002
	ALIGN
error_code:
1003
	/* the function address is in %gs's slot on the stack */
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	pushl	%fs
	pushl	%es
	pushl	%ds
	pushl	%eax
	pushl	%ebp
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx
1014
	cld
1015 1016
	movl	$(__KERNEL_PERCPU), %ecx
	movl	%ecx, %fs
1017
	UNWIND_ESPFIX_STACK
1018
	GS_TO_REG %ecx
1019 1020 1021
	movl	PT_GS(%esp), %edi		# get the function address
	movl	PT_ORIG_EAX(%esp), %edx		# get the error code
	movl	$-1, PT_ORIG_EAX(%esp)		# no syscall to restart
1022 1023
	REG_TO_PTGS %ecx
	SET_KERNEL_GS %ecx
1024 1025 1026
	movl	$(__USER_DS), %ecx
	movl	%ecx, %ds
	movl	%ecx, %es
1027
	TRACE_IRQS_OFF
1028 1029 1030
	movl	%esp, %eax			# pt_regs pointer
	call	*%edi
	jmp	ret_from_exception
1031 1032 1033
END(page_fault)

ENTRY(debug)
1034 1035 1036 1037 1038 1039 1040 1041 1042
	/*
	 * #DB can happen at the first instruction of
	 * entry_SYSENTER_32 or in Xen's SYSENTER prologue.  If this
	 * happens, then we will be running on a very small stack.  We
	 * need to detect this condition and switch to the thread
	 * stack before calling any C code at all.
	 *
	 * If you edit this code, keep in mind that NMIs can happen in here.
	 */
1043
	ASM_CLAC
1044
	pushl	$-1				# mark this as an int
1045
	SAVE_ALL
1046 1047
	xorl	%edx, %edx			# error code 0
	movl	%esp, %eax			# pt_regs pointer
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063

	/* Are we currently on the SYSENTER stack? */
	PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
	subl	%eax, %ecx	/* ecx = (end of SYSENTER_stack) - esp */
	cmpl	$SIZEOF_SYSENTER_stack, %ecx
	jb	.Ldebug_from_sysenter_stack

	TRACE_IRQS_OFF
	call	do_debug
	jmp	ret_from_exception

.Ldebug_from_sysenter_stack:
	/* We're on the SYSENTER stack.  Switch off. */
	movl	%esp, %ebp
	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esp
	TRACE_IRQS_OFF
1064
	call	do_debug
1065
	movl	%ebp, %esp
1066
	jmp	ret_from_exception
1067 1068 1069
END(debug)

/*
1070 1071 1072 1073 1074
 * NMI is doubly nasty.  It can happen on the first instruction of
 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
 * switched stacks.  We handle both conditions by simply checking whether we
 * interrupted kernel code running on the SYSENTER stack.
1075 1076
 */
ENTRY(nmi)
1077
	ASM_CLAC
1078
#ifdef CONFIG_X86_ESPFIX32
1079 1080 1081 1082 1083
	pushl	%eax
	movl	%ss, %eax
	cmpw	$__ESPFIX_SS, %ax
	popl	%eax
	je	nmi_espfix_stack
1084
#endif
1085 1086

	pushl	%eax				# pt_regs->orig_ax
1087
	SAVE_ALL
1088 1089
	xorl	%edx, %edx			# zero error code
	movl	%esp, %eax			# pt_regs pointer
1090 1091 1092 1093 1094 1095 1096 1097

	/* Are we currently on the SYSENTER stack? */
	PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
	subl	%eax, %ecx	/* ecx = (end of SYSENTER_stack) - esp */
	cmpl	$SIZEOF_SYSENTER_stack, %ecx
	jb	.Lnmi_from_sysenter_stack

	/* Not on SYSENTER stack. */
1098 1099
	call	do_nmi
	jmp	restore_all_notrace
1100

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
.Lnmi_from_sysenter_stack:
	/*
	 * We're on the SYSENTER stack.  Switch off.  No one (not even debug)
	 * is using the thread stack right now, so it's safe for us to use it.
	 */
	movl	%esp, %ebp
	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esp
	call	do_nmi
	movl	%ebp, %esp
	jmp	restore_all_notrace
1111

1112
#ifdef CONFIG_X86_ESPFIX32
1113
nmi_espfix_stack:
1114
	/*
1115 1116
	 * create the pointer to lss back
	 */
1117 1118 1119
	pushl	%ss
	pushl	%esp
	addl	$4, (%esp)
1120 1121
	/* copy the iret frame of 12 bytes */
	.rept 3
1122
	pushl	16(%esp)
1123
	.endr
1124
	pushl	%eax
1125
	SAVE_ALL
1126 1127 1128
	FIXUP_ESPFIX_STACK			# %eax == %esp
	xorl	%edx, %edx			# zero error code
	call	do_nmi
1129
	RESTORE_REGS
1130 1131
	lss	12+4(%esp), %esp		# back to espfix stack
	jmp	irq_return
1132
#endif
1133 1134 1135
END(nmi)

ENTRY(int3)
1136
	ASM_CLAC
1137
	pushl	$-1				# mark this as an int
1138 1139
	SAVE_ALL
	TRACE_IRQS_OFF
1140 1141 1142 1143
	xorl	%edx, %edx			# zero error code
	movl	%esp, %eax			# pt_regs pointer
	call	do_int3
	jmp	ret_from_exception
1144 1145 1146
END(int3)

ENTRY(general_protection)
1147 1148
	pushl	$do_general_protection
	jmp	error_code
1149 1150
END(general_protection)

G
Gleb Natapov 已提交
1151 1152
#ifdef CONFIG_KVM_GUEST
ENTRY(async_page_fault)
1153
	ASM_CLAC
1154 1155
	pushl	$do_async_page_fault
	jmp	error_code
1156
END(async_page_fault)
G
Gleb Natapov 已提交
1157
#endif
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168

ENTRY(rewind_stack_do_exit)
	/* Prevent any naive code from trying to unwind to our caller. */
	xorl	%ebp, %ebp

	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esi
	leal	-TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp

	call	do_exit
1:	jmp 1b
END(rewind_stack_do_exit)