entry_32.S 26.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 * This also contains the timer-interrupt handler, as well as all interrupts
 * and faults that can result in a task-switch.
 *
 * NOTE: This code handles signal-recognition, which happens every time
 * after a timer-interrupt and after each system call.
 *
 * I changed all the .align's to 4 (16 byte alignment), as that's faster
 * on a 486.
 *
17
 * Stack layout in 'syscall_exit':
L
Linus Torvalds 已提交
18 19 20 21 22 23 24 25 26 27 28 29 30 31
 * 	ptrace needs to have all regs on the stack.
 *	if the order here is changed, it needs to be
 *	updated in fork.c:copy_process, signal.c:do_signal,
 *	ptrace.c and ptrace.h
 *
 *	 0(%esp) - %ebx
 *	 4(%esp) - %ecx
 *	 8(%esp) - %edx
 *       C(%esp) - %esi
 *	10(%esp) - %edi
 *	14(%esp) - %ebp
 *	18(%esp) - %eax
 *	1C(%esp) - %ds
 *	20(%esp) - %es
32
 *	24(%esp) - %fs
33 34 35 36 37 38
 *	28(%esp) - orig_eax
 *	2C(%esp) - %eip
 *	30(%esp) - %cs
 *	34(%esp) - %eflags
 *	38(%esp) - %oldesp
 *	3C(%esp) - %oldss
L
Linus Torvalds 已提交
39 40 41 42 43 44
 *
 * "current" is in register %ebx during any slow entries.
 */

#include <linux/linkage.h>
#include <asm/thread_info.h>
45
#include <asm/irqflags.h>
L
Linus Torvalds 已提交
46 47 48 49 50
#include <asm/errno.h>
#include <asm/segment.h>
#include <asm/smp.h>
#include <asm/page.h>
#include <asm/desc.h>
S
Stas Sergeev 已提交
51
#include <asm/percpu.h>
52
#include <asm/dwarf2.h>
53
#include <asm/processor-flags.h>
54
#include <asm/ftrace.h>
55
#include <asm/irq_vectors.h>
L
Linus Torvalds 已提交
56

57 58 59 60 61
/*
 * We use macros for low-level operations which need to be overridden
 * for paravirtualization.  The following will never clobber any registers:
 *   INTERRUPT_RETURN (aka. "iret")
 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
62
 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
63 64 65 66 67 68 69
 *
 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
 * Allowing a register to be clobbered can shrink the paravirt replacement
 * enough to patch inline, increasing performance.
 */

L
Linus Torvalds 已提交
70 71 72
#define nr_syscalls ((syscall_table_size)/4)

#ifdef CONFIG_PREEMPT
73
#define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
L
Linus Torvalds 已提交
74
#else
75
#define preempt_stop(clobbers)
L
Linus Torvalds 已提交
76 77 78
#define resume_kernel		restore_nocheck
#endif

79 80
.macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS
81
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
82 83 84 85 86 87
	jz 1f
	TRACE_IRQS_ON
1:
#endif
.endm

88 89 90 91 92 93
#ifdef CONFIG_VM86
#define resume_userspace_sig	check_userspace
#else
#define resume_userspace_sig	resume_userspace
#endif

L
Linus Torvalds 已提交
94 95
#define SAVE_ALL \
	cld; \
96
	pushl %fs; \
97
	CFI_ADJUST_CFA_OFFSET 4;\
98
	/*CFI_REL_OFFSET fs, 0;*/\
L
Linus Torvalds 已提交
99
	pushl %es; \
100 101
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET es, 0;*/\
L
Linus Torvalds 已提交
102
	pushl %ds; \
103 104
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET ds, 0;*/\
L
Linus Torvalds 已提交
105
	pushl %eax; \
106 107
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET eax, 0;\
L
Linus Torvalds 已提交
108
	pushl %ebp; \
109 110
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebp, 0;\
L
Linus Torvalds 已提交
111
	pushl %edi; \
112 113
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edi, 0;\
L
Linus Torvalds 已提交
114
	pushl %esi; \
115 116
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET esi, 0;\
L
Linus Torvalds 已提交
117
	pushl %edx; \
118 119
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edx, 0;\
L
Linus Torvalds 已提交
120
	pushl %ecx; \
121 122
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ecx, 0;\
L
Linus Torvalds 已提交
123
	pushl %ebx; \
124 125
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebx, 0;\
L
Linus Torvalds 已提交
126 127
	movl $(__USER_DS), %edx; \
	movl %edx, %ds; \
128
	movl %edx, %es; \
129
	movl $(__KERNEL_PERCPU), %edx; \
130
	movl %edx, %fs
L
Linus Torvalds 已提交
131 132 133

#define RESTORE_INT_REGS \
	popl %ebx;	\
134 135
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebx;\
L
Linus Torvalds 已提交
136
	popl %ecx;	\
137 138
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ecx;\
L
Linus Torvalds 已提交
139
	popl %edx;	\
140 141
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edx;\
L
Linus Torvalds 已提交
142
	popl %esi;	\
143 144
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE esi;\
L
Linus Torvalds 已提交
145
	popl %edi;	\
146 147
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edi;\
L
Linus Torvalds 已提交
148
	popl %ebp;	\
149 150 151 152 153
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebp;\
	popl %eax;	\
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE eax
L
Linus Torvalds 已提交
154 155 156 157

#define RESTORE_REGS	\
	RESTORE_INT_REGS; \
1:	popl %ds;	\
158 159
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE ds;*/\
L
Linus Torvalds 已提交
160
2:	popl %es;	\
161 162
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE es;*/\
163
3:	popl %fs;	\
164
	CFI_ADJUST_CFA_OFFSET -4;\
165
	/*CFI_RESTORE fs;*/\
166
.pushsection .fixup,"ax";	\
L
Linus Torvalds 已提交
167
4:	movl $0,(%esp);	\
168 169
	jmp 1b;		\
5:	movl $0,(%esp);	\
L
Linus Torvalds 已提交
170
	jmp 2b;		\
171 172
6:	movl $0,(%esp);	\
	jmp 3b;		\
L
Linus Torvalds 已提交
173 174
.section __ex_table,"a";\
	.align 4;	\
175 176 177 178
	.long 1b,4b;	\
	.long 2b,5b;	\
	.long 3b,6b;	\
.popsection
L
Linus Torvalds 已提交
179

180 181
#define RING0_INT_FRAME \
	CFI_STARTPROC simple;\
182
	CFI_SIGNAL_FRAME;\
183 184 185 186 187 188
	CFI_DEF_CFA esp, 3*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_EC_FRAME \
	CFI_STARTPROC simple;\
189
	CFI_SIGNAL_FRAME;\
190 191 192 193 194 195
	CFI_DEF_CFA esp, 4*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_PTREGS_FRAME \
	CFI_STARTPROC simple;\
196
	CFI_SIGNAL_FRAME;\
197 198 199 200 201 202 203 204 205 206 207 208
	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
	CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
	CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
	CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
	CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
	CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
	CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
	CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
L
Linus Torvalds 已提交
209 210

ENTRY(ret_from_fork)
211
	CFI_STARTPROC
L
Linus Torvalds 已提交
212
	pushl %eax
213
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
214 215 216
	call schedule_tail
	GET_THREAD_INFO(%ebp)
	popl %eax
217
	CFI_ADJUST_CFA_OFFSET -4
218 219 220 221
	pushl $0x0202			# Reset kernel eflags
	CFI_ADJUST_CFA_OFFSET 4
	popfl
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
222
	jmp syscall_exit
223
	CFI_ENDPROC
224
END(ret_from_fork)
L
Linus Torvalds 已提交
225 226 227 228 229 230 231 232 233 234

/*
 * Return to user mode is not as complex as all this looks,
 * but we want the default path for a system call return to
 * go as quickly as possible which is why some of this is
 * less clear than it otherwise should be.
 */

	# userspace resumption stub bypassing syscall exit tracing
	ALIGN
235
	RING0_PTREGS_FRAME
L
Linus Torvalds 已提交
236
ret_from_exception:
237
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
238 239
ret_from_intr:
	GET_THREAD_INFO(%ebp)
240
check_userspace:
241 242
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
	movb PT_CS(%esp), %al
243
	andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
244 245
	cmpl $USER_RPL, %eax
	jb resume_kernel		# not returning to v8086 or userspace
246

L
Linus Torvalds 已提交
247
ENTRY(resume_userspace)
248
	LOCKDEP_SYS_EXIT
249
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
250 251
					# setting need_resched or sigpending
					# between sampling and the iret
252
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
253 254 255 256 257
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
					# int/exception return?
	jne work_pending
	jmp restore_all
258
END(ret_from_exception)
L
Linus Torvalds 已提交
259 260 261

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
262
	DISABLE_INTERRUPTS(CLBR_ANY)
L
Linus Torvalds 已提交
263 264 265 266 267 268
	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
	jnz restore_nocheck
need_resched:
	movl TI_flags(%ebp), %ecx	# need_resched set ?
	testb $_TIF_NEED_RESCHED, %cl
	jz restore_all
269
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
L
Linus Torvalds 已提交
270 271 272
	jz restore_all
	call preempt_schedule_irq
	jmp need_resched
273
END(resume_kernel)
L
Linus Torvalds 已提交
274
#endif
275
	CFI_ENDPROC
L
Linus Torvalds 已提交
276 277 278 279 280

/* SYSENTER_RETURN points to after the "sysenter" instruction in
   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */

	# sysenter call handler stub
R
Roland McGrath 已提交
281
ENTRY(ia32_sysenter_target)
282
	CFI_STARTPROC simple
283
	CFI_SIGNAL_FRAME
284 285
	CFI_DEF_CFA esp, 0
	CFI_REGISTER esp, ebp
286
	movl TSS_sysenter_sp0(%esp),%esp
L
Linus Torvalds 已提交
287
sysenter_past_esp:
288
	/*
289 290 291
	 * Interrupts are disabled here, but we can't trace it until
	 * enough kernel state to call TRACE_IRQS_OFF can be called - but
	 * we immediately enable interrupts at that point anyway.
292
	 */
L
Linus Torvalds 已提交
293
	pushl $(__USER_DS)
294 295
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ss, 0*/
L
Linus Torvalds 已提交
296
	pushl %ebp
297 298
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esp, 0
L
Linus Torvalds 已提交
299
	pushfl
300
	orl $X86_EFLAGS_IF, (%esp)
301
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
302
	pushl $(__USER_CS)
303 304
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET cs, 0*/
305 306 307 308 309 310
	/*
	 * Push current_thread_info()->sysenter_return to the stack.
	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
	 */
	pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
311 312
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
313

314 315 316 317 318
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	ENABLE_INTERRUPTS(CLBR_NONE)

L
Linus Torvalds 已提交
319 320 321 322 323 324 325
/*
 * Load the potential sixth argument from user stack.
 * Careful about security.
 */
	cmpl $__PAGE_OFFSET-3,%ebp
	jae syscall_fault
1:	movl (%ebp),%ebp
326
	movl %ebp,PT_EBP(%esp)
L
Linus Torvalds 已提交
327 328 329 330 331 332 333 334
.section __ex_table,"a"
	.align 4
	.long 1b,syscall_fault
.previous

	GET_THREAD_INFO(%ebp)

	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
335
	testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
L
Linus Torvalds 已提交
336 337 338 339
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
	call *sys_call_table(,%eax,4)
340
	movl %eax,PT_EAX(%esp)
341
	LOCKDEP_SYS_EXIT
342
	DISABLE_INTERRUPTS(CLBR_ANY)
343
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
344 345 346 347
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx
	jne syscall_exit_work
/* if something modifies registers it must also disable sysexit */
348 349
	movl PT_EIP(%esp), %edx
	movl PT_OLDESP(%esp), %ecx
L
Linus Torvalds 已提交
350
	xorl %ebp,%ebp
351
	TRACE_IRQS_ON
352
1:	mov  PT_FS(%esp), %fs
353
	ENABLE_INTERRUPTS_SYSEXIT
354
	CFI_ENDPROC
355
.pushsection .fixup,"ax"
356
2:	movl $0,PT_FS(%esp)
357 358 359 360 361
	jmp 1b
.section __ex_table,"a"
	.align 4
	.long 1b,2b
.popsection
R
Roland McGrath 已提交
362
ENDPROC(ia32_sysenter_target)
L
Linus Torvalds 已提交
363 364 365

	# system call handler stub
ENTRY(system_call)
366
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
367
	pushl %eax			# save orig_eax
368
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
369 370
	SAVE_ALL
	GET_THREAD_INFO(%ebp)
371
					# system call tracing in operation / emulation
L
Linus Torvalds 已提交
372
	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
373
	testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
L
Linus Torvalds 已提交
374 375 376 377 378
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
syscall_call:
	call *sys_call_table(,%eax,4)
379
	movl %eax,PT_EAX(%esp)		# store the return value
L
Linus Torvalds 已提交
380
syscall_exit:
381
	LOCKDEP_SYS_EXIT
382
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
383 384
					# setting need_resched or sigpending
					# between sampling and the iret
385
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
386 387 388 389 390
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx	# current->work
	jne syscall_exit_work

restore_all:
391 392
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
393 394
	# are returning to the kernel.
	# See comments in process.c:copy_thread() for details.
395 396
	movb PT_OLDSS(%esp), %ah
	movb PT_CS(%esp), %al
397
	andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
398
	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
399
	CFI_REMEMBER_STATE
L
Linus Torvalds 已提交
400 401
	je ldt_ss			# returning to user-space with LDT SS
restore_nocheck:
402 403
	TRACE_IRQS_IRET
restore_nocheck_notrace:
L
Linus Torvalds 已提交
404
	RESTORE_REGS
405
	addl $4, %esp			# skip orig_eax/error_code
406
	CFI_ADJUST_CFA_OFFSET -4
A
Adrian Bunk 已提交
407
irq_return:
I
Ingo Molnar 已提交
408
	INTERRUPT_RETURN
L
Linus Torvalds 已提交
409
.section .fixup,"ax"
410
ENTRY(iret_exc)
411 412 413
	pushl $0			# no error code
	pushl $do_iret_error
	jmp error_code
L
Linus Torvalds 已提交
414 415 416
.previous
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
417
	.long irq_return,iret_exc
L
Linus Torvalds 已提交
418 419
.previous

420
	CFI_RESTORE_STATE
L
Linus Torvalds 已提交
421
ldt_ss:
422
	larl PT_OLDSS(%esp), %eax
L
Linus Torvalds 已提交
423 424 425
	jnz restore_nocheck
	testl $0x00400000, %eax		# returning to 32bit stack?
	jnz restore_nocheck		# allright, normal return
426 427 428 429 430 431 432 433 434 435

#ifdef CONFIG_PARAVIRT
	/*
	 * The kernel can't run on a non-flat stack if paravirt mode
	 * is active.  Rather than try to fixup the high bits of
	 * ESP, bypass this code entirely.  This may break DOSemu
	 * and/or Wine support in a paravirt VM, although the option
	 * is still available to implement the setting of the high
	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
	 */
436
	cmpl $0, pv_info+PARAVIRT_enabled
437 438 439
	jne restore_nocheck
#endif

L
Linus Torvalds 已提交
440 441 442 443 444 445
	/* If returning to userspace with 16bit stack,
	 * try to fix the higher word of ESP, as the CPU
	 * won't restore it.
	 * This is an "official" bug of all the x86-compatible
	 * CPUs, which we can try to work around to make
	 * dosemu and wine happy. */
446
	movl PT_OLDESP(%esp), %eax
S
Stas Sergeev 已提交
447 448 449 450 451 452
	movl %esp, %edx
	call patch_espfix_desc
	pushl $__ESPFIX_SS
	CFI_ADJUST_CFA_OFFSET 4
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
453
	DISABLE_INTERRUPTS(CLBR_EAX)
454
	TRACE_IRQS_OFF
S
Stas Sergeev 已提交
455 456 457
	lss (%esp), %esp
	CFI_ADJUST_CFA_OFFSET -8
	jmp restore_nocheck
458
	CFI_ENDPROC
459
ENDPROC(system_call)
L
Linus Torvalds 已提交
460 461 462

	# perform work that needs to be done immediately before resumption
	ALIGN
463
	RING0_PTREGS_FRAME		# can't unwind into user space anyway
L
Linus Torvalds 已提交
464 465 466 467 468
work_pending:
	testb $_TIF_NEED_RESCHED, %cl
	jz work_notifysig
work_resched:
	call schedule
469
	LOCKDEP_SYS_EXIT
470
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
471 472
					# setting need_resched or sigpending
					# between sampling and the iret
473
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
474 475 476 477 478 479 480 481 482
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
					# than syscall tracing?
	jz restore_all
	testb $_TIF_NEED_RESCHED, %cl
	jnz work_resched

work_notifysig:				# deal with pending signals and
					# notify-resume requests
483
#ifdef CONFIG_VM86
484
	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
L
Linus Torvalds 已提交
485 486 487 488 489
	movl %esp, %eax
	jne work_notifysig_v86		# returning to kernel-space or
					# vm86-space
	xorl %edx, %edx
	call do_notify_resume
490
	jmp resume_userspace_sig
L
Linus Torvalds 已提交
491 492 493 494

	ALIGN
work_notifysig_v86:
	pushl %ecx			# save ti_flags for do_notify_resume
495
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
496 497
	call save_v86_state		# %eax contains pt_regs pointer
	popl %ecx
498
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
499
	movl %eax, %esp
500 501 502
#else
	movl %esp, %eax
#endif
L
Linus Torvalds 已提交
503 504
	xorl %edx, %edx
	call do_notify_resume
505
	jmp resume_userspace_sig
506
END(work_pending)
L
Linus Torvalds 已提交
507 508 509 510

	# perform syscall exit tracing
	ALIGN
syscall_trace_entry:
511
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
512
	movl %esp, %eax
513 514
	call syscall_trace_enter
	/* What it returned is what we'll actually use.  */
L
Linus Torvalds 已提交
515 516 517
	cmpl $(nr_syscalls), %eax
	jnae syscall_call
	jmp syscall_exit
518
END(syscall_trace_entry)
L
Linus Torvalds 已提交
519 520 521 522

	# perform syscall exit tracing
	ALIGN
syscall_exit_work:
523
	testb $_TIF_WORK_SYSCALL_EXIT, %cl
L
Linus Torvalds 已提交
524
	jz work_pending
525
	TRACE_IRQS_ON
526
	ENABLE_INTERRUPTS(CLBR_ANY)	# could let syscall_trace_leave() call
L
Linus Torvalds 已提交
527 528
					# schedule() instead
	movl %esp, %eax
529
	call syscall_trace_leave
L
Linus Torvalds 已提交
530
	jmp resume_userspace
531
END(syscall_exit_work)
532
	CFI_ENDPROC
L
Linus Torvalds 已提交
533

534
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
535 536
syscall_fault:
	GET_THREAD_INFO(%ebp)
537
	movl $-EFAULT,PT_EAX(%esp)
L
Linus Torvalds 已提交
538
	jmp resume_userspace
539
END(syscall_fault)
L
Linus Torvalds 已提交
540 541

syscall_badsys:
542
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
543
	jmp resume_userspace
544
END(syscall_badsys)
545
	CFI_ENDPROC
L
Linus Torvalds 已提交
546 547

#define FIXUP_ESPFIX_STACK \
S
Stas Sergeev 已提交
548
	/* since we are on a wrong stack, we cant make it a C code :( */ \
549
	PER_CPU(gdt_page, %ebx); \
S
Stas Sergeev 已提交
550 551 552 553
	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
	addl %esp, %eax; \
	pushl $__KERNEL_DS; \
	CFI_ADJUST_CFA_OFFSET 4; \
L
Linus Torvalds 已提交
554
	pushl %eax; \
555
	CFI_ADJUST_CFA_OFFSET 4; \
S
Stas Sergeev 已提交
556 557 558
	lss (%esp), %esp; \
	CFI_ADJUST_CFA_OFFSET -8;
#define UNWIND_ESPFIX_STACK \
L
Linus Torvalds 已提交
559
	movl %ss, %eax; \
S
Stas Sergeev 已提交
560
	/* see if on espfix stack */ \
L
Linus Torvalds 已提交
561
	cmpw $__ESPFIX_SS, %ax; \
S
Stas Sergeev 已提交
562 563
	jne 27f; \
	movl $__KERNEL_DS, %eax; \
564 565
	movl %eax, %ds; \
	movl %eax, %es; \
S
Stas Sergeev 已提交
566
	/* switch to normal stack */ \
567
	FIXUP_ESPFIX_STACK; \
S
Stas Sergeev 已提交
568
27:;
L
Linus Torvalds 已提交
569 570 571 572 573

/*
 * Build the entry stubs and pointer table with
 * some assembler magic.
 */
J
Jan Beulich 已提交
574
.section .rodata,"a"
L
Linus Torvalds 已提交
575 576 577 578
ENTRY(interrupt)
.text

ENTRY(irq_entries_start)
579
	RING0_INT_FRAME
580
vector=0
L
Linus Torvalds 已提交
581 582
.rept NR_IRQS
	ALIGN
583 584 585
 .if vector
	CFI_ADJUST_CFA_OFFSET -4
 .endif
586
1:	pushl $~(vector)
587
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
588
	jmp common_interrupt
589
 .previous
L
Linus Torvalds 已提交
590
	.long 1b
591
 .text
L
Linus Torvalds 已提交
592 593
vector=vector+1
.endr
594 595 596 597 598
END(irq_entries_start)

.previous
END(interrupt)
.previous
L
Linus Torvalds 已提交
599

600 601 602 603
/*
 * the CPU automatically disables interrupts when executing an IRQ vector,
 * so IRQ-flags tracing has to follow that:
 */
L
Linus Torvalds 已提交
604 605 606
	ALIGN
common_interrupt:
	SAVE_ALL
607
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
608 609 610
	movl %esp,%eax
	call do_IRQ
	jmp ret_from_intr
611
ENDPROC(common_interrupt)
612
	CFI_ENDPROC
L
Linus Torvalds 已提交
613 614 615

#define BUILD_INTERRUPT(name, nr)	\
ENTRY(name)				\
616
	RING0_INT_FRAME;		\
617
	pushl $~(nr);			\
618 619
	CFI_ADJUST_CFA_OFFSET 4;	\
	SAVE_ALL;			\
620
	TRACE_IRQS_OFF			\
L
Linus Torvalds 已提交
621
	movl %esp,%eax;			\
622
	call smp_##name;		\
623
	jmp ret_from_intr;		\
624 625
	CFI_ENDPROC;			\
ENDPROC(name)
L
Linus Torvalds 已提交
626 627 628 629

/* The include is where all of the SMP etc. interrupts come from */
#include "entry_arch.h"

630 631 632
KPROBE_ENTRY(page_fault)
	RING0_EC_FRAME
	pushl $do_page_fault
633
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
634 635
	ALIGN
error_code:
636
	/* the function address is in %fs's slot on the stack */
637 638 639
	pushl %es
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET es, 0*/
L
Linus Torvalds 已提交
640
	pushl %ds
641 642
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ds, 0*/
L
Linus Torvalds 已提交
643
	pushl %eax
644 645
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eax, 0
L
Linus Torvalds 已提交
646
	pushl %ebp
647 648
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebp, 0
L
Linus Torvalds 已提交
649
	pushl %edi
650 651
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edi, 0
L
Linus Torvalds 已提交
652
	pushl %esi
653 654
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esi, 0
L
Linus Torvalds 已提交
655
	pushl %edx
656 657
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edx, 0
L
Linus Torvalds 已提交
658
	pushl %ecx
659 660
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ecx, 0
L
Linus Torvalds 已提交
661
	pushl %ebx
662 663
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebx, 0
L
Linus Torvalds 已提交
664
	cld
665
	pushl %fs
666
	CFI_ADJUST_CFA_OFFSET 4
667
	/*CFI_REL_OFFSET fs, 0*/
668
	movl $(__KERNEL_PERCPU), %ecx
669
	movl %ecx, %fs
L
Linus Torvalds 已提交
670 671
	UNWIND_ESPFIX_STACK
	popl %ecx
672 673
	CFI_ADJUST_CFA_OFFSET -4
	/*CFI_REGISTER es, ecx*/
674
	movl PT_FS(%esp), %edi		# get the function address
675
	movl PT_ORIG_EAX(%esp), %edx	# get the error code
676
	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
677 678
	mov  %ecx, PT_FS(%esp)
	/*CFI_REL_OFFSET fs, ES*/
L
Linus Torvalds 已提交
679 680 681 682 683 684
	movl $(__USER_DS), %ecx
	movl %ecx, %ds
	movl %ecx, %es
	movl %esp,%eax			# pt_regs pointer
	call *%edi
	jmp ret_from_exception
685
	CFI_ENDPROC
686
KPROBE_END(page_fault)
L
Linus Torvalds 已提交
687 688

ENTRY(coprocessor_error)
689
	RING0_INT_FRAME
L
Linus Torvalds 已提交
690
	pushl $0
691
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
692
	pushl $do_coprocessor_error
693
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
694
	jmp error_code
695
	CFI_ENDPROC
696
END(coprocessor_error)
L
Linus Torvalds 已提交
697 698

ENTRY(simd_coprocessor_error)
699
	RING0_INT_FRAME
L
Linus Torvalds 已提交
700
	pushl $0
701
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
702
	pushl $do_simd_coprocessor_error
703
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
704
	jmp error_code
705
	CFI_ENDPROC
706
END(simd_coprocessor_error)
L
Linus Torvalds 已提交
707 708

ENTRY(device_not_available)
709
	RING0_INT_FRAME
L
Linus Torvalds 已提交
710
	pushl $-1			# mark this as an int
711
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
712
	SAVE_ALL
713
	GET_CR0_INTO_EAX
L
Linus Torvalds 已提交
714 715
	testl $0x4, %eax		# EM (math emulation bit)
	jne device_not_available_emulate
716
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
717 718 719 720
	call math_state_restore
	jmp ret_from_exception
device_not_available_emulate:
	pushl $0			# temporary storage for ORIG_EIP
721
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
722 723
	call math_emulate
	addl $4, %esp
724
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
725
	jmp ret_from_exception
726
	CFI_ENDPROC
727
END(device_not_available)
L
Linus Torvalds 已提交
728 729 730 731 732 733

/*
 * Debug traps and NMI can happen at the one SYSENTER instruction
 * that sets up the real kernel stack. Check here, since we can't
 * allow the wrong stack to be used.
 *
734
 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
L
Linus Torvalds 已提交
735 736 737 738 739 740 741 742 743 744 745
 * already pushed 3 words if it hits on the sysenter instruction:
 * eflags, cs and eip.
 *
 * We just load the right stack, and push the three (known) values
 * by hand onto the new stack - while updating the return eip past
 * the instruction that would have done it for sysenter.
 */
#define FIX_STACK(offset, ok, label)		\
	cmpw $__KERNEL_CS,4(%esp);		\
	jne ok;					\
label:						\
746
	movl TSS_sysenter_sp0+offset(%esp),%esp;	\
747 748
	CFI_DEF_CFA esp, 0;			\
	CFI_UNDEFINED eip;			\
L
Linus Torvalds 已提交
749
	pushfl;					\
750
	CFI_ADJUST_CFA_OFFSET 4;		\
L
Linus Torvalds 已提交
751
	pushl $__KERNEL_CS;			\
752 753 754 755
	CFI_ADJUST_CFA_OFFSET 4;		\
	pushl $sysenter_past_esp;		\
	CFI_ADJUST_CFA_OFFSET 4;		\
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
756

757
KPROBE_ENTRY(debug)
758
	RING0_INT_FRAME
R
Roland McGrath 已提交
759
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
760 761 762 763
	jne debug_stack_correct
	FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
debug_stack_correct:
	pushl $-1			# mark this as an int
764
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
765 766 767 768 769
	SAVE_ALL
	xorl %edx,%edx			# error code 0
	movl %esp,%eax			# pt_regs pointer
	call do_debug
	jmp ret_from_exception
770
	CFI_ENDPROC
771 772
KPROBE_END(debug)

L
Linus Torvalds 已提交
773 774 775 776 777 778 779 780
/*
 * NMI is doubly nasty. It can happen _while_ we're handling
 * a debug fault, and the debug fault hasn't yet been able to
 * clear up the stack. So we first check whether we got  an
 * NMI on the sysenter entry path, but after that we need to
 * check whether we got an NMI on the debug path where the debug
 * fault happened on the sysenter path.
 */
781
KPROBE_ENTRY(nmi)
782
	RING0_INT_FRAME
L
Linus Torvalds 已提交
783
	pushl %eax
784
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
785 786 787
	movl %ss, %eax
	cmpw $__ESPFIX_SS, %ax
	popl %eax
788
	CFI_ADJUST_CFA_OFFSET -4
S
Stas Sergeev 已提交
789
	je nmi_espfix_stack
R
Roland McGrath 已提交
790
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
791 792
	je nmi_stack_fixup
	pushl %eax
793
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
794 795 796 797 798 799 800
	movl %esp,%eax
	/* Do not access memory above the end of our stack page,
	 * it might not exist.
	 */
	andl $(THREAD_SIZE-1),%eax
	cmpl $(THREAD_SIZE-20),%eax
	popl %eax
801
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
802
	jae nmi_stack_correct
R
Roland McGrath 已提交
803
	cmpl $ia32_sysenter_target,12(%esp)
L
Linus Torvalds 已提交
804 805
	je nmi_debug_stack_check
nmi_stack_correct:
806
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
807
	pushl %eax
808
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
809 810 811 812
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_nmi
813
	jmp restore_nocheck_notrace
814
	CFI_ENDPROC
L
Linus Torvalds 已提交
815 816

nmi_stack_fixup:
817
	RING0_INT_FRAME
L
Linus Torvalds 已提交
818 819
	FIX_STACK(12,nmi_stack_correct, 1)
	jmp nmi_stack_correct
820

L
Linus Torvalds 已提交
821
nmi_debug_stack_check:
822
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
823 824
	cmpw $__KERNEL_CS,16(%esp)
	jne nmi_stack_correct
825 826
	cmpl $debug,(%esp)
	jb nmi_stack_correct
L
Linus Torvalds 已提交
827
	cmpl $debug_esp_fix_insn,(%esp)
828
	ja nmi_stack_correct
L
Linus Torvalds 已提交
829 830 831
	FIX_STACK(24,nmi_stack_correct, 1)
	jmp nmi_stack_correct

S
Stas Sergeev 已提交
832
nmi_espfix_stack:
833 834 835 836
	/* We have a RING0_INT_FRAME here.
	 *
	 * create the pointer to lss back
	 */
L
Linus Torvalds 已提交
837
	pushl %ss
838
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
839
	pushl %esp
840
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
841 842 843 844
	addw $4, (%esp)
	/* copy the iret frame of 12 bytes */
	.rept 3
	pushl 16(%esp)
845
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
846 847
	.endr
	pushl %eax
848
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
849 850 851 852 853
	SAVE_ALL
	FIXUP_ESPFIX_STACK		# %eax == %esp
	xorl %edx,%edx			# zero error code
	call do_nmi
	RESTORE_REGS
S
Stas Sergeev 已提交
854 855
	lss 12+4(%esp), %esp		# back to espfix stack
	CFI_ADJUST_CFA_OFFSET -24
I
Ingo Molnar 已提交
856
	jmp irq_return
857
	CFI_ENDPROC
858
KPROBE_END(nmi)
L
Linus Torvalds 已提交
859

860 861
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
I
Ingo Molnar 已提交
862
	iret
863 864
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
865
	.long native_iret, iret_exc
866
.previous
867
END(native_iret)
868

869
ENTRY(native_irq_enable_sysexit)
870 871
	sti
	sysexit
872
END(native_irq_enable_sysexit)
873 874
#endif

875
KPROBE_ENTRY(int3)
876
	RING0_INT_FRAME
L
Linus Torvalds 已提交
877
	pushl $-1			# mark this as an int
878
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
879 880 881 882 883
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_int3
	jmp ret_from_exception
884
	CFI_ENDPROC
885
KPROBE_END(int3)
L
Linus Torvalds 已提交
886 887

ENTRY(overflow)
888
	RING0_INT_FRAME
L
Linus Torvalds 已提交
889
	pushl $0
890
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
891
	pushl $do_overflow
892
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
893
	jmp error_code
894
	CFI_ENDPROC
895
END(overflow)
L
Linus Torvalds 已提交
896 897

ENTRY(bounds)
898
	RING0_INT_FRAME
L
Linus Torvalds 已提交
899
	pushl $0
900
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
901
	pushl $do_bounds
902
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
903
	jmp error_code
904
	CFI_ENDPROC
905
END(bounds)
L
Linus Torvalds 已提交
906 907

ENTRY(invalid_op)
908
	RING0_INT_FRAME
L
Linus Torvalds 已提交
909
	pushl $0
910
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
911
	pushl $do_invalid_op
912
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
913
	jmp error_code
914
	CFI_ENDPROC
915
END(invalid_op)
L
Linus Torvalds 已提交
916 917

ENTRY(coprocessor_segment_overrun)
918
	RING0_INT_FRAME
L
Linus Torvalds 已提交
919
	pushl $0
920
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
921
	pushl $do_coprocessor_segment_overrun
922
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
923
	jmp error_code
924
	CFI_ENDPROC
925
END(coprocessor_segment_overrun)
L
Linus Torvalds 已提交
926 927

ENTRY(invalid_TSS)
928
	RING0_EC_FRAME
L
Linus Torvalds 已提交
929
	pushl $do_invalid_TSS
930
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
931
	jmp error_code
932
	CFI_ENDPROC
933
END(invalid_TSS)
L
Linus Torvalds 已提交
934 935

ENTRY(segment_not_present)
936
	RING0_EC_FRAME
L
Linus Torvalds 已提交
937
	pushl $do_segment_not_present
938
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
939
	jmp error_code
940
	CFI_ENDPROC
941
END(segment_not_present)
L
Linus Torvalds 已提交
942 943

ENTRY(stack_segment)
944
	RING0_EC_FRAME
L
Linus Torvalds 已提交
945
	pushl $do_stack_segment
946
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
947
	jmp error_code
948
	CFI_ENDPROC
949
END(stack_segment)
L
Linus Torvalds 已提交
950

951
KPROBE_ENTRY(general_protection)
952
	RING0_EC_FRAME
L
Linus Torvalds 已提交
953
	pushl $do_general_protection
954
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
955
	jmp error_code
956
	CFI_ENDPROC
957
KPROBE_END(general_protection)
L
Linus Torvalds 已提交
958 959

ENTRY(alignment_check)
960
	RING0_EC_FRAME
L
Linus Torvalds 已提交
961
	pushl $do_alignment_check
962
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
963
	jmp error_code
964
	CFI_ENDPROC
965
END(alignment_check)
L
Linus Torvalds 已提交
966

967 968 969 970 971
ENTRY(divide_error)
	RING0_INT_FRAME
	pushl $0			# no error code
	CFI_ADJUST_CFA_OFFSET 4
	pushl $do_divide_error
972
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
973
	jmp error_code
974
	CFI_ENDPROC
975
END(divide_error)
L
Linus Torvalds 已提交
976 977 978

#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
979
	RING0_INT_FRAME
L
Linus Torvalds 已提交
980
	pushl $0
981
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
982
	pushl machine_check_vector
983
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
984
	jmp error_code
985
	CFI_ENDPROC
986
END(machine_check)
L
Linus Torvalds 已提交
987 988 989
#endif

ENTRY(spurious_interrupt_bug)
990
	RING0_INT_FRAME
L
Linus Torvalds 已提交
991
	pushl $0
992
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
993
	pushl $do_spurious_interrupt_bug
994
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
995
	jmp error_code
996
	CFI_ENDPROC
997
END(spurious_interrupt_bug)
L
Linus Torvalds 已提交
998

999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
ENTRY(kernel_thread_helper)
	pushl $0		# fake return address for unwinder
	CFI_STARTPROC
	movl %edx,%eax
	push %edx
	CFI_ADJUST_CFA_OFFSET 4
	call *%ebx
	push %eax
	CFI_ADJUST_CFA_OFFSET 4
	call do_exit
	CFI_ENDPROC
ENDPROC(kernel_thread_helper)

1012
#ifdef CONFIG_XEN
1013 1014 1015 1016 1017 1018
/* Xen doesn't set %esp to be precisely what the normal sysenter
   entrypoint expects, so fix it up before using the normal path. */
ENTRY(xen_sysenter_target)
	RING0_INT_FRAME
	addl $5*4, %esp		/* remove xen-provided frame */
	jmp sysenter_past_esp
G
Glauber Costa 已提交
1019
	CFI_ENDPROC
1020

1021 1022 1023 1024 1025 1026
ENTRY(xen_hypervisor_callback)
	CFI_STARTPROC
	pushl $0
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	TRACE_IRQS_OFF
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038

	/* Check to see if we got the event in the critical
	   region in xen_iret_direct, after we've reenabled
	   events and checked for pending events.  This simulates
	   iret instruction's behaviour where it delivers a
	   pending interrupt when enabling interrupts. */
	movl PT_EIP(%esp),%eax
	cmpl $xen_iret_start_crit,%eax
	jb   1f
	cmpl $xen_iret_end_crit,%eax
	jae  1f

J
Jeremy Fitzhardinge 已提交
1039
	jmp  xen_iret_crit_fixup
1040 1041

ENTRY(xen_do_upcall)
1042
1:	mov %esp, %eax
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	call xen_evtchn_do_upcall
	jmp  ret_from_intr
	CFI_ENDPROC
ENDPROC(xen_hypervisor_callback)

# Hypervisor uses this for application faults while it executes.
# We get here for two reasons:
#  1. Fault while reloading DS, ES, FS or GS
#  2. Fault while executing IRET
# Category 1 we fix up by reattempting the load, and zeroing the segment
# register if the load fails.
# Category 2 we fix up by jumping to do_iret_error. We cannot use the
# normal Linux return path in this case because if we use the IRET hypercall
# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
# We distinguish between categories by maintaining a status value in EAX.
ENTRY(xen_failsafe_callback)
	CFI_STARTPROC
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	movl $1,%eax
1:	mov 4(%esp),%ds
2:	mov 8(%esp),%es
3:	mov 12(%esp),%fs
4:	mov 16(%esp),%gs
	testl %eax,%eax
	popl %eax
	CFI_ADJUST_CFA_OFFSET -4
	lea 16(%esp),%esp
	CFI_ADJUST_CFA_OFFSET -16
	jz 5f
	addl $16,%esp
	jmp iret_exc		# EAX != 0 => Category 2 (Bad IRET)
5:	pushl $0		# EAX == 0 => Category 1 (Bad segment)
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	jmp ret_from_exception
	CFI_ENDPROC

.section .fixup,"ax"
6:	xorl %eax,%eax
	movl %eax,4(%esp)
	jmp 1b
7:	xorl %eax,%eax
	movl %eax,8(%esp)
	jmp 2b
8:	xorl %eax,%eax
	movl %eax,12(%esp)
	jmp 3b
9:	xorl %eax,%eax
	movl %eax,16(%esp)
	jmp 4b
.previous
.section __ex_table,"a"
	.align 4
	.long 1b,6b
	.long 2b,7b
	.long 3b,8b
	.long 4b,9b
.previous
ENDPROC(xen_failsafe_callback)

#endif	/* CONFIG_XEN */

1106
#ifdef CONFIG_FTRACE
1107 1108 1109 1110 1111 1112 1113
#ifdef CONFIG_DYNAMIC_FTRACE

ENTRY(mcount)
	pushl %eax
	pushl %ecx
	pushl %edx
	movl 0xc(%esp), %eax
1114
	subl $MCOUNT_INSN_SIZE, %eax
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132

.globl mcount_call
mcount_call:
	call ftrace_stub

	popl %edx
	popl %ecx
	popl %eax

	ret
END(mcount)

ENTRY(ftrace_caller)
	pushl %eax
	pushl %ecx
	pushl %edx
	movl 0xc(%esp), %eax
	movl 0x4(%ebp), %edx
1133
	subl $MCOUNT_INSN_SIZE, %eax
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149

.globl ftrace_call
ftrace_call:
	call ftrace_stub

	popl %edx
	popl %ecx
	popl %eax

.globl ftrace_stub
ftrace_stub:
	ret
END(ftrace_caller)

#else /* ! CONFIG_DYNAMIC_FTRACE */

1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
ENTRY(mcount)
	cmpl $ftrace_stub, ftrace_trace_function
	jnz trace
.globl ftrace_stub
ftrace_stub:
	ret

	/* taken from glibc */
trace:
	pushl %eax
	pushl %ecx
	pushl %edx
	movl 0xc(%esp), %eax
	movl 0x4(%ebp), %edx
1164
	subl $MCOUNT_INSN_SIZE, %eax
1165

1166
	call *ftrace_trace_function
1167 1168 1169 1170 1171 1172 1173

	popl %edx
	popl %ecx
	popl %eax

	jmp ftrace_stub
END(mcount)
1174 1175
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FTRACE */
1176

1177
.section .rodata,"a"
1178
#include "syscall_table_32.S"
L
Linus Torvalds 已提交
1179 1180

syscall_table_size=(.-sys_call_table)