entry_32.S 26.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 * This also contains the timer-interrupt handler, as well as all interrupts
 * and faults that can result in a task-switch.
 *
 * NOTE: This code handles signal-recognition, which happens every time
 * after a timer-interrupt and after each system call.
 *
 * I changed all the .align's to 4 (16 byte alignment), as that's faster
 * on a 486.
 *
17
 * Stack layout in 'syscall_exit':
L
Linus Torvalds 已提交
18 19 20 21 22 23 24 25 26 27 28 29 30 31
 * 	ptrace needs to have all regs on the stack.
 *	if the order here is changed, it needs to be
 *	updated in fork.c:copy_process, signal.c:do_signal,
 *	ptrace.c and ptrace.h
 *
 *	 0(%esp) - %ebx
 *	 4(%esp) - %ecx
 *	 8(%esp) - %edx
 *       C(%esp) - %esi
 *	10(%esp) - %edi
 *	14(%esp) - %ebp
 *	18(%esp) - %eax
 *	1C(%esp) - %ds
 *	20(%esp) - %es
32
 *	24(%esp) - %fs
33 34 35 36 37 38
 *	28(%esp) - orig_eax
 *	2C(%esp) - %eip
 *	30(%esp) - %cs
 *	34(%esp) - %eflags
 *	38(%esp) - %oldesp
 *	3C(%esp) - %oldss
L
Linus Torvalds 已提交
39 40 41 42 43 44
 *
 * "current" is in register %ebx during any slow entries.
 */

#include <linux/linkage.h>
#include <asm/thread_info.h>
45
#include <asm/irqflags.h>
L
Linus Torvalds 已提交
46 47 48 49 50
#include <asm/errno.h>
#include <asm/segment.h>
#include <asm/smp.h>
#include <asm/page.h>
#include <asm/desc.h>
S
Stas Sergeev 已提交
51
#include <asm/percpu.h>
52
#include <asm/dwarf2.h>
53
#include <asm/processor-flags.h>
L
Linus Torvalds 已提交
54 55
#include "irq_vectors.h"

56 57 58 59 60
/*
 * We use macros for low-level operations which need to be overridden
 * for paravirtualization.  The following will never clobber any registers:
 *   INTERRUPT_RETURN (aka. "iret")
 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61
 *   ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
62 63 64 65 66 67 68
 *
 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
 * Allowing a register to be clobbered can shrink the paravirt replacement
 * enough to patch inline, increasing performance.
 */

L
Linus Torvalds 已提交
69 70 71
#define nr_syscalls ((syscall_table_size)/4)

#ifdef CONFIG_PREEMPT
72
#define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
L
Linus Torvalds 已提交
73
#else
74
#define preempt_stop(clobbers)
L
Linus Torvalds 已提交
75 76 77
#define resume_kernel		restore_nocheck
#endif

78 79
.macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS
80
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
81 82 83 84 85 86
	jz 1f
	TRACE_IRQS_ON
1:
#endif
.endm

87 88 89 90 91 92
#ifdef CONFIG_VM86
#define resume_userspace_sig	check_userspace
#else
#define resume_userspace_sig	resume_userspace
#endif

L
Linus Torvalds 已提交
93 94
#define SAVE_ALL \
	cld; \
95
	pushl %fs; \
96
	CFI_ADJUST_CFA_OFFSET 4;\
97
	/*CFI_REL_OFFSET fs, 0;*/\
L
Linus Torvalds 已提交
98
	pushl %es; \
99 100
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET es, 0;*/\
L
Linus Torvalds 已提交
101
	pushl %ds; \
102 103
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET ds, 0;*/\
L
Linus Torvalds 已提交
104
	pushl %eax; \
105 106
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET eax, 0;\
L
Linus Torvalds 已提交
107
	pushl %ebp; \
108 109
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebp, 0;\
L
Linus Torvalds 已提交
110
	pushl %edi; \
111 112
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edi, 0;\
L
Linus Torvalds 已提交
113
	pushl %esi; \
114 115
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET esi, 0;\
L
Linus Torvalds 已提交
116
	pushl %edx; \
117 118
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edx, 0;\
L
Linus Torvalds 已提交
119
	pushl %ecx; \
120 121
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ecx, 0;\
L
Linus Torvalds 已提交
122
	pushl %ebx; \
123 124
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebx, 0;\
L
Linus Torvalds 已提交
125 126
	movl $(__USER_DS), %edx; \
	movl %edx, %ds; \
127
	movl %edx, %es; \
128
	movl $(__KERNEL_PERCPU), %edx; \
129
	movl %edx, %fs
L
Linus Torvalds 已提交
130 131 132

#define RESTORE_INT_REGS \
	popl %ebx;	\
133 134
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebx;\
L
Linus Torvalds 已提交
135
	popl %ecx;	\
136 137
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ecx;\
L
Linus Torvalds 已提交
138
	popl %edx;	\
139 140
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edx;\
L
Linus Torvalds 已提交
141
	popl %esi;	\
142 143
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE esi;\
L
Linus Torvalds 已提交
144
	popl %edi;	\
145 146
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edi;\
L
Linus Torvalds 已提交
147
	popl %ebp;	\
148 149 150 151 152
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebp;\
	popl %eax;	\
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE eax
L
Linus Torvalds 已提交
153 154 155 156

#define RESTORE_REGS	\
	RESTORE_INT_REGS; \
1:	popl %ds;	\
157 158
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE ds;*/\
L
Linus Torvalds 已提交
159
2:	popl %es;	\
160 161
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE es;*/\
162
3:	popl %fs;	\
163
	CFI_ADJUST_CFA_OFFSET -4;\
164
	/*CFI_RESTORE fs;*/\
165
.pushsection .fixup,"ax";	\
L
Linus Torvalds 已提交
166
4:	movl $0,(%esp);	\
167 168
	jmp 1b;		\
5:	movl $0,(%esp);	\
L
Linus Torvalds 已提交
169
	jmp 2b;		\
170 171
6:	movl $0,(%esp);	\
	jmp 3b;		\
L
Linus Torvalds 已提交
172 173
.section __ex_table,"a";\
	.align 4;	\
174 175 176 177
	.long 1b,4b;	\
	.long 2b,5b;	\
	.long 3b,6b;	\
.popsection
L
Linus Torvalds 已提交
178

179 180
#define RING0_INT_FRAME \
	CFI_STARTPROC simple;\
181
	CFI_SIGNAL_FRAME;\
182 183 184 185 186 187
	CFI_DEF_CFA esp, 3*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_EC_FRAME \
	CFI_STARTPROC simple;\
188
	CFI_SIGNAL_FRAME;\
189 190 191 192 193 194
	CFI_DEF_CFA esp, 4*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_PTREGS_FRAME \
	CFI_STARTPROC simple;\
195
	CFI_SIGNAL_FRAME;\
196 197 198 199 200 201 202 203 204 205 206 207
	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
	CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
	CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
	CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
	CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
	CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
	CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
	CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
L
Linus Torvalds 已提交
208 209

ENTRY(ret_from_fork)
210
	CFI_STARTPROC
L
Linus Torvalds 已提交
211
	pushl %eax
212
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
213 214 215
	call schedule_tail
	GET_THREAD_INFO(%ebp)
	popl %eax
216
	CFI_ADJUST_CFA_OFFSET -4
217 218 219 220
	pushl $0x0202			# Reset kernel eflags
	CFI_ADJUST_CFA_OFFSET 4
	popfl
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
221
	jmp syscall_exit
222
	CFI_ENDPROC
223
END(ret_from_fork)
L
Linus Torvalds 已提交
224 225 226 227 228 229 230 231 232 233

/*
 * Return to user mode is not as complex as all this looks,
 * but we want the default path for a system call return to
 * go as quickly as possible which is why some of this is
 * less clear than it otherwise should be.
 */

	# userspace resumption stub bypassing syscall exit tracing
	ALIGN
234
	RING0_PTREGS_FRAME
L
Linus Torvalds 已提交
235
ret_from_exception:
236
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
237 238
ret_from_intr:
	GET_THREAD_INFO(%ebp)
239
check_userspace:
240 241
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
	movb PT_CS(%esp), %al
242
	andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
243 244
	cmpl $USER_RPL, %eax
	jb resume_kernel		# not returning to v8086 or userspace
245

L
Linus Torvalds 已提交
246
ENTRY(resume_userspace)
247
	LOCKDEP_SYS_EXIT
248
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
249 250 251 252 253 254 255
					# setting need_resched or sigpending
					# between sampling and the iret
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
					# int/exception return?
	jne work_pending
	jmp restore_all
256
END(ret_from_exception)
L
Linus Torvalds 已提交
257 258 259

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
260
	DISABLE_INTERRUPTS(CLBR_ANY)
L
Linus Torvalds 已提交
261 262 263 264 265 266
	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
	jnz restore_nocheck
need_resched:
	movl TI_flags(%ebp), %ecx	# need_resched set ?
	testb $_TIF_NEED_RESCHED, %cl
	jz restore_all
267
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
L
Linus Torvalds 已提交
268 269 270
	jz restore_all
	call preempt_schedule_irq
	jmp need_resched
271
END(resume_kernel)
L
Linus Torvalds 已提交
272
#endif
273
	CFI_ENDPROC
L
Linus Torvalds 已提交
274 275 276 277 278

/* SYSENTER_RETURN points to after the "sysenter" instruction in
   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */

	# sysenter call handler stub
R
Roland McGrath 已提交
279
ENTRY(ia32_sysenter_target)
280
	CFI_STARTPROC simple
281
	CFI_SIGNAL_FRAME
282 283
	CFI_DEF_CFA esp, 0
	CFI_REGISTER esp, ebp
284
	movl TSS_sysenter_sp0(%esp),%esp
L
Linus Torvalds 已提交
285
sysenter_past_esp:
286
	/*
287 288 289
	 * Interrupts are disabled here, but we can't trace it until
	 * enough kernel state to call TRACE_IRQS_OFF can be called - but
	 * we immediately enable interrupts at that point anyway.
290
	 */
L
Linus Torvalds 已提交
291
	pushl $(__USER_DS)
292 293
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ss, 0*/
L
Linus Torvalds 已提交
294
	pushl %ebp
295 296
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esp, 0
L
Linus Torvalds 已提交
297
	pushfl
298
	orl $X86_EFLAGS_IF, (%esp)
299
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
300
	pushl $(__USER_CS)
301 302
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET cs, 0*/
303 304 305 306 307 308
	/*
	 * Push current_thread_info()->sysenter_return to the stack.
	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
	 */
	pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
309 310
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
311

312 313 314 315 316
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	ENABLE_INTERRUPTS(CLBR_NONE)

L
Linus Torvalds 已提交
317 318 319 320 321 322 323
/*
 * Load the potential sixth argument from user stack.
 * Careful about security.
 */
	cmpl $__PAGE_OFFSET-3,%ebp
	jae syscall_fault
1:	movl (%ebp),%ebp
324
	movl %ebp,PT_EBP(%esp)
L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332
.section __ex_table,"a"
	.align 4
	.long 1b,syscall_fault
.previous

	GET_THREAD_INFO(%ebp)

	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
333
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
334 335 336 337
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
	call *sys_call_table(,%eax,4)
338
	movl %eax,PT_EAX(%esp)
339
	LOCKDEP_SYS_EXIT
340
	DISABLE_INTERRUPTS(CLBR_ANY)
341
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
342 343 344 345
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx
	jne syscall_exit_work
/* if something modifies registers it must also disable sysexit */
346 347
	movl PT_EIP(%esp), %edx
	movl PT_OLDESP(%esp), %ecx
L
Linus Torvalds 已提交
348
	xorl %ebp,%ebp
349
	TRACE_IRQS_ON
350
1:	mov  PT_FS(%esp), %fs
351
	ENABLE_INTERRUPTS_SYSCALL_RET
352
	CFI_ENDPROC
353
.pushsection .fixup,"ax"
354
2:	movl $0,PT_FS(%esp)
355 356 357 358 359
	jmp 1b
.section __ex_table,"a"
	.align 4
	.long 1b,2b
.popsection
R
Roland McGrath 已提交
360
ENDPROC(ia32_sysenter_target)
L
Linus Torvalds 已提交
361 362 363

	# system call handler stub
ENTRY(system_call)
364
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
365
	pushl %eax			# save orig_eax
366
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
367 368
	SAVE_ALL
	GET_THREAD_INFO(%ebp)
369
					# system call tracing in operation / emulation
L
Linus Torvalds 已提交
370
	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
371
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
372 373 374 375 376
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
syscall_call:
	call *sys_call_table(,%eax,4)
377
	movl %eax,PT_EAX(%esp)		# store the return value
L
Linus Torvalds 已提交
378
syscall_exit:
379
	LOCKDEP_SYS_EXIT
380
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
381 382
					# setting need_resched or sigpending
					# between sampling and the iret
383
	TRACE_IRQS_OFF
384
	testl $X86_EFLAGS_TF,PT_EFLAGS(%esp)	# If tracing set singlestep flag on exit
385 386 387
	jz no_singlestep
	orl $_TIF_SINGLESTEP,TI_flags(%ebp)
no_singlestep:
L
Linus Torvalds 已提交
388 389 390 391 392
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx	# current->work
	jne syscall_exit_work

restore_all:
393 394
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
395 396
	# are returning to the kernel.
	# See comments in process.c:copy_thread() for details.
397 398
	movb PT_OLDSS(%esp), %ah
	movb PT_CS(%esp), %al
399
	andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
400
	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
401
	CFI_REMEMBER_STATE
L
Linus Torvalds 已提交
402 403
	je ldt_ss			# returning to user-space with LDT SS
restore_nocheck:
404 405
	TRACE_IRQS_IRET
restore_nocheck_notrace:
L
Linus Torvalds 已提交
406
	RESTORE_REGS
407
	addl $4, %esp			# skip orig_eax/error_code
408
	CFI_ADJUST_CFA_OFFSET -4
A
Adrian Bunk 已提交
409
irq_return:
I
Ingo Molnar 已提交
410
	INTERRUPT_RETURN
L
Linus Torvalds 已提交
411
.section .fixup,"ax"
412
ENTRY(iret_exc)
413 414 415
	pushl $0			# no error code
	pushl $do_iret_error
	jmp error_code
L
Linus Torvalds 已提交
416 417 418
.previous
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
419
	.long irq_return,iret_exc
L
Linus Torvalds 已提交
420 421
.previous

422
	CFI_RESTORE_STATE
L
Linus Torvalds 已提交
423
ldt_ss:
424
	larl PT_OLDSS(%esp), %eax
L
Linus Torvalds 已提交
425 426 427
	jnz restore_nocheck
	testl $0x00400000, %eax		# returning to 32bit stack?
	jnz restore_nocheck		# allright, normal return
428 429 430 431 432 433 434 435 436 437

#ifdef CONFIG_PARAVIRT
	/*
	 * The kernel can't run on a non-flat stack if paravirt mode
	 * is active.  Rather than try to fixup the high bits of
	 * ESP, bypass this code entirely.  This may break DOSemu
	 * and/or Wine support in a paravirt VM, although the option
	 * is still available to implement the setting of the high
	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
	 */
438
	cmpl $0, pv_info+PARAVIRT_enabled
439 440 441
	jne restore_nocheck
#endif

L
Linus Torvalds 已提交
442 443 444 445 446 447
	/* If returning to userspace with 16bit stack,
	 * try to fix the higher word of ESP, as the CPU
	 * won't restore it.
	 * This is an "official" bug of all the x86-compatible
	 * CPUs, which we can try to work around to make
	 * dosemu and wine happy. */
448
	movl PT_OLDESP(%esp), %eax
S
Stas Sergeev 已提交
449 450 451 452 453 454
	movl %esp, %edx
	call patch_espfix_desc
	pushl $__ESPFIX_SS
	CFI_ADJUST_CFA_OFFSET 4
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
455
	DISABLE_INTERRUPTS(CLBR_EAX)
456
	TRACE_IRQS_OFF
S
Stas Sergeev 已提交
457 458 459
	lss (%esp), %esp
	CFI_ADJUST_CFA_OFFSET -8
	jmp restore_nocheck
460
	CFI_ENDPROC
461
ENDPROC(system_call)
L
Linus Torvalds 已提交
462 463 464

	# perform work that needs to be done immediately before resumption
	ALIGN
465
	RING0_PTREGS_FRAME		# can't unwind into user space anyway
L
Linus Torvalds 已提交
466 467 468 469 470
work_pending:
	testb $_TIF_NEED_RESCHED, %cl
	jz work_notifysig
work_resched:
	call schedule
471
	LOCKDEP_SYS_EXIT
472
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
473 474
					# setting need_resched or sigpending
					# between sampling and the iret
475
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
476 477 478 479 480 481 482 483 484
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
					# than syscall tracing?
	jz restore_all
	testb $_TIF_NEED_RESCHED, %cl
	jnz work_resched

work_notifysig:				# deal with pending signals and
					# notify-resume requests
485
#ifdef CONFIG_VM86
486
	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
L
Linus Torvalds 已提交
487 488 489 490 491
	movl %esp, %eax
	jne work_notifysig_v86		# returning to kernel-space or
					# vm86-space
	xorl %edx, %edx
	call do_notify_resume
492
	jmp resume_userspace_sig
L
Linus Torvalds 已提交
493 494 495 496

	ALIGN
work_notifysig_v86:
	pushl %ecx			# save ti_flags for do_notify_resume
497
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
498 499
	call save_v86_state		# %eax contains pt_regs pointer
	popl %ecx
500
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
501
	movl %eax, %esp
502 503 504
#else
	movl %esp, %eax
#endif
L
Linus Torvalds 已提交
505 506
	xorl %edx, %edx
	call do_notify_resume
507
	jmp resume_userspace_sig
508
END(work_pending)
L
Linus Torvalds 已提交
509 510 511 512

	# perform syscall exit tracing
	ALIGN
syscall_trace_entry:
513
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
514 515 516
	movl %esp, %eax
	xorl %edx,%edx
	call do_syscall_trace
517
	cmpl $0, %eax
518
	jne resume_userspace		# ret != 0 -> running under PTRACE_SYSEMU,
519
					# so must skip actual syscall
520
	movl PT_ORIG_EAX(%esp), %eax
L
Linus Torvalds 已提交
521 522 523
	cmpl $(nr_syscalls), %eax
	jnae syscall_call
	jmp syscall_exit
524
END(syscall_trace_entry)
L
Linus Torvalds 已提交
525 526 527 528 529 530

	# perform syscall exit tracing
	ALIGN
syscall_exit_work:
	testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
	jz work_pending
531
	TRACE_IRQS_ON
532
	ENABLE_INTERRUPTS(CLBR_ANY)	# could let do_syscall_trace() call
L
Linus Torvalds 已提交
533 534 535 536 537
					# schedule() instead
	movl %esp, %eax
	movl $1, %edx
	call do_syscall_trace
	jmp resume_userspace
538
END(syscall_exit_work)
539
	CFI_ENDPROC
L
Linus Torvalds 已提交
540

541
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
542 543
syscall_fault:
	GET_THREAD_INFO(%ebp)
544
	movl $-EFAULT,PT_EAX(%esp)
L
Linus Torvalds 已提交
545
	jmp resume_userspace
546
END(syscall_fault)
L
Linus Torvalds 已提交
547 548

syscall_badsys:
549
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
550
	jmp resume_userspace
551
END(syscall_badsys)
552
	CFI_ENDPROC
L
Linus Torvalds 已提交
553 554

#define FIXUP_ESPFIX_STACK \
S
Stas Sergeev 已提交
555
	/* since we are on a wrong stack, we cant make it a C code :( */ \
556
	PER_CPU(gdt_page, %ebx); \
S
Stas Sergeev 已提交
557 558 559 560
	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
	addl %esp, %eax; \
	pushl $__KERNEL_DS; \
	CFI_ADJUST_CFA_OFFSET 4; \
L
Linus Torvalds 已提交
561
	pushl %eax; \
562
	CFI_ADJUST_CFA_OFFSET 4; \
S
Stas Sergeev 已提交
563 564 565
	lss (%esp), %esp; \
	CFI_ADJUST_CFA_OFFSET -8;
#define UNWIND_ESPFIX_STACK \
L
Linus Torvalds 已提交
566
	movl %ss, %eax; \
S
Stas Sergeev 已提交
567
	/* see if on espfix stack */ \
L
Linus Torvalds 已提交
568
	cmpw $__ESPFIX_SS, %ax; \
S
Stas Sergeev 已提交
569 570
	jne 27f; \
	movl $__KERNEL_DS, %eax; \
571 572
	movl %eax, %ds; \
	movl %eax, %es; \
S
Stas Sergeev 已提交
573
	/* switch to normal stack */ \
574
	FIXUP_ESPFIX_STACK; \
S
Stas Sergeev 已提交
575
27:;
L
Linus Torvalds 已提交
576 577 578 579 580

/*
 * Build the entry stubs and pointer table with
 * some assembler magic.
 */
J
Jan Beulich 已提交
581
.section .rodata,"a"
L
Linus Torvalds 已提交
582 583 584 585
ENTRY(interrupt)
.text

ENTRY(irq_entries_start)
586
	RING0_INT_FRAME
587
vector=0
L
Linus Torvalds 已提交
588 589
.rept NR_IRQS
	ALIGN
590 591 592
 .if vector
	CFI_ADJUST_CFA_OFFSET -4
 .endif
593
1:	pushl $~(vector)
594
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
595
	jmp common_interrupt
596
 .previous
L
Linus Torvalds 已提交
597
	.long 1b
598
 .text
L
Linus Torvalds 已提交
599 600
vector=vector+1
.endr
601 602 603 604 605
END(irq_entries_start)

.previous
END(interrupt)
.previous
L
Linus Torvalds 已提交
606

607 608 609 610
/*
 * the CPU automatically disables interrupts when executing an IRQ vector,
 * so IRQ-flags tracing has to follow that:
 */
L
Linus Torvalds 已提交
611 612 613
	ALIGN
common_interrupt:
	SAVE_ALL
614
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
615 616 617
	movl %esp,%eax
	call do_IRQ
	jmp ret_from_intr
618
ENDPROC(common_interrupt)
619
	CFI_ENDPROC
L
Linus Torvalds 已提交
620 621 622

#define BUILD_INTERRUPT(name, nr)	\
ENTRY(name)				\
623
	RING0_INT_FRAME;		\
624
	pushl $~(nr);			\
625 626
	CFI_ADJUST_CFA_OFFSET 4;	\
	SAVE_ALL;			\
627
	TRACE_IRQS_OFF			\
L
Linus Torvalds 已提交
628
	movl %esp,%eax;			\
629
	call smp_##name;		\
630
	jmp ret_from_intr;		\
631 632
	CFI_ENDPROC;			\
ENDPROC(name)
L
Linus Torvalds 已提交
633 634 635 636

/* The include is where all of the SMP etc. interrupts come from */
#include "entry_arch.h"

637 638 639
KPROBE_ENTRY(page_fault)
	RING0_EC_FRAME
	pushl $do_page_fault
640
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
641 642
	ALIGN
error_code:
643
	/* the function address is in %fs's slot on the stack */
644 645 646
	pushl %es
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET es, 0*/
L
Linus Torvalds 已提交
647
	pushl %ds
648 649
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ds, 0*/
L
Linus Torvalds 已提交
650
	pushl %eax
651 652
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eax, 0
L
Linus Torvalds 已提交
653
	pushl %ebp
654 655
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebp, 0
L
Linus Torvalds 已提交
656
	pushl %edi
657 658
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edi, 0
L
Linus Torvalds 已提交
659
	pushl %esi
660 661
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esi, 0
L
Linus Torvalds 已提交
662
	pushl %edx
663 664
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edx, 0
L
Linus Torvalds 已提交
665
	pushl %ecx
666 667
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ecx, 0
L
Linus Torvalds 已提交
668
	pushl %ebx
669 670
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebx, 0
L
Linus Torvalds 已提交
671
	cld
672
	pushl %fs
673
	CFI_ADJUST_CFA_OFFSET 4
674
	/*CFI_REL_OFFSET fs, 0*/
675
	movl $(__KERNEL_PERCPU), %ecx
676
	movl %ecx, %fs
L
Linus Torvalds 已提交
677 678
	UNWIND_ESPFIX_STACK
	popl %ecx
679 680
	CFI_ADJUST_CFA_OFFSET -4
	/*CFI_REGISTER es, ecx*/
681
	movl PT_FS(%esp), %edi		# get the function address
682
	movl PT_ORIG_EAX(%esp), %edx	# get the error code
683
	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
684 685
	mov  %ecx, PT_FS(%esp)
	/*CFI_REL_OFFSET fs, ES*/
L
Linus Torvalds 已提交
686 687 688 689 690 691
	movl $(__USER_DS), %ecx
	movl %ecx, %ds
	movl %ecx, %es
	movl %esp,%eax			# pt_regs pointer
	call *%edi
	jmp ret_from_exception
692
	CFI_ENDPROC
693
KPROBE_END(page_fault)
L
Linus Torvalds 已提交
694 695

ENTRY(coprocessor_error)
696
	RING0_INT_FRAME
L
Linus Torvalds 已提交
697
	pushl $0
698
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
699
	pushl $do_coprocessor_error
700
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
701
	jmp error_code
702
	CFI_ENDPROC
703
END(coprocessor_error)
L
Linus Torvalds 已提交
704 705

ENTRY(simd_coprocessor_error)
706
	RING0_INT_FRAME
L
Linus Torvalds 已提交
707
	pushl $0
708
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
709
	pushl $do_simd_coprocessor_error
710
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
711
	jmp error_code
712
	CFI_ENDPROC
713
END(simd_coprocessor_error)
L
Linus Torvalds 已提交
714 715

ENTRY(device_not_available)
716
	RING0_INT_FRAME
L
Linus Torvalds 已提交
717
	pushl $-1			# mark this as an int
718
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
719
	SAVE_ALL
720
	GET_CR0_INTO_EAX
L
Linus Torvalds 已提交
721 722
	testl $0x4, %eax		# EM (math emulation bit)
	jne device_not_available_emulate
723
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
724 725 726 727
	call math_state_restore
	jmp ret_from_exception
device_not_available_emulate:
	pushl $0			# temporary storage for ORIG_EIP
728
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
729 730
	call math_emulate
	addl $4, %esp
731
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
732
	jmp ret_from_exception
733
	CFI_ENDPROC
734
END(device_not_available)
L
Linus Torvalds 已提交
735 736 737 738 739 740

/*
 * Debug traps and NMI can happen at the one SYSENTER instruction
 * that sets up the real kernel stack. Check here, since we can't
 * allow the wrong stack to be used.
 *
741
 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
L
Linus Torvalds 已提交
742 743 744 745 746 747 748 749 750 751 752
 * already pushed 3 words if it hits on the sysenter instruction:
 * eflags, cs and eip.
 *
 * We just load the right stack, and push the three (known) values
 * by hand onto the new stack - while updating the return eip past
 * the instruction that would have done it for sysenter.
 */
#define FIX_STACK(offset, ok, label)		\
	cmpw $__KERNEL_CS,4(%esp);		\
	jne ok;					\
label:						\
753
	movl TSS_sysenter_sp0+offset(%esp),%esp;	\
754 755
	CFI_DEF_CFA esp, 0;			\
	CFI_UNDEFINED eip;			\
L
Linus Torvalds 已提交
756
	pushfl;					\
757
	CFI_ADJUST_CFA_OFFSET 4;		\
L
Linus Torvalds 已提交
758
	pushl $__KERNEL_CS;			\
759 760 761 762
	CFI_ADJUST_CFA_OFFSET 4;		\
	pushl $sysenter_past_esp;		\
	CFI_ADJUST_CFA_OFFSET 4;		\
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
763

764
KPROBE_ENTRY(debug)
765
	RING0_INT_FRAME
R
Roland McGrath 已提交
766
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
767 768 769 770
	jne debug_stack_correct
	FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
debug_stack_correct:
	pushl $-1			# mark this as an int
771
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
772 773 774 775 776
	SAVE_ALL
	xorl %edx,%edx			# error code 0
	movl %esp,%eax			# pt_regs pointer
	call do_debug
	jmp ret_from_exception
777
	CFI_ENDPROC
778 779
KPROBE_END(debug)

L
Linus Torvalds 已提交
780 781 782 783 784 785 786 787
/*
 * NMI is doubly nasty. It can happen _while_ we're handling
 * a debug fault, and the debug fault hasn't yet been able to
 * clear up the stack. So we first check whether we got  an
 * NMI on the sysenter entry path, but after that we need to
 * check whether we got an NMI on the debug path where the debug
 * fault happened on the sysenter path.
 */
788
KPROBE_ENTRY(nmi)
789
	RING0_INT_FRAME
L
Linus Torvalds 已提交
790
	pushl %eax
791
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
792 793 794
	movl %ss, %eax
	cmpw $__ESPFIX_SS, %ax
	popl %eax
795
	CFI_ADJUST_CFA_OFFSET -4
S
Stas Sergeev 已提交
796
	je nmi_espfix_stack
R
Roland McGrath 已提交
797
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
798 799
	je nmi_stack_fixup
	pushl %eax
800
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
801 802 803 804 805 806 807
	movl %esp,%eax
	/* Do not access memory above the end of our stack page,
	 * it might not exist.
	 */
	andl $(THREAD_SIZE-1),%eax
	cmpl $(THREAD_SIZE-20),%eax
	popl %eax
808
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
809
	jae nmi_stack_correct
R
Roland McGrath 已提交
810
	cmpl $ia32_sysenter_target,12(%esp)
L
Linus Torvalds 已提交
811 812
	je nmi_debug_stack_check
nmi_stack_correct:
813
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
814
	pushl %eax
815
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
816 817 818 819
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_nmi
820
	jmp restore_nocheck_notrace
821
	CFI_ENDPROC
L
Linus Torvalds 已提交
822 823

nmi_stack_fixup:
824
	RING0_INT_FRAME
L
Linus Torvalds 已提交
825 826
	FIX_STACK(12,nmi_stack_correct, 1)
	jmp nmi_stack_correct
827

L
Linus Torvalds 已提交
828
nmi_debug_stack_check:
829
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
830 831
	cmpw $__KERNEL_CS,16(%esp)
	jne nmi_stack_correct
832 833
	cmpl $debug,(%esp)
	jb nmi_stack_correct
L
Linus Torvalds 已提交
834
	cmpl $debug_esp_fix_insn,(%esp)
835
	ja nmi_stack_correct
L
Linus Torvalds 已提交
836 837 838
	FIX_STACK(24,nmi_stack_correct, 1)
	jmp nmi_stack_correct

S
Stas Sergeev 已提交
839
nmi_espfix_stack:
840 841 842 843
	/* We have a RING0_INT_FRAME here.
	 *
	 * create the pointer to lss back
	 */
L
Linus Torvalds 已提交
844
	pushl %ss
845
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
846
	pushl %esp
847
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
848 849 850 851
	addw $4, (%esp)
	/* copy the iret frame of 12 bytes */
	.rept 3
	pushl 16(%esp)
852
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
853 854
	.endr
	pushl %eax
855
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
856 857 858 859 860
	SAVE_ALL
	FIXUP_ESPFIX_STACK		# %eax == %esp
	xorl %edx,%edx			# zero error code
	call do_nmi
	RESTORE_REGS
S
Stas Sergeev 已提交
861 862
	lss 12+4(%esp), %esp		# back to espfix stack
	CFI_ADJUST_CFA_OFFSET -24
I
Ingo Molnar 已提交
863
	jmp irq_return
864
	CFI_ENDPROC
865
KPROBE_END(nmi)
L
Linus Torvalds 已提交
866

867 868
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
I
Ingo Molnar 已提交
869
	iret
870 871
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
872
	.long native_iret, iret_exc
873
.previous
874
END(native_iret)
875

876
ENTRY(native_irq_enable_syscall_ret)
877 878
	sti
	sysexit
879
END(native_irq_enable_syscall_ret)
880 881
#endif

882
KPROBE_ENTRY(int3)
883
	RING0_INT_FRAME
L
Linus Torvalds 已提交
884
	pushl $-1			# mark this as an int
885
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
886 887 888 889 890
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_int3
	jmp ret_from_exception
891
	CFI_ENDPROC
892
KPROBE_END(int3)
L
Linus Torvalds 已提交
893 894

ENTRY(overflow)
895
	RING0_INT_FRAME
L
Linus Torvalds 已提交
896
	pushl $0
897
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
898
	pushl $do_overflow
899
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
900
	jmp error_code
901
	CFI_ENDPROC
902
END(overflow)
L
Linus Torvalds 已提交
903 904

ENTRY(bounds)
905
	RING0_INT_FRAME
L
Linus Torvalds 已提交
906
	pushl $0
907
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
908
	pushl $do_bounds
909
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
910
	jmp error_code
911
	CFI_ENDPROC
912
END(bounds)
L
Linus Torvalds 已提交
913 914

ENTRY(invalid_op)
915
	RING0_INT_FRAME
L
Linus Torvalds 已提交
916
	pushl $0
917
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
918
	pushl $do_invalid_op
919
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
920
	jmp error_code
921
	CFI_ENDPROC
922
END(invalid_op)
L
Linus Torvalds 已提交
923 924

ENTRY(coprocessor_segment_overrun)
925
	RING0_INT_FRAME
L
Linus Torvalds 已提交
926
	pushl $0
927
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
928
	pushl $do_coprocessor_segment_overrun
929
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
930
	jmp error_code
931
	CFI_ENDPROC
932
END(coprocessor_segment_overrun)
L
Linus Torvalds 已提交
933 934

ENTRY(invalid_TSS)
935
	RING0_EC_FRAME
L
Linus Torvalds 已提交
936
	pushl $do_invalid_TSS
937
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
938
	jmp error_code
939
	CFI_ENDPROC
940
END(invalid_TSS)
L
Linus Torvalds 已提交
941 942

ENTRY(segment_not_present)
943
	RING0_EC_FRAME
L
Linus Torvalds 已提交
944
	pushl $do_segment_not_present
945
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
946
	jmp error_code
947
	CFI_ENDPROC
948
END(segment_not_present)
L
Linus Torvalds 已提交
949 950

ENTRY(stack_segment)
951
	RING0_EC_FRAME
L
Linus Torvalds 已提交
952
	pushl $do_stack_segment
953
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
954
	jmp error_code
955
	CFI_ENDPROC
956
END(stack_segment)
L
Linus Torvalds 已提交
957

958
KPROBE_ENTRY(general_protection)
959
	RING0_EC_FRAME
L
Linus Torvalds 已提交
960
	pushl $do_general_protection
961
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
962
	jmp error_code
963
	CFI_ENDPROC
964
KPROBE_END(general_protection)
L
Linus Torvalds 已提交
965 966

ENTRY(alignment_check)
967
	RING0_EC_FRAME
L
Linus Torvalds 已提交
968
	pushl $do_alignment_check
969
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
970
	jmp error_code
971
	CFI_ENDPROC
972
END(alignment_check)
L
Linus Torvalds 已提交
973

974 975 976 977 978
ENTRY(divide_error)
	RING0_INT_FRAME
	pushl $0			# no error code
	CFI_ADJUST_CFA_OFFSET 4
	pushl $do_divide_error
979
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
980
	jmp error_code
981
	CFI_ENDPROC
982
END(divide_error)
L
Linus Torvalds 已提交
983 984 985

#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
986
	RING0_INT_FRAME
L
Linus Torvalds 已提交
987
	pushl $0
988
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
989
	pushl machine_check_vector
990
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
991
	jmp error_code
992
	CFI_ENDPROC
993
END(machine_check)
L
Linus Torvalds 已提交
994 995 996
#endif

ENTRY(spurious_interrupt_bug)
997
	RING0_INT_FRAME
L
Linus Torvalds 已提交
998
	pushl $0
999
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1000
	pushl $do_spurious_interrupt_bug
1001
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1002
	jmp error_code
1003
	CFI_ENDPROC
1004
END(spurious_interrupt_bug)
L
Linus Torvalds 已提交
1005

1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
ENTRY(kernel_thread_helper)
	pushl $0		# fake return address for unwinder
	CFI_STARTPROC
	movl %edx,%eax
	push %edx
	CFI_ADJUST_CFA_OFFSET 4
	call *%ebx
	push %eax
	CFI_ADJUST_CFA_OFFSET 4
	call do_exit
	CFI_ENDPROC
ENDPROC(kernel_thread_helper)

1019
#ifdef CONFIG_XEN
1020 1021 1022 1023 1024 1025 1026
/* Xen doesn't set %esp to be precisely what the normal sysenter
   entrypoint expects, so fix it up before using the normal path. */
ENTRY(xen_sysenter_target)
	RING0_INT_FRAME
	addl $5*4, %esp		/* remove xen-provided frame */
	jmp sysenter_past_esp

1027 1028 1029 1030 1031 1032
ENTRY(xen_hypervisor_callback)
	CFI_STARTPROC
	pushl $0
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	TRACE_IRQS_OFF
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044

	/* Check to see if we got the event in the critical
	   region in xen_iret_direct, after we've reenabled
	   events and checked for pending events.  This simulates
	   iret instruction's behaviour where it delivers a
	   pending interrupt when enabling interrupts. */
	movl PT_EIP(%esp),%eax
	cmpl $xen_iret_start_crit,%eax
	jb   1f
	cmpl $xen_iret_end_crit,%eax
	jae  1f

J
Jeremy Fitzhardinge 已提交
1045
	jmp  xen_iret_crit_fixup
1046 1047

ENTRY(xen_do_upcall)
1048
1:	mov %esp, %eax
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
	call xen_evtchn_do_upcall
	jmp  ret_from_intr
	CFI_ENDPROC
ENDPROC(xen_hypervisor_callback)

# Hypervisor uses this for application faults while it executes.
# We get here for two reasons:
#  1. Fault while reloading DS, ES, FS or GS
#  2. Fault while executing IRET
# Category 1 we fix up by reattempting the load, and zeroing the segment
# register if the load fails.
# Category 2 we fix up by jumping to do_iret_error. We cannot use the
# normal Linux return path in this case because if we use the IRET hypercall
# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
# We distinguish between categories by maintaining a status value in EAX.
ENTRY(xen_failsafe_callback)
	CFI_STARTPROC
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	movl $1,%eax
1:	mov 4(%esp),%ds
2:	mov 8(%esp),%es
3:	mov 12(%esp),%fs
4:	mov 16(%esp),%gs
	testl %eax,%eax
	popl %eax
	CFI_ADJUST_CFA_OFFSET -4
	lea 16(%esp),%esp
	CFI_ADJUST_CFA_OFFSET -16
	jz 5f
	addl $16,%esp
	jmp iret_exc		# EAX != 0 => Category 2 (Bad IRET)
5:	pushl $0		# EAX == 0 => Category 1 (Bad segment)
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	jmp ret_from_exception
	CFI_ENDPROC

.section .fixup,"ax"
6:	xorl %eax,%eax
	movl %eax,4(%esp)
	jmp 1b
7:	xorl %eax,%eax
	movl %eax,8(%esp)
	jmp 2b
8:	xorl %eax,%eax
	movl %eax,12(%esp)
	jmp 3b
9:	xorl %eax,%eax
	movl %eax,16(%esp)
	jmp 4b
.previous
.section __ex_table,"a"
	.align 4
	.long 1b,6b
	.long 2b,7b
	.long 3b,8b
	.long 4b,9b
.previous
ENDPROC(xen_failsafe_callback)

#endif	/* CONFIG_XEN */

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
#ifdef CONFIG_FTRACE
ENTRY(mcount)
	cmpl $ftrace_stub, ftrace_trace_function
	jnz trace

.globl ftrace_stub
ftrace_stub:
	ret

	/* taken from glibc */
trace:
	pushl %eax
	pushl %ecx
	pushl %edx
	movl 0xc(%esp), %eax
	movl 0x4(%ebp), %edx

	call   *ftrace_trace_function

	popl %edx
	popl %ecx
	popl %eax

	jmp ftrace_stub
END(mcount)
#endif

1139
.section .rodata,"a"
1140
#include "syscall_table_32.S"
L
Linus Torvalds 已提交
1141 1142

syscall_table_size=(.-sys_call_table)