entry_32.S 25.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 * This also contains the timer-interrupt handler, as well as all interrupts
 * and faults that can result in a task-switch.
 *
 * NOTE: This code handles signal-recognition, which happens every time
 * after a timer-interrupt and after each system call.
 *
 * I changed all the .align's to 4 (16 byte alignment), as that's faster
 * on a 486.
 *
17
 * Stack layout in 'syscall_exit':
L
Linus Torvalds 已提交
18 19 20 21 22 23 24 25 26 27 28 29 30 31
 * 	ptrace needs to have all regs on the stack.
 *	if the order here is changed, it needs to be
 *	updated in fork.c:copy_process, signal.c:do_signal,
 *	ptrace.c and ptrace.h
 *
 *	 0(%esp) - %ebx
 *	 4(%esp) - %ecx
 *	 8(%esp) - %edx
 *       C(%esp) - %esi
 *	10(%esp) - %edi
 *	14(%esp) - %ebp
 *	18(%esp) - %eax
 *	1C(%esp) - %ds
 *	20(%esp) - %es
32
 *	24(%esp) - %fs
33 34 35 36 37 38
 *	28(%esp) - orig_eax
 *	2C(%esp) - %eip
 *	30(%esp) - %cs
 *	34(%esp) - %eflags
 *	38(%esp) - %oldesp
 *	3C(%esp) - %oldss
L
Linus Torvalds 已提交
39 40 41 42 43 44
 *
 * "current" is in register %ebx during any slow entries.
 */

#include <linux/linkage.h>
#include <asm/thread_info.h>
45
#include <asm/irqflags.h>
L
Linus Torvalds 已提交
46 47 48 49 50
#include <asm/errno.h>
#include <asm/segment.h>
#include <asm/smp.h>
#include <asm/page.h>
#include <asm/desc.h>
S
Stas Sergeev 已提交
51
#include <asm/percpu.h>
52
#include <asm/dwarf2.h>
53
#include <asm/processor-flags.h>
54
#include <asm/irq_vectors.h>
L
Linus Torvalds 已提交
55

56 57 58 59 60
/*
 * We use macros for low-level operations which need to be overridden
 * for paravirtualization.  The following will never clobber any registers:
 *   INTERRUPT_RETURN (aka. "iret")
 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61
 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
62 63 64 65 66 67 68
 *
 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
 * Allowing a register to be clobbered can shrink the paravirt replacement
 * enough to patch inline, increasing performance.
 */

L
Linus Torvalds 已提交
69 70 71
#define nr_syscalls ((syscall_table_size)/4)

#ifdef CONFIG_PREEMPT
72
#define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
L
Linus Torvalds 已提交
73
#else
74
#define preempt_stop(clobbers)
L
Linus Torvalds 已提交
75 76 77
#define resume_kernel		restore_nocheck
#endif

78 79
.macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS
80
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
81 82 83 84 85 86
	jz 1f
	TRACE_IRQS_ON
1:
#endif
.endm

87 88 89 90 91 92
#ifdef CONFIG_VM86
#define resume_userspace_sig	check_userspace
#else
#define resume_userspace_sig	resume_userspace
#endif

L
Linus Torvalds 已提交
93 94
#define SAVE_ALL \
	cld; \
95
	pushl %fs; \
96
	CFI_ADJUST_CFA_OFFSET 4;\
97
	/*CFI_REL_OFFSET fs, 0;*/\
L
Linus Torvalds 已提交
98
	pushl %es; \
99 100
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET es, 0;*/\
L
Linus Torvalds 已提交
101
	pushl %ds; \
102 103
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET ds, 0;*/\
L
Linus Torvalds 已提交
104
	pushl %eax; \
105 106
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET eax, 0;\
L
Linus Torvalds 已提交
107
	pushl %ebp; \
108 109
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebp, 0;\
L
Linus Torvalds 已提交
110
	pushl %edi; \
111 112
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edi, 0;\
L
Linus Torvalds 已提交
113
	pushl %esi; \
114 115
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET esi, 0;\
L
Linus Torvalds 已提交
116
	pushl %edx; \
117 118
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edx, 0;\
L
Linus Torvalds 已提交
119
	pushl %ecx; \
120 121
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ecx, 0;\
L
Linus Torvalds 已提交
122
	pushl %ebx; \
123 124
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebx, 0;\
L
Linus Torvalds 已提交
125 126
	movl $(__USER_DS), %edx; \
	movl %edx, %ds; \
127
	movl %edx, %es; \
128
	movl $(__KERNEL_PERCPU), %edx; \
129
	movl %edx, %fs
L
Linus Torvalds 已提交
130 131 132

#define RESTORE_INT_REGS \
	popl %ebx;	\
133 134
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebx;\
L
Linus Torvalds 已提交
135
	popl %ecx;	\
136 137
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ecx;\
L
Linus Torvalds 已提交
138
	popl %edx;	\
139 140
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edx;\
L
Linus Torvalds 已提交
141
	popl %esi;	\
142 143
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE esi;\
L
Linus Torvalds 已提交
144
	popl %edi;	\
145 146
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edi;\
L
Linus Torvalds 已提交
147
	popl %ebp;	\
148 149 150 151 152
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebp;\
	popl %eax;	\
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE eax
L
Linus Torvalds 已提交
153 154 155 156

#define RESTORE_REGS	\
	RESTORE_INT_REGS; \
1:	popl %ds;	\
157 158
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE ds;*/\
L
Linus Torvalds 已提交
159
2:	popl %es;	\
160 161
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE es;*/\
162
3:	popl %fs;	\
163
	CFI_ADJUST_CFA_OFFSET -4;\
164
	/*CFI_RESTORE fs;*/\
165
.pushsection .fixup,"ax";	\
L
Linus Torvalds 已提交
166
4:	movl $0,(%esp);	\
167 168
	jmp 1b;		\
5:	movl $0,(%esp);	\
L
Linus Torvalds 已提交
169
	jmp 2b;		\
170 171
6:	movl $0,(%esp);	\
	jmp 3b;		\
L
Linus Torvalds 已提交
172 173
.section __ex_table,"a";\
	.align 4;	\
174 175 176 177
	.long 1b,4b;	\
	.long 2b,5b;	\
	.long 3b,6b;	\
.popsection
L
Linus Torvalds 已提交
178

179 180
#define RING0_INT_FRAME \
	CFI_STARTPROC simple;\
181
	CFI_SIGNAL_FRAME;\
182 183 184 185 186 187
	CFI_DEF_CFA esp, 3*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_EC_FRAME \
	CFI_STARTPROC simple;\
188
	CFI_SIGNAL_FRAME;\
189 190 191 192 193 194
	CFI_DEF_CFA esp, 4*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_PTREGS_FRAME \
	CFI_STARTPROC simple;\
195
	CFI_SIGNAL_FRAME;\
196 197 198 199 200 201 202 203 204 205 206 207
	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
	CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
	CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
	CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
	CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
	CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
	CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
	CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
L
Linus Torvalds 已提交
208 209

ENTRY(ret_from_fork)
210
	CFI_STARTPROC
L
Linus Torvalds 已提交
211
	pushl %eax
212
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
213 214 215
	call schedule_tail
	GET_THREAD_INFO(%ebp)
	popl %eax
216
	CFI_ADJUST_CFA_OFFSET -4
217 218 219 220
	pushl $0x0202			# Reset kernel eflags
	CFI_ADJUST_CFA_OFFSET 4
	popfl
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
221
	jmp syscall_exit
222
	CFI_ENDPROC
223
END(ret_from_fork)
L
Linus Torvalds 已提交
224 225 226 227 228 229 230 231 232 233

/*
 * Return to user mode is not as complex as all this looks,
 * but we want the default path for a system call return to
 * go as quickly as possible which is why some of this is
 * less clear than it otherwise should be.
 */

	# userspace resumption stub bypassing syscall exit tracing
	ALIGN
234
	RING0_PTREGS_FRAME
L
Linus Torvalds 已提交
235
ret_from_exception:
236
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
237 238
ret_from_intr:
	GET_THREAD_INFO(%ebp)
239
check_userspace:
240 241
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
	movb PT_CS(%esp), %al
242
	andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
243 244
	cmpl $USER_RPL, %eax
	jb resume_kernel		# not returning to v8086 or userspace
245

L
Linus Torvalds 已提交
246
ENTRY(resume_userspace)
247
	LOCKDEP_SYS_EXIT
248
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
249 250
					# setting need_resched or sigpending
					# between sampling and the iret
251
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
252 253 254 255 256
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
					# int/exception return?
	jne work_pending
	jmp restore_all
257
END(ret_from_exception)
L
Linus Torvalds 已提交
258 259 260

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
261
	DISABLE_INTERRUPTS(CLBR_ANY)
L
Linus Torvalds 已提交
262 263 264 265 266 267
	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
	jnz restore_nocheck
need_resched:
	movl TI_flags(%ebp), %ecx	# need_resched set ?
	testb $_TIF_NEED_RESCHED, %cl
	jz restore_all
268
	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
L
Linus Torvalds 已提交
269 270 271
	jz restore_all
	call preempt_schedule_irq
	jmp need_resched
272
END(resume_kernel)
L
Linus Torvalds 已提交
273
#endif
274
	CFI_ENDPROC
L
Linus Torvalds 已提交
275 276 277 278 279

/* SYSENTER_RETURN points to after the "sysenter" instruction in
   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */

	# sysenter call handler stub
R
Roland McGrath 已提交
280
ENTRY(ia32_sysenter_target)
281
	CFI_STARTPROC simple
282
	CFI_SIGNAL_FRAME
283 284
	CFI_DEF_CFA esp, 0
	CFI_REGISTER esp, ebp
285
	movl TSS_sysenter_sp0(%esp),%esp
L
Linus Torvalds 已提交
286
sysenter_past_esp:
287
	/*
288 289 290
	 * Interrupts are disabled here, but we can't trace it until
	 * enough kernel state to call TRACE_IRQS_OFF can be called - but
	 * we immediately enable interrupts at that point anyway.
291
	 */
L
Linus Torvalds 已提交
292
	pushl $(__USER_DS)
293 294
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ss, 0*/
L
Linus Torvalds 已提交
295
	pushl %ebp
296 297
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esp, 0
L
Linus Torvalds 已提交
298
	pushfl
299
	orl $X86_EFLAGS_IF, (%esp)
300
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
301
	pushl $(__USER_CS)
302 303
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET cs, 0*/
304 305 306 307 308 309
	/*
	 * Push current_thread_info()->sysenter_return to the stack.
	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
	 */
	pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
310 311
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
312

313 314 315 316 317
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	ENABLE_INTERRUPTS(CLBR_NONE)

L
Linus Torvalds 已提交
318 319 320 321 322 323 324
/*
 * Load the potential sixth argument from user stack.
 * Careful about security.
 */
	cmpl $__PAGE_OFFSET-3,%ebp
	jae syscall_fault
1:	movl (%ebp),%ebp
325
	movl %ebp,PT_EBP(%esp)
L
Linus Torvalds 已提交
326 327 328 329 330 331 332 333
.section __ex_table,"a"
	.align 4
	.long 1b,syscall_fault
.previous

	GET_THREAD_INFO(%ebp)

	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
334
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
335 336 337 338
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
	call *sys_call_table(,%eax,4)
339
	movl %eax,PT_EAX(%esp)
340
	LOCKDEP_SYS_EXIT
341
	DISABLE_INTERRUPTS(CLBR_ANY)
342
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
343 344 345 346
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx
	jne syscall_exit_work
/* if something modifies registers it must also disable sysexit */
347 348
	movl PT_EIP(%esp), %edx
	movl PT_OLDESP(%esp), %ecx
L
Linus Torvalds 已提交
349
	xorl %ebp,%ebp
350
	TRACE_IRQS_ON
351
1:	mov  PT_FS(%esp), %fs
352
	ENABLE_INTERRUPTS_SYSEXIT
353
	CFI_ENDPROC
354
.pushsection .fixup,"ax"
355
2:	movl $0,PT_FS(%esp)
356 357 358 359 360
	jmp 1b
.section __ex_table,"a"
	.align 4
	.long 1b,2b
.popsection
R
Roland McGrath 已提交
361
ENDPROC(ia32_sysenter_target)
L
Linus Torvalds 已提交
362 363 364

	# system call handler stub
ENTRY(system_call)
365
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
366
	pushl %eax			# save orig_eax
367
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
368 369
	SAVE_ALL
	GET_THREAD_INFO(%ebp)
370
					# system call tracing in operation / emulation
L
Linus Torvalds 已提交
371
	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
372
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
373 374 375 376 377
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
syscall_call:
	call *sys_call_table(,%eax,4)
378
	movl %eax,PT_EAX(%esp)		# store the return value
L
Linus Torvalds 已提交
379
syscall_exit:
380
	LOCKDEP_SYS_EXIT
381
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
382 383
					# setting need_resched or sigpending
					# between sampling and the iret
384
	TRACE_IRQS_OFF
385
	testl $X86_EFLAGS_TF,PT_EFLAGS(%esp)	# If tracing set singlestep flag on exit
386 387 388
	jz no_singlestep
	orl $_TIF_SINGLESTEP,TI_flags(%ebp)
no_singlestep:
L
Linus Torvalds 已提交
389 390 391 392 393
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx	# current->work
	jne syscall_exit_work

restore_all:
394 395
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
396 397
	# are returning to the kernel.
	# See comments in process.c:copy_thread() for details.
398 399
	movb PT_OLDSS(%esp), %ah
	movb PT_CS(%esp), %al
400
	andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
401
	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
402
	CFI_REMEMBER_STATE
L
Linus Torvalds 已提交
403 404
	je ldt_ss			# returning to user-space with LDT SS
restore_nocheck:
405 406
	TRACE_IRQS_IRET
restore_nocheck_notrace:
L
Linus Torvalds 已提交
407
	RESTORE_REGS
408
	addl $4, %esp			# skip orig_eax/error_code
409
	CFI_ADJUST_CFA_OFFSET -4
A
Adrian Bunk 已提交
410
irq_return:
I
Ingo Molnar 已提交
411
	INTERRUPT_RETURN
L
Linus Torvalds 已提交
412
.section .fixup,"ax"
413
ENTRY(iret_exc)
414 415 416
	pushl $0			# no error code
	pushl $do_iret_error
	jmp error_code
L
Linus Torvalds 已提交
417 418 419
.previous
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
420
	.long irq_return,iret_exc
L
Linus Torvalds 已提交
421 422
.previous

423
	CFI_RESTORE_STATE
L
Linus Torvalds 已提交
424
ldt_ss:
425
	larl PT_OLDSS(%esp), %eax
L
Linus Torvalds 已提交
426 427 428
	jnz restore_nocheck
	testl $0x00400000, %eax		# returning to 32bit stack?
	jnz restore_nocheck		# allright, normal return
429 430 431 432 433 434 435 436 437 438

#ifdef CONFIG_PARAVIRT
	/*
	 * The kernel can't run on a non-flat stack if paravirt mode
	 * is active.  Rather than try to fixup the high bits of
	 * ESP, bypass this code entirely.  This may break DOSemu
	 * and/or Wine support in a paravirt VM, although the option
	 * is still available to implement the setting of the high
	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
	 */
439
	cmpl $0, pv_info+PARAVIRT_enabled
440 441 442
	jne restore_nocheck
#endif

L
Linus Torvalds 已提交
443 444 445 446 447 448
	/* If returning to userspace with 16bit stack,
	 * try to fix the higher word of ESP, as the CPU
	 * won't restore it.
	 * This is an "official" bug of all the x86-compatible
	 * CPUs, which we can try to work around to make
	 * dosemu and wine happy. */
449
	movl PT_OLDESP(%esp), %eax
S
Stas Sergeev 已提交
450 451 452 453 454 455
	movl %esp, %edx
	call patch_espfix_desc
	pushl $__ESPFIX_SS
	CFI_ADJUST_CFA_OFFSET 4
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
456
	DISABLE_INTERRUPTS(CLBR_EAX)
457
	TRACE_IRQS_OFF
S
Stas Sergeev 已提交
458 459 460
	lss (%esp), %esp
	CFI_ADJUST_CFA_OFFSET -8
	jmp restore_nocheck
461
	CFI_ENDPROC
462
ENDPROC(system_call)
L
Linus Torvalds 已提交
463 464 465

	# perform work that needs to be done immediately before resumption
	ALIGN
466
	RING0_PTREGS_FRAME		# can't unwind into user space anyway
L
Linus Torvalds 已提交
467 468 469 470 471
work_pending:
	testb $_TIF_NEED_RESCHED, %cl
	jz work_notifysig
work_resched:
	call schedule
472
	LOCKDEP_SYS_EXIT
473
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
474 475
					# setting need_resched or sigpending
					# between sampling and the iret
476
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
477 478 479 480 481 482 483 484 485
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
					# than syscall tracing?
	jz restore_all
	testb $_TIF_NEED_RESCHED, %cl
	jnz work_resched

work_notifysig:				# deal with pending signals and
					# notify-resume requests
486
#ifdef CONFIG_VM86
487
	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
L
Linus Torvalds 已提交
488 489 490 491 492
	movl %esp, %eax
	jne work_notifysig_v86		# returning to kernel-space or
					# vm86-space
	xorl %edx, %edx
	call do_notify_resume
493
	jmp resume_userspace_sig
L
Linus Torvalds 已提交
494 495 496 497

	ALIGN
work_notifysig_v86:
	pushl %ecx			# save ti_flags for do_notify_resume
498
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
499 500
	call save_v86_state		# %eax contains pt_regs pointer
	popl %ecx
501
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
502
	movl %eax, %esp
503 504 505
#else
	movl %esp, %eax
#endif
L
Linus Torvalds 已提交
506 507
	xorl %edx, %edx
	call do_notify_resume
508
	jmp resume_userspace_sig
509
END(work_pending)
L
Linus Torvalds 已提交
510 511 512 513

	# perform syscall exit tracing
	ALIGN
syscall_trace_entry:
514
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
515 516 517
	movl %esp, %eax
	xorl %edx,%edx
	call do_syscall_trace
518
	cmpl $0, %eax
519
	jne resume_userspace		# ret != 0 -> running under PTRACE_SYSEMU,
520
					# so must skip actual syscall
521
	movl PT_ORIG_EAX(%esp), %eax
L
Linus Torvalds 已提交
522 523 524
	cmpl $(nr_syscalls), %eax
	jnae syscall_call
	jmp syscall_exit
525
END(syscall_trace_entry)
L
Linus Torvalds 已提交
526 527 528 529 530 531

	# perform syscall exit tracing
	ALIGN
syscall_exit_work:
	testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
	jz work_pending
532
	TRACE_IRQS_ON
533
	ENABLE_INTERRUPTS(CLBR_ANY)	# could let do_syscall_trace() call
L
Linus Torvalds 已提交
534 535 536 537 538
					# schedule() instead
	movl %esp, %eax
	movl $1, %edx
	call do_syscall_trace
	jmp resume_userspace
539
END(syscall_exit_work)
540
	CFI_ENDPROC
L
Linus Torvalds 已提交
541

542
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
543 544
syscall_fault:
	GET_THREAD_INFO(%ebp)
545
	movl $-EFAULT,PT_EAX(%esp)
L
Linus Torvalds 已提交
546
	jmp resume_userspace
547
END(syscall_fault)
L
Linus Torvalds 已提交
548 549

syscall_badsys:
550
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
551
	jmp resume_userspace
552
END(syscall_badsys)
553
	CFI_ENDPROC
L
Linus Torvalds 已提交
554 555

#define FIXUP_ESPFIX_STACK \
S
Stas Sergeev 已提交
556
	/* since we are on a wrong stack, we cant make it a C code :( */ \
557
	PER_CPU(gdt_page, %ebx); \
S
Stas Sergeev 已提交
558 559 560 561
	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
	addl %esp, %eax; \
	pushl $__KERNEL_DS; \
	CFI_ADJUST_CFA_OFFSET 4; \
L
Linus Torvalds 已提交
562
	pushl %eax; \
563
	CFI_ADJUST_CFA_OFFSET 4; \
S
Stas Sergeev 已提交
564 565 566
	lss (%esp), %esp; \
	CFI_ADJUST_CFA_OFFSET -8;
#define UNWIND_ESPFIX_STACK \
L
Linus Torvalds 已提交
567
	movl %ss, %eax; \
S
Stas Sergeev 已提交
568
	/* see if on espfix stack */ \
L
Linus Torvalds 已提交
569
	cmpw $__ESPFIX_SS, %ax; \
S
Stas Sergeev 已提交
570 571
	jne 27f; \
	movl $__KERNEL_DS, %eax; \
572 573
	movl %eax, %ds; \
	movl %eax, %es; \
S
Stas Sergeev 已提交
574
	/* switch to normal stack */ \
575
	FIXUP_ESPFIX_STACK; \
S
Stas Sergeev 已提交
576
27:;
L
Linus Torvalds 已提交
577 578 579 580 581

/*
 * Build the entry stubs and pointer table with
 * some assembler magic.
 */
J
Jan Beulich 已提交
582
.section .rodata,"a"
L
Linus Torvalds 已提交
583 584 585 586
ENTRY(interrupt)
.text

ENTRY(irq_entries_start)
587
	RING0_INT_FRAME
588
vector=0
L
Linus Torvalds 已提交
589 590
.rept NR_IRQS
	ALIGN
591 592 593
 .if vector
	CFI_ADJUST_CFA_OFFSET -4
 .endif
594
1:	pushl $~(vector)
595
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
596
	jmp common_interrupt
597
 .previous
L
Linus Torvalds 已提交
598
	.long 1b
599
 .text
L
Linus Torvalds 已提交
600 601
vector=vector+1
.endr
602 603 604 605 606
END(irq_entries_start)

.previous
END(interrupt)
.previous
L
Linus Torvalds 已提交
607

608 609 610 611
/*
 * the CPU automatically disables interrupts when executing an IRQ vector,
 * so IRQ-flags tracing has to follow that:
 */
L
Linus Torvalds 已提交
612 613 614
	ALIGN
common_interrupt:
	SAVE_ALL
615
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
616 617 618
	movl %esp,%eax
	call do_IRQ
	jmp ret_from_intr
619
ENDPROC(common_interrupt)
620
	CFI_ENDPROC
L
Linus Torvalds 已提交
621 622 623

#define BUILD_INTERRUPT(name, nr)	\
ENTRY(name)				\
624
	RING0_INT_FRAME;		\
625
	pushl $~(nr);			\
626 627
	CFI_ADJUST_CFA_OFFSET 4;	\
	SAVE_ALL;			\
628
	TRACE_IRQS_OFF			\
L
Linus Torvalds 已提交
629
	movl %esp,%eax;			\
630
	call smp_##name;		\
631
	jmp ret_from_intr;		\
632 633
	CFI_ENDPROC;			\
ENDPROC(name)
L
Linus Torvalds 已提交
634 635 636 637

/* The include is where all of the SMP etc. interrupts come from */
#include "entry_arch.h"

638 639 640
KPROBE_ENTRY(page_fault)
	RING0_EC_FRAME
	pushl $do_page_fault
641
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
642 643
	ALIGN
error_code:
644
	/* the function address is in %fs's slot on the stack */
645 646 647
	pushl %es
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET es, 0*/
L
Linus Torvalds 已提交
648
	pushl %ds
649 650
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ds, 0*/
L
Linus Torvalds 已提交
651
	pushl %eax
652 653
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eax, 0
L
Linus Torvalds 已提交
654
	pushl %ebp
655 656
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebp, 0
L
Linus Torvalds 已提交
657
	pushl %edi
658 659
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edi, 0
L
Linus Torvalds 已提交
660
	pushl %esi
661 662
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esi, 0
L
Linus Torvalds 已提交
663
	pushl %edx
664 665
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edx, 0
L
Linus Torvalds 已提交
666
	pushl %ecx
667 668
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ecx, 0
L
Linus Torvalds 已提交
669
	pushl %ebx
670 671
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebx, 0
L
Linus Torvalds 已提交
672
	cld
673
	pushl %fs
674
	CFI_ADJUST_CFA_OFFSET 4
675
	/*CFI_REL_OFFSET fs, 0*/
676
	movl $(__KERNEL_PERCPU), %ecx
677
	movl %ecx, %fs
L
Linus Torvalds 已提交
678 679
	UNWIND_ESPFIX_STACK
	popl %ecx
680 681
	CFI_ADJUST_CFA_OFFSET -4
	/*CFI_REGISTER es, ecx*/
682
	movl PT_FS(%esp), %edi		# get the function address
683
	movl PT_ORIG_EAX(%esp), %edx	# get the error code
684
	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
685 686
	mov  %ecx, PT_FS(%esp)
	/*CFI_REL_OFFSET fs, ES*/
L
Linus Torvalds 已提交
687 688 689 690 691 692
	movl $(__USER_DS), %ecx
	movl %ecx, %ds
	movl %ecx, %es
	movl %esp,%eax			# pt_regs pointer
	call *%edi
	jmp ret_from_exception
693
	CFI_ENDPROC
694
KPROBE_END(page_fault)
L
Linus Torvalds 已提交
695 696

ENTRY(coprocessor_error)
697
	RING0_INT_FRAME
L
Linus Torvalds 已提交
698
	pushl $0
699
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
700
	pushl $do_coprocessor_error
701
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
702
	jmp error_code
703
	CFI_ENDPROC
704
END(coprocessor_error)
L
Linus Torvalds 已提交
705 706

ENTRY(simd_coprocessor_error)
707
	RING0_INT_FRAME
L
Linus Torvalds 已提交
708
	pushl $0
709
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
710
	pushl $do_simd_coprocessor_error
711
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
712
	jmp error_code
713
	CFI_ENDPROC
714
END(simd_coprocessor_error)
L
Linus Torvalds 已提交
715 716

ENTRY(device_not_available)
717
	RING0_INT_FRAME
L
Linus Torvalds 已提交
718
	pushl $-1			# mark this as an int
719
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
720
	SAVE_ALL
721
	GET_CR0_INTO_EAX
L
Linus Torvalds 已提交
722 723
	testl $0x4, %eax		# EM (math emulation bit)
	jne device_not_available_emulate
724
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
725 726 727 728
	call math_state_restore
	jmp ret_from_exception
device_not_available_emulate:
	pushl $0			# temporary storage for ORIG_EIP
729
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
730 731
	call math_emulate
	addl $4, %esp
732
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
733
	jmp ret_from_exception
734
	CFI_ENDPROC
735
END(device_not_available)
L
Linus Torvalds 已提交
736 737 738 739 740 741

/*
 * Debug traps and NMI can happen at the one SYSENTER instruction
 * that sets up the real kernel stack. Check here, since we can't
 * allow the wrong stack to be used.
 *
742
 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
L
Linus Torvalds 已提交
743 744 745 746 747 748 749 750 751 752 753
 * already pushed 3 words if it hits on the sysenter instruction:
 * eflags, cs and eip.
 *
 * We just load the right stack, and push the three (known) values
 * by hand onto the new stack - while updating the return eip past
 * the instruction that would have done it for sysenter.
 */
#define FIX_STACK(offset, ok, label)		\
	cmpw $__KERNEL_CS,4(%esp);		\
	jne ok;					\
label:						\
754
	movl TSS_sysenter_sp0+offset(%esp),%esp;	\
755 756
	CFI_DEF_CFA esp, 0;			\
	CFI_UNDEFINED eip;			\
L
Linus Torvalds 已提交
757
	pushfl;					\
758
	CFI_ADJUST_CFA_OFFSET 4;		\
L
Linus Torvalds 已提交
759
	pushl $__KERNEL_CS;			\
760 761 762 763
	CFI_ADJUST_CFA_OFFSET 4;		\
	pushl $sysenter_past_esp;		\
	CFI_ADJUST_CFA_OFFSET 4;		\
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
764

765
KPROBE_ENTRY(debug)
766
	RING0_INT_FRAME
R
Roland McGrath 已提交
767
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
768 769 770 771
	jne debug_stack_correct
	FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
debug_stack_correct:
	pushl $-1			# mark this as an int
772
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
773 774 775 776 777
	SAVE_ALL
	xorl %edx,%edx			# error code 0
	movl %esp,%eax			# pt_regs pointer
	call do_debug
	jmp ret_from_exception
778
	CFI_ENDPROC
779 780
KPROBE_END(debug)

L
Linus Torvalds 已提交
781 782 783 784 785 786 787 788
/*
 * NMI is doubly nasty. It can happen _while_ we're handling
 * a debug fault, and the debug fault hasn't yet been able to
 * clear up the stack. So we first check whether we got  an
 * NMI on the sysenter entry path, but after that we need to
 * check whether we got an NMI on the debug path where the debug
 * fault happened on the sysenter path.
 */
789
KPROBE_ENTRY(nmi)
790
	RING0_INT_FRAME
L
Linus Torvalds 已提交
791
	pushl %eax
792
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
793 794 795
	movl %ss, %eax
	cmpw $__ESPFIX_SS, %ax
	popl %eax
796
	CFI_ADJUST_CFA_OFFSET -4
S
Stas Sergeev 已提交
797
	je nmi_espfix_stack
R
Roland McGrath 已提交
798
	cmpl $ia32_sysenter_target,(%esp)
L
Linus Torvalds 已提交
799 800
	je nmi_stack_fixup
	pushl %eax
801
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
802 803 804 805 806 807 808
	movl %esp,%eax
	/* Do not access memory above the end of our stack page,
	 * it might not exist.
	 */
	andl $(THREAD_SIZE-1),%eax
	cmpl $(THREAD_SIZE-20),%eax
	popl %eax
809
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
810
	jae nmi_stack_correct
R
Roland McGrath 已提交
811
	cmpl $ia32_sysenter_target,12(%esp)
L
Linus Torvalds 已提交
812 813
	je nmi_debug_stack_check
nmi_stack_correct:
814
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
815
	pushl %eax
816
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
817 818 819 820
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_nmi
821
	jmp restore_nocheck_notrace
822
	CFI_ENDPROC
L
Linus Torvalds 已提交
823 824

nmi_stack_fixup:
825
	RING0_INT_FRAME
L
Linus Torvalds 已提交
826 827
	FIX_STACK(12,nmi_stack_correct, 1)
	jmp nmi_stack_correct
828

L
Linus Torvalds 已提交
829
nmi_debug_stack_check:
830
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
831 832
	cmpw $__KERNEL_CS,16(%esp)
	jne nmi_stack_correct
833 834
	cmpl $debug,(%esp)
	jb nmi_stack_correct
L
Linus Torvalds 已提交
835
	cmpl $debug_esp_fix_insn,(%esp)
836
	ja nmi_stack_correct
L
Linus Torvalds 已提交
837 838 839
	FIX_STACK(24,nmi_stack_correct, 1)
	jmp nmi_stack_correct

S
Stas Sergeev 已提交
840
nmi_espfix_stack:
841 842 843 844
	/* We have a RING0_INT_FRAME here.
	 *
	 * create the pointer to lss back
	 */
L
Linus Torvalds 已提交
845
	pushl %ss
846
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
847
	pushl %esp
848
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
849 850 851 852
	addw $4, (%esp)
	/* copy the iret frame of 12 bytes */
	.rept 3
	pushl 16(%esp)
853
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
854 855
	.endr
	pushl %eax
856
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
857 858 859 860 861
	SAVE_ALL
	FIXUP_ESPFIX_STACK		# %eax == %esp
	xorl %edx,%edx			# zero error code
	call do_nmi
	RESTORE_REGS
S
Stas Sergeev 已提交
862 863
	lss 12+4(%esp), %esp		# back to espfix stack
	CFI_ADJUST_CFA_OFFSET -24
I
Ingo Molnar 已提交
864
	jmp irq_return
865
	CFI_ENDPROC
866
KPROBE_END(nmi)
L
Linus Torvalds 已提交
867

868 869
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
I
Ingo Molnar 已提交
870
	iret
871 872
.section __ex_table,"a"
	.align 4
I
Ingo Molnar 已提交
873
	.long native_iret, iret_exc
874
.previous
875
END(native_iret)
876

877
ENTRY(native_irq_enable_sysexit)
878 879
	sti
	sysexit
880
END(native_irq_enable_sysexit)
881 882
#endif

883
KPROBE_ENTRY(int3)
884
	RING0_INT_FRAME
L
Linus Torvalds 已提交
885
	pushl $-1			# mark this as an int
886
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
887 888 889 890 891
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_int3
	jmp ret_from_exception
892
	CFI_ENDPROC
893
KPROBE_END(int3)
L
Linus Torvalds 已提交
894 895

ENTRY(overflow)
896
	RING0_INT_FRAME
L
Linus Torvalds 已提交
897
	pushl $0
898
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
899
	pushl $do_overflow
900
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
901
	jmp error_code
902
	CFI_ENDPROC
903
END(overflow)
L
Linus Torvalds 已提交
904 905

ENTRY(bounds)
906
	RING0_INT_FRAME
L
Linus Torvalds 已提交
907
	pushl $0
908
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
909
	pushl $do_bounds
910
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
911
	jmp error_code
912
	CFI_ENDPROC
913
END(bounds)
L
Linus Torvalds 已提交
914 915

ENTRY(invalid_op)
916
	RING0_INT_FRAME
L
Linus Torvalds 已提交
917
	pushl $0
918
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
919
	pushl $do_invalid_op
920
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
921
	jmp error_code
922
	CFI_ENDPROC
923
END(invalid_op)
L
Linus Torvalds 已提交
924 925

ENTRY(coprocessor_segment_overrun)
926
	RING0_INT_FRAME
L
Linus Torvalds 已提交
927
	pushl $0
928
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
929
	pushl $do_coprocessor_segment_overrun
930
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
931
	jmp error_code
932
	CFI_ENDPROC
933
END(coprocessor_segment_overrun)
L
Linus Torvalds 已提交
934 935

ENTRY(invalid_TSS)
936
	RING0_EC_FRAME
L
Linus Torvalds 已提交
937
	pushl $do_invalid_TSS
938
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
939
	jmp error_code
940
	CFI_ENDPROC
941
END(invalid_TSS)
L
Linus Torvalds 已提交
942 943

ENTRY(segment_not_present)
944
	RING0_EC_FRAME
L
Linus Torvalds 已提交
945
	pushl $do_segment_not_present
946
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
947
	jmp error_code
948
	CFI_ENDPROC
949
END(segment_not_present)
L
Linus Torvalds 已提交
950 951

ENTRY(stack_segment)
952
	RING0_EC_FRAME
L
Linus Torvalds 已提交
953
	pushl $do_stack_segment
954
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
955
	jmp error_code
956
	CFI_ENDPROC
957
END(stack_segment)
L
Linus Torvalds 已提交
958

959
KPROBE_ENTRY(general_protection)
960
	RING0_EC_FRAME
L
Linus Torvalds 已提交
961
	pushl $do_general_protection
962
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
963
	jmp error_code
964
	CFI_ENDPROC
965
KPROBE_END(general_protection)
L
Linus Torvalds 已提交
966 967

ENTRY(alignment_check)
968
	RING0_EC_FRAME
L
Linus Torvalds 已提交
969
	pushl $do_alignment_check
970
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
971
	jmp error_code
972
	CFI_ENDPROC
973
END(alignment_check)
L
Linus Torvalds 已提交
974

975 976 977 978 979
ENTRY(divide_error)
	RING0_INT_FRAME
	pushl $0			# no error code
	CFI_ADJUST_CFA_OFFSET 4
	pushl $do_divide_error
980
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
981
	jmp error_code
982
	CFI_ENDPROC
983
END(divide_error)
L
Linus Torvalds 已提交
984 985 986

#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
987
	RING0_INT_FRAME
L
Linus Torvalds 已提交
988
	pushl $0
989
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
990
	pushl machine_check_vector
991
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
992
	jmp error_code
993
	CFI_ENDPROC
994
END(machine_check)
L
Linus Torvalds 已提交
995 996 997
#endif

ENTRY(spurious_interrupt_bug)
998
	RING0_INT_FRAME
L
Linus Torvalds 已提交
999
	pushl $0
1000
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1001
	pushl $do_spurious_interrupt_bug
1002
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1003
	jmp error_code
1004
	CFI_ENDPROC
1005
END(spurious_interrupt_bug)
L
Linus Torvalds 已提交
1006

1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
ENTRY(kernel_thread_helper)
	pushl $0		# fake return address for unwinder
	CFI_STARTPROC
	movl %edx,%eax
	push %edx
	CFI_ADJUST_CFA_OFFSET 4
	call *%ebx
	push %eax
	CFI_ADJUST_CFA_OFFSET 4
	call do_exit
	CFI_ENDPROC
ENDPROC(kernel_thread_helper)

1020
#ifdef CONFIG_XEN
1021 1022 1023 1024 1025 1026
/* Xen doesn't set %esp to be precisely what the normal sysenter
   entrypoint expects, so fix it up before using the normal path. */
ENTRY(xen_sysenter_target)
	RING0_INT_FRAME
	addl $5*4, %esp		/* remove xen-provided frame */
	jmp sysenter_past_esp
G
Glauber Costa 已提交
1027
	CFI_ENDPROC
1028

1029 1030 1031 1032 1033 1034
ENTRY(xen_hypervisor_callback)
	CFI_STARTPROC
	pushl $0
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	TRACE_IRQS_OFF
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046

	/* Check to see if we got the event in the critical
	   region in xen_iret_direct, after we've reenabled
	   events and checked for pending events.  This simulates
	   iret instruction's behaviour where it delivers a
	   pending interrupt when enabling interrupts. */
	movl PT_EIP(%esp),%eax
	cmpl $xen_iret_start_crit,%eax
	jb   1f
	cmpl $xen_iret_end_crit,%eax
	jae  1f

J
Jeremy Fitzhardinge 已提交
1047
	jmp  xen_iret_crit_fixup
1048 1049

ENTRY(xen_do_upcall)
1050
1:	mov %esp, %eax
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	call xen_evtchn_do_upcall
	jmp  ret_from_intr
	CFI_ENDPROC
ENDPROC(xen_hypervisor_callback)

# Hypervisor uses this for application faults while it executes.
# We get here for two reasons:
#  1. Fault while reloading DS, ES, FS or GS
#  2. Fault while executing IRET
# Category 1 we fix up by reattempting the load, and zeroing the segment
# register if the load fails.
# Category 2 we fix up by jumping to do_iret_error. We cannot use the
# normal Linux return path in this case because if we use the IRET hypercall
# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
# We distinguish between categories by maintaining a status value in EAX.
ENTRY(xen_failsafe_callback)
	CFI_STARTPROC
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
	movl $1,%eax
1:	mov 4(%esp),%ds
2:	mov 8(%esp),%es
3:	mov 12(%esp),%fs
4:	mov 16(%esp),%gs
	testl %eax,%eax
	popl %eax
	CFI_ADJUST_CFA_OFFSET -4
	lea 16(%esp),%esp
	CFI_ADJUST_CFA_OFFSET -16
	jz 5f
	addl $16,%esp
	jmp iret_exc		# EAX != 0 => Category 2 (Bad IRET)
5:	pushl $0		# EAX == 0 => Category 1 (Bad segment)
	CFI_ADJUST_CFA_OFFSET 4
	SAVE_ALL
	jmp ret_from_exception
	CFI_ENDPROC

.section .fixup,"ax"
6:	xorl %eax,%eax
	movl %eax,4(%esp)
	jmp 1b
7:	xorl %eax,%eax
	movl %eax,8(%esp)
	jmp 2b
8:	xorl %eax,%eax
	movl %eax,12(%esp)
	jmp 3b
9:	xorl %eax,%eax
	movl %eax,16(%esp)
	jmp 4b
.previous
.section __ex_table,"a"
	.align 4
	.long 1b,6b
	.long 2b,7b
	.long 3b,8b
	.long 4b,9b
.previous
ENDPROC(xen_failsafe_callback)

#endif	/* CONFIG_XEN */

1114
.section .rodata,"a"
1115
#include "syscall_table_32.S"
L
Linus Torvalds 已提交
1116 1117

syscall_table_size=(.-sys_call_table)