entry.S 23.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 *  linux/arch/i386/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 * This also contains the timer-interrupt handler, as well as all interrupts
 * and faults that can result in a task-switch.
 *
 * NOTE: This code handles signal-recognition, which happens every time
 * after a timer-interrupt and after each system call.
 *
 * I changed all the .align's to 4 (16 byte alignment), as that's faster
 * on a 486.
 *
 * Stack layout in 'ret_from_system_call':
 * 	ptrace needs to have all regs on the stack.
 *	if the order here is changed, it needs to be
 *	updated in fork.c:copy_process, signal.c:do_signal,
 *	ptrace.c and ptrace.h
 *
 *	 0(%esp) - %ebx
 *	 4(%esp) - %ecx
 *	 8(%esp) - %edx
 *       C(%esp) - %esi
 *	10(%esp) - %edi
 *	14(%esp) - %ebp
 *	18(%esp) - %eax
 *	1C(%esp) - %ds
 *	20(%esp) - %es
33
 *	24(%esp) - %fs
34 35 36 37 38 39
 *	28(%esp) - orig_eax
 *	2C(%esp) - %eip
 *	30(%esp) - %cs
 *	34(%esp) - %eflags
 *	38(%esp) - %oldesp
 *	3C(%esp) - %oldss
L
Linus Torvalds 已提交
40 41 42 43 44 45
 *
 * "current" is in register %ebx during any slow entries.
 */

#include <linux/linkage.h>
#include <asm/thread_info.h>
46
#include <asm/irqflags.h>
L
Linus Torvalds 已提交
47 48 49 50 51
#include <asm/errno.h>
#include <asm/segment.h>
#include <asm/smp.h>
#include <asm/page.h>
#include <asm/desc.h>
S
Stas Sergeev 已提交
52
#include <asm/percpu.h>
53
#include <asm/dwarf2.h>
L
Linus Torvalds 已提交
54 55
#include "irq_vectors.h"

56 57 58 59 60 61 62 63 64 65 66 67 68
/*
 * We use macros for low-level operations which need to be overridden
 * for paravirtualization.  The following will never clobber any registers:
 *   INTERRUPT_RETURN (aka. "iret")
 *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
 *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
 *
 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
 * Allowing a register to be clobbered can shrink the paravirt replacement
 * enough to patch inline, increasing performance.
 */

L
Linus Torvalds 已提交
69 70 71 72 73 74 75 76 77 78
#define nr_syscalls ((syscall_table_size)/4)

CF_MASK		= 0x00000001
TF_MASK		= 0x00000100
IF_MASK		= 0x00000200
DF_MASK		= 0x00000400 
NT_MASK		= 0x00004000
VM_MASK		= 0x00020000

#ifdef CONFIG_PREEMPT
79
#define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
L
Linus Torvalds 已提交
80
#else
81
#define preempt_stop(clobbers)
L
Linus Torvalds 已提交
82 83 84
#define resume_kernel		restore_nocheck
#endif

85 86
.macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS
87
	testl $IF_MASK,PT_EFLAGS(%esp)     # interrupts off?
88 89 90 91 92 93
	jz 1f
	TRACE_IRQS_ON
1:
#endif
.endm

94 95 96 97 98 99
#ifdef CONFIG_VM86
#define resume_userspace_sig	check_userspace
#else
#define resume_userspace_sig	resume_userspace
#endif

L
Linus Torvalds 已提交
100 101
#define SAVE_ALL \
	cld; \
102
	pushl %fs; \
103
	CFI_ADJUST_CFA_OFFSET 4;\
104
	/*CFI_REL_OFFSET fs, 0;*/\
L
Linus Torvalds 已提交
105
	pushl %es; \
106 107
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET es, 0;*/\
L
Linus Torvalds 已提交
108
	pushl %ds; \
109 110
	CFI_ADJUST_CFA_OFFSET 4;\
	/*CFI_REL_OFFSET ds, 0;*/\
L
Linus Torvalds 已提交
111
	pushl %eax; \
112 113
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET eax, 0;\
L
Linus Torvalds 已提交
114
	pushl %ebp; \
115 116
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebp, 0;\
L
Linus Torvalds 已提交
117
	pushl %edi; \
118 119
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edi, 0;\
L
Linus Torvalds 已提交
120
	pushl %esi; \
121 122
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET esi, 0;\
L
Linus Torvalds 已提交
123
	pushl %edx; \
124 125
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET edx, 0;\
L
Linus Torvalds 已提交
126
	pushl %ecx; \
127 128
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ecx, 0;\
L
Linus Torvalds 已提交
129
	pushl %ebx; \
130 131
	CFI_ADJUST_CFA_OFFSET 4;\
	CFI_REL_OFFSET ebx, 0;\
L
Linus Torvalds 已提交
132 133
	movl $(__USER_DS), %edx; \
	movl %edx, %ds; \
134 135
	movl %edx, %es; \
	movl $(__KERNEL_PDA), %edx; \
136
	movl %edx, %fs
L
Linus Torvalds 已提交
137 138 139

#define RESTORE_INT_REGS \
	popl %ebx;	\
140 141
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebx;\
L
Linus Torvalds 已提交
142
	popl %ecx;	\
143 144
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ecx;\
L
Linus Torvalds 已提交
145
	popl %edx;	\
146 147
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edx;\
L
Linus Torvalds 已提交
148
	popl %esi;	\
149 150
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE esi;\
L
Linus Torvalds 已提交
151
	popl %edi;	\
152 153
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE edi;\
L
Linus Torvalds 已提交
154
	popl %ebp;	\
155 156 157 158 159
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE ebp;\
	popl %eax;	\
	CFI_ADJUST_CFA_OFFSET -4;\
	CFI_RESTORE eax
L
Linus Torvalds 已提交
160 161 162 163

#define RESTORE_REGS	\
	RESTORE_INT_REGS; \
1:	popl %ds;	\
164 165
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE ds;*/\
L
Linus Torvalds 已提交
166
2:	popl %es;	\
167 168
	CFI_ADJUST_CFA_OFFSET -4;\
	/*CFI_RESTORE es;*/\
169
3:	popl %fs;	\
170
	CFI_ADJUST_CFA_OFFSET -4;\
171
	/*CFI_RESTORE fs;*/\
172
.pushsection .fixup,"ax";	\
L
Linus Torvalds 已提交
173
4:	movl $0,(%esp);	\
174 175
	jmp 1b;		\
5:	movl $0,(%esp);	\
L
Linus Torvalds 已提交
176
	jmp 2b;		\
177 178
6:	movl $0,(%esp);	\
	jmp 3b;		\
L
Linus Torvalds 已提交
179 180
.section __ex_table,"a";\
	.align 4;	\
181 182 183 184
	.long 1b,4b;	\
	.long 2b,5b;	\
	.long 3b,6b;	\
.popsection
L
Linus Torvalds 已提交
185

186 187
#define RING0_INT_FRAME \
	CFI_STARTPROC simple;\
188
	CFI_SIGNAL_FRAME;\
189 190 191 192 193 194
	CFI_DEF_CFA esp, 3*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_EC_FRAME \
	CFI_STARTPROC simple;\
195
	CFI_SIGNAL_FRAME;\
196 197 198 199 200 201
	CFI_DEF_CFA esp, 4*4;\
	/*CFI_OFFSET cs, -2*4;*/\
	CFI_OFFSET eip, -3*4

#define RING0_PTREGS_FRAME \
	CFI_STARTPROC simple;\
202
	CFI_SIGNAL_FRAME;\
203 204 205 206 207 208 209 210 211 212 213 214
	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
	CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
	CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
	CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
	CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
	CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
	CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
	CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
L
Linus Torvalds 已提交
215 216

ENTRY(ret_from_fork)
217
	CFI_STARTPROC
L
Linus Torvalds 已提交
218
	pushl %eax
219
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
220 221 222
	call schedule_tail
	GET_THREAD_INFO(%ebp)
	popl %eax
223
	CFI_ADJUST_CFA_OFFSET -4
224 225 226 227
	pushl $0x0202			# Reset kernel eflags
	CFI_ADJUST_CFA_OFFSET 4
	popfl
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
228
	jmp syscall_exit
229
	CFI_ENDPROC
230
END(ret_from_fork)
L
Linus Torvalds 已提交
231 232 233 234 235 236 237 238 239 240

/*
 * Return to user mode is not as complex as all this looks,
 * but we want the default path for a system call return to
 * go as quickly as possible which is why some of this is
 * less clear than it otherwise should be.
 */

	# userspace resumption stub bypassing syscall exit tracing
	ALIGN
241
	RING0_PTREGS_FRAME
L
Linus Torvalds 已提交
242
ret_from_exception:
243
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
244 245
ret_from_intr:
	GET_THREAD_INFO(%ebp)
246
check_userspace:
247 248
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
	movb PT_CS(%esp), %al
249 250 251
	andl $(VM_MASK | SEGMENT_RPL_MASK), %eax
	cmpl $USER_RPL, %eax
	jb resume_kernel		# not returning to v8086 or userspace
252

L
Linus Torvalds 已提交
253
ENTRY(resume_userspace)
254
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
255 256 257 258 259 260 261
					# setting need_resched or sigpending
					# between sampling and the iret
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
					# int/exception return?
	jne work_pending
	jmp restore_all
262
END(ret_from_exception)
L
Linus Torvalds 已提交
263 264 265

#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
266
	DISABLE_INTERRUPTS(CLBR_ANY)
L
Linus Torvalds 已提交
267 268 269 270 271 272
	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
	jnz restore_nocheck
need_resched:
	movl TI_flags(%ebp), %ecx	# need_resched set ?
	testb $_TIF_NEED_RESCHED, %cl
	jz restore_all
273
	testl $IF_MASK,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
L
Linus Torvalds 已提交
274 275 276
	jz restore_all
	call preempt_schedule_irq
	jmp need_resched
277
END(resume_kernel)
L
Linus Torvalds 已提交
278
#endif
279
	CFI_ENDPROC
L
Linus Torvalds 已提交
280 281 282 283 284 285

/* SYSENTER_RETURN points to after the "sysenter" instruction in
   the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */

	# sysenter call handler stub
ENTRY(sysenter_entry)
286
	CFI_STARTPROC simple
287
	CFI_SIGNAL_FRAME
288 289
	CFI_DEF_CFA esp, 0
	CFI_REGISTER esp, ebp
L
Linus Torvalds 已提交
290 291
	movl TSS_sysenter_esp0(%esp),%esp
sysenter_past_esp:
292 293 294 295
	/*
	 * No need to follow this irqs on/off section: the syscall
	 * disabled irqs and here we enable it straight after entry:
	 */
296
	ENABLE_INTERRUPTS(CLBR_NONE)
L
Linus Torvalds 已提交
297
	pushl $(__USER_DS)
298 299
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ss, 0*/
L
Linus Torvalds 已提交
300
	pushl %ebp
301 302
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esp, 0
L
Linus Torvalds 已提交
303
	pushfl
304
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
305
	pushl $(__USER_CS)
306 307
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET cs, 0*/
308 309 310 311 312 313
	/*
	 * Push current_thread_info()->sysenter_return to the stack.
	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
	 */
	pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
314 315
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
316 317 318 319 320 321 322 323 324 325 326 327 328 329

/*
 * Load the potential sixth argument from user stack.
 * Careful about security.
 */
	cmpl $__PAGE_OFFSET-3,%ebp
	jae syscall_fault
1:	movl (%ebp),%ebp
.section __ex_table,"a"
	.align 4
	.long 1b,syscall_fault
.previous

	pushl %eax
330
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
331 332 333 334
	SAVE_ALL
	GET_THREAD_INFO(%ebp)

	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
335
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
336 337 338 339
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
	call *sys_call_table(,%eax,4)
340
	movl %eax,PT_EAX(%esp)
341
	DISABLE_INTERRUPTS(CLBR_ANY)
342
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
343 344 345 346
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx
	jne syscall_exit_work
/* if something modifies registers it must also disable sysexit */
347 348
	movl PT_EIP(%esp), %edx
	movl PT_OLDESP(%esp), %ecx
L
Linus Torvalds 已提交
349
	xorl %ebp,%ebp
350
	TRACE_IRQS_ON
351
1:	mov  PT_FS(%esp), %fs
352
	ENABLE_INTERRUPTS_SYSEXIT
353
	CFI_ENDPROC
354
.pushsection .fixup,"ax"
355
2:	movl $0,PT_FS(%esp)
356 357 358 359 360
	jmp 1b
.section __ex_table,"a"
	.align 4
	.long 1b,2b
.popsection
361
ENDPROC(sysenter_entry)
L
Linus Torvalds 已提交
362 363 364

	# system call handler stub
ENTRY(system_call)
365
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
366
	pushl %eax			# save orig_eax
367
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
368 369
	SAVE_ALL
	GET_THREAD_INFO(%ebp)
370
	testl $TF_MASK,PT_EFLAGS(%esp)
371 372 373
	jz no_singlestep
	orl $_TIF_SINGLESTEP,TI_flags(%ebp)
no_singlestep:
374
					# system call tracing in operation / emulation
L
Linus Torvalds 已提交
375
	/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
376
	testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
L
Linus Torvalds 已提交
377 378 379 380 381
	jnz syscall_trace_entry
	cmpl $(nr_syscalls), %eax
	jae syscall_badsys
syscall_call:
	call *sys_call_table(,%eax,4)
382
	movl %eax,PT_EAX(%esp)		# store the return value
L
Linus Torvalds 已提交
383
syscall_exit:
384
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
385 386
					# setting need_resched or sigpending
					# between sampling and the iret
387
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
388 389 390 391 392
	movl TI_flags(%ebp), %ecx
	testw $_TIF_ALLWORK_MASK, %cx	# current->work
	jne syscall_exit_work

restore_all:
393 394
	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
395 396
	# are returning to the kernel.
	# See comments in process.c:copy_thread() for details.
397 398
	movb PT_OLDSS(%esp), %ah
	movb PT_CS(%esp), %al
399 400
	andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
401
	CFI_REMEMBER_STATE
L
Linus Torvalds 已提交
402 403
	je ldt_ss			# returning to user-space with LDT SS
restore_nocheck:
404 405
	TRACE_IRQS_IRET
restore_nocheck_notrace:
L
Linus Torvalds 已提交
406
	RESTORE_REGS
407
	addl $4, %esp			# skip orig_eax/error_code
408
	CFI_ADJUST_CFA_OFFSET -4
409
1:	INTERRUPT_RETURN
L
Linus Torvalds 已提交
410 411
.section .fixup,"ax"
iret_exc:
412
	TRACE_IRQS_ON
413
	ENABLE_INTERRUPTS(CLBR_NONE)
414 415 416
	pushl $0			# no error code
	pushl $do_iret_error
	jmp error_code
L
Linus Torvalds 已提交
417 418 419 420 421 422
.previous
.section __ex_table,"a"
	.align 4
	.long 1b,iret_exc
.previous

423
	CFI_RESTORE_STATE
L
Linus Torvalds 已提交
424
ldt_ss:
425
	larl PT_OLDSS(%esp), %eax
L
Linus Torvalds 已提交
426 427 428
	jnz restore_nocheck
	testl $0x00400000, %eax		# returning to 32bit stack?
	jnz restore_nocheck		# allright, normal return
429 430 431 432 433 434 435 436 437 438 439 440 441 442

#ifdef CONFIG_PARAVIRT
	/*
	 * The kernel can't run on a non-flat stack if paravirt mode
	 * is active.  Rather than try to fixup the high bits of
	 * ESP, bypass this code entirely.  This may break DOSemu
	 * and/or Wine support in a paravirt VM, although the option
	 * is still available to implement the setting of the high
	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
	 */
	cmpl $0, paravirt_ops+PARAVIRT_enabled
	jne restore_nocheck
#endif

L
Linus Torvalds 已提交
443 444 445 446 447 448
	/* If returning to userspace with 16bit stack,
	 * try to fix the higher word of ESP, as the CPU
	 * won't restore it.
	 * This is an "official" bug of all the x86-compatible
	 * CPUs, which we can try to work around to make
	 * dosemu and wine happy. */
449
	movl PT_OLDESP(%esp), %eax
S
Stas Sergeev 已提交
450 451 452 453 454 455
	movl %esp, %edx
	call patch_espfix_desc
	pushl $__ESPFIX_SS
	CFI_ADJUST_CFA_OFFSET 4
	pushl %eax
	CFI_ADJUST_CFA_OFFSET 4
456
	DISABLE_INTERRUPTS(CLBR_EAX)
457
	TRACE_IRQS_OFF
S
Stas Sergeev 已提交
458 459 460
	lss (%esp), %esp
	CFI_ADJUST_CFA_OFFSET -8
	jmp restore_nocheck
461
	CFI_ENDPROC
462
ENDPROC(system_call)
L
Linus Torvalds 已提交
463 464 465

	# perform work that needs to be done immediately before resumption
	ALIGN
466
	RING0_PTREGS_FRAME		# can't unwind into user space anyway
L
Linus Torvalds 已提交
467 468 469 470 471
work_pending:
	testb $_TIF_NEED_RESCHED, %cl
	jz work_notifysig
work_resched:
	call schedule
472
	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
L
Linus Torvalds 已提交
473 474
					# setting need_resched or sigpending
					# between sampling and the iret
475
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
476 477 478 479 480 481 482 483 484
	movl TI_flags(%ebp), %ecx
	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
					# than syscall tracing?
	jz restore_all
	testb $_TIF_NEED_RESCHED, %cl
	jnz work_resched

work_notifysig:				# deal with pending signals and
					# notify-resume requests
485
#ifdef CONFIG_VM86
486
	testl $VM_MASK, PT_EFLAGS(%esp)
L
Linus Torvalds 已提交
487 488 489 490 491
	movl %esp, %eax
	jne work_notifysig_v86		# returning to kernel-space or
					# vm86-space
	xorl %edx, %edx
	call do_notify_resume
492
	jmp resume_userspace_sig
L
Linus Torvalds 已提交
493 494 495 496

	ALIGN
work_notifysig_v86:
	pushl %ecx			# save ti_flags for do_notify_resume
497
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
498 499
	call save_v86_state		# %eax contains pt_regs pointer
	popl %ecx
500
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
501
	movl %eax, %esp
502 503 504
#else
	movl %esp, %eax
#endif
L
Linus Torvalds 已提交
505 506
	xorl %edx, %edx
	call do_notify_resume
507
	jmp resume_userspace_sig
508
END(work_pending)
L
Linus Torvalds 已提交
509 510 511 512

	# perform syscall exit tracing
	ALIGN
syscall_trace_entry:
513
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
514 515 516
	movl %esp, %eax
	xorl %edx,%edx
	call do_syscall_trace
517
	cmpl $0, %eax
518
	jne resume_userspace		# ret != 0 -> running under PTRACE_SYSEMU,
519
					# so must skip actual syscall
520
	movl PT_ORIG_EAX(%esp), %eax
L
Linus Torvalds 已提交
521 522 523
	cmpl $(nr_syscalls), %eax
	jnae syscall_call
	jmp syscall_exit
524
END(syscall_trace_entry)
L
Linus Torvalds 已提交
525 526 527 528 529 530

	# perform syscall exit tracing
	ALIGN
syscall_exit_work:
	testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
	jz work_pending
531
	TRACE_IRQS_ON
532
	ENABLE_INTERRUPTS(CLBR_ANY)	# could let do_syscall_trace() call
L
Linus Torvalds 已提交
533 534 535 536 537
					# schedule() instead
	movl %esp, %eax
	movl $1, %edx
	call do_syscall_trace
	jmp resume_userspace
538
END(syscall_exit_work)
539
	CFI_ENDPROC
L
Linus Torvalds 已提交
540

541
	RING0_INT_FRAME			# can't unwind into user space anyway
L
Linus Torvalds 已提交
542 543
syscall_fault:
	pushl %eax			# save orig_eax
544
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
545 546
	SAVE_ALL
	GET_THREAD_INFO(%ebp)
547
	movl $-EFAULT,PT_EAX(%esp)
L
Linus Torvalds 已提交
548
	jmp resume_userspace
549
END(syscall_fault)
L
Linus Torvalds 已提交
550 551

syscall_badsys:
552
	movl $-ENOSYS,PT_EAX(%esp)
L
Linus Torvalds 已提交
553
	jmp resume_userspace
554
END(syscall_badsys)
555
	CFI_ENDPROC
L
Linus Torvalds 已提交
556 557

#define FIXUP_ESPFIX_STACK \
S
Stas Sergeev 已提交
558
	/* since we are on a wrong stack, we cant make it a C code :( */ \
559
	movl %fs:PDA_cpu, %ebx; \
R
Rusty Russell 已提交
560
	PER_CPU(cpu_gdt, %ebx); \
S
Stas Sergeev 已提交
561 562 563 564
	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
	addl %esp, %eax; \
	pushl $__KERNEL_DS; \
	CFI_ADJUST_CFA_OFFSET 4; \
L
Linus Torvalds 已提交
565
	pushl %eax; \
566
	CFI_ADJUST_CFA_OFFSET 4; \
S
Stas Sergeev 已提交
567 568 569
	lss (%esp), %esp; \
	CFI_ADJUST_CFA_OFFSET -8;
#define UNWIND_ESPFIX_STACK \
L
Linus Torvalds 已提交
570
	movl %ss, %eax; \
S
Stas Sergeev 已提交
571
	/* see if on espfix stack */ \
L
Linus Torvalds 已提交
572
	cmpw $__ESPFIX_SS, %ax; \
S
Stas Sergeev 已提交
573 574
	jne 27f; \
	movl $__KERNEL_DS, %eax; \
575 576
	movl %eax, %ds; \
	movl %eax, %es; \
S
Stas Sergeev 已提交
577
	/* switch to normal stack */ \
578
	FIXUP_ESPFIX_STACK; \
S
Stas Sergeev 已提交
579
27:;
L
Linus Torvalds 已提交
580 581 582 583 584 585 586 587 588 589

/*
 * Build the entry stubs and pointer table with
 * some assembler magic.
 */
.data
ENTRY(interrupt)
.text

ENTRY(irq_entries_start)
590
	RING0_INT_FRAME
591
vector=0
L
Linus Torvalds 已提交
592 593
.rept NR_IRQS
	ALIGN
594 595 596
 .if vector
	CFI_ADJUST_CFA_OFFSET -4
 .endif
597
1:	pushl $~(vector)
598
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
599
	jmp common_interrupt
600
 .previous
L
Linus Torvalds 已提交
601
	.long 1b
602
 .text
L
Linus Torvalds 已提交
603 604
vector=vector+1
.endr
605 606 607 608 609
END(irq_entries_start)

.previous
END(interrupt)
.previous
L
Linus Torvalds 已提交
610

611 612 613 614
/*
 * the CPU automatically disables interrupts when executing an IRQ vector,
 * so IRQ-flags tracing has to follow that:
 */
L
Linus Torvalds 已提交
615 616 617
	ALIGN
common_interrupt:
	SAVE_ALL
618
	TRACE_IRQS_OFF
L
Linus Torvalds 已提交
619 620 621
	movl %esp,%eax
	call do_IRQ
	jmp ret_from_intr
622
ENDPROC(common_interrupt)
623
	CFI_ENDPROC
L
Linus Torvalds 已提交
624 625 626

#define BUILD_INTERRUPT(name, nr)	\
ENTRY(name)				\
627
	RING0_INT_FRAME;		\
628
	pushl $~(nr);			\
629 630
	CFI_ADJUST_CFA_OFFSET 4;	\
	SAVE_ALL;			\
631
	TRACE_IRQS_OFF			\
L
Linus Torvalds 已提交
632
	movl %esp,%eax;			\
633
	call smp_##name;		\
634
	jmp ret_from_intr;		\
635 636
	CFI_ENDPROC;			\
ENDPROC(name)
L
Linus Torvalds 已提交
637 638 639 640

/* The include is where all of the SMP etc. interrupts come from */
#include "entry_arch.h"

Z
Zachary Amsden 已提交
641 642 643 644 645
/* This alternate entry is needed because we hijack the apic LVTT */
#if defined(CONFIG_VMI) && defined(CONFIG_X86_LOCAL_APIC)
BUILD_INTERRUPT(apic_vmi_timer_interrupt,LOCAL_TIMER_VECTOR)
#endif

646 647 648
KPROBE_ENTRY(page_fault)
	RING0_EC_FRAME
	pushl $do_page_fault
649
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
650 651
	ALIGN
error_code:
652
	/* the function address is in %fs's slot on the stack */
653 654 655
	pushl %es
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET es, 0*/
L
Linus Torvalds 已提交
656
	pushl %ds
657 658
	CFI_ADJUST_CFA_OFFSET 4
	/*CFI_REL_OFFSET ds, 0*/
L
Linus Torvalds 已提交
659
	pushl %eax
660 661
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET eax, 0
L
Linus Torvalds 已提交
662
	pushl %ebp
663 664
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebp, 0
L
Linus Torvalds 已提交
665
	pushl %edi
666 667
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edi, 0
L
Linus Torvalds 已提交
668
	pushl %esi
669 670
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET esi, 0
L
Linus Torvalds 已提交
671
	pushl %edx
672 673
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET edx, 0
L
Linus Torvalds 已提交
674
	pushl %ecx
675 676
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ecx, 0
L
Linus Torvalds 已提交
677
	pushl %ebx
678 679
	CFI_ADJUST_CFA_OFFSET 4
	CFI_REL_OFFSET ebx, 0
L
Linus Torvalds 已提交
680
	cld
681
	pushl %fs
682
	CFI_ADJUST_CFA_OFFSET 4
683
	/*CFI_REL_OFFSET fs, 0*/
684
	movl $(__KERNEL_PDA), %ecx
685
	movl %ecx, %fs
L
Linus Torvalds 已提交
686 687
	UNWIND_ESPFIX_STACK
	popl %ecx
688 689
	CFI_ADJUST_CFA_OFFSET -4
	/*CFI_REGISTER es, ecx*/
690
	movl PT_FS(%esp), %edi		# get the function address
691
	movl PT_ORIG_EAX(%esp), %edx	# get the error code
692
	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
693 694
	mov  %ecx, PT_FS(%esp)
	/*CFI_REL_OFFSET fs, ES*/
L
Linus Torvalds 已提交
695 696 697 698 699 700
	movl $(__USER_DS), %ecx
	movl %ecx, %ds
	movl %ecx, %es
	movl %esp,%eax			# pt_regs pointer
	call *%edi
	jmp ret_from_exception
701
	CFI_ENDPROC
702
KPROBE_END(page_fault)
L
Linus Torvalds 已提交
703 704

ENTRY(coprocessor_error)
705
	RING0_INT_FRAME
L
Linus Torvalds 已提交
706
	pushl $0
707
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
708
	pushl $do_coprocessor_error
709
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
710
	jmp error_code
711
	CFI_ENDPROC
712
END(coprocessor_error)
L
Linus Torvalds 已提交
713 714

ENTRY(simd_coprocessor_error)
715
	RING0_INT_FRAME
L
Linus Torvalds 已提交
716
	pushl $0
717
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
718
	pushl $do_simd_coprocessor_error
719
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
720
	jmp error_code
721
	CFI_ENDPROC
722
END(simd_coprocessor_error)
L
Linus Torvalds 已提交
723 724

ENTRY(device_not_available)
725
	RING0_INT_FRAME
L
Linus Torvalds 已提交
726
	pushl $-1			# mark this as an int
727
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
728
	SAVE_ALL
729
	GET_CR0_INTO_EAX
L
Linus Torvalds 已提交
730 731
	testl $0x4, %eax		# EM (math emulation bit)
	jne device_not_available_emulate
732
	preempt_stop(CLBR_ANY)
L
Linus Torvalds 已提交
733 734 735 736
	call math_state_restore
	jmp ret_from_exception
device_not_available_emulate:
	pushl $0			# temporary storage for ORIG_EIP
737
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
738 739
	call math_emulate
	addl $4, %esp
740
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
741
	jmp ret_from_exception
742
	CFI_ENDPROC
743
END(device_not_available)
L
Linus Torvalds 已提交
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762

/*
 * Debug traps and NMI can happen at the one SYSENTER instruction
 * that sets up the real kernel stack. Check here, since we can't
 * allow the wrong stack to be used.
 *
 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
 * already pushed 3 words if it hits on the sysenter instruction:
 * eflags, cs and eip.
 *
 * We just load the right stack, and push the three (known) values
 * by hand onto the new stack - while updating the return eip past
 * the instruction that would have done it for sysenter.
 */
#define FIX_STACK(offset, ok, label)		\
	cmpw $__KERNEL_CS,4(%esp);		\
	jne ok;					\
label:						\
	movl TSS_sysenter_esp0+offset(%esp),%esp;	\
763 764
	CFI_DEF_CFA esp, 0;			\
	CFI_UNDEFINED eip;			\
L
Linus Torvalds 已提交
765
	pushfl;					\
766
	CFI_ADJUST_CFA_OFFSET 4;		\
L
Linus Torvalds 已提交
767
	pushl $__KERNEL_CS;			\
768 769 770 771
	CFI_ADJUST_CFA_OFFSET 4;		\
	pushl $sysenter_past_esp;		\
	CFI_ADJUST_CFA_OFFSET 4;		\
	CFI_REL_OFFSET eip, 0
L
Linus Torvalds 已提交
772

773
KPROBE_ENTRY(debug)
774
	RING0_INT_FRAME
L
Linus Torvalds 已提交
775 776 777 778 779
	cmpl $sysenter_entry,(%esp)
	jne debug_stack_correct
	FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
debug_stack_correct:
	pushl $-1			# mark this as an int
780
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
781 782 783 784 785
	SAVE_ALL
	xorl %edx,%edx			# error code 0
	movl %esp,%eax			# pt_regs pointer
	call do_debug
	jmp ret_from_exception
786
	CFI_ENDPROC
787 788
KPROBE_END(debug)

L
Linus Torvalds 已提交
789 790 791 792 793 794 795 796
/*
 * NMI is doubly nasty. It can happen _while_ we're handling
 * a debug fault, and the debug fault hasn't yet been able to
 * clear up the stack. So we first check whether we got  an
 * NMI on the sysenter entry path, but after that we need to
 * check whether we got an NMI on the debug path where the debug
 * fault happened on the sysenter path.
 */
797
KPROBE_ENTRY(nmi)
798
	RING0_INT_FRAME
L
Linus Torvalds 已提交
799
	pushl %eax
800
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
801 802 803
	movl %ss, %eax
	cmpw $__ESPFIX_SS, %ax
	popl %eax
804
	CFI_ADJUST_CFA_OFFSET -4
S
Stas Sergeev 已提交
805
	je nmi_espfix_stack
L
Linus Torvalds 已提交
806 807 808
	cmpl $sysenter_entry,(%esp)
	je nmi_stack_fixup
	pushl %eax
809
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
810 811 812 813 814 815 816
	movl %esp,%eax
	/* Do not access memory above the end of our stack page,
	 * it might not exist.
	 */
	andl $(THREAD_SIZE-1),%eax
	cmpl $(THREAD_SIZE-20),%eax
	popl %eax
817
	CFI_ADJUST_CFA_OFFSET -4
L
Linus Torvalds 已提交
818 819 820 821
	jae nmi_stack_correct
	cmpl $sysenter_entry,12(%esp)
	je nmi_debug_stack_check
nmi_stack_correct:
822
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
823
	pushl %eax
824
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
825 826 827 828
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_nmi
829
	jmp restore_nocheck_notrace
830
	CFI_ENDPROC
L
Linus Torvalds 已提交
831 832

nmi_stack_fixup:
833
	RING0_INT_FRAME
L
Linus Torvalds 已提交
834 835
	FIX_STACK(12,nmi_stack_correct, 1)
	jmp nmi_stack_correct
836

L
Linus Torvalds 已提交
837
nmi_debug_stack_check:
838
	/* We have a RING0_INT_FRAME here */
L
Linus Torvalds 已提交
839 840
	cmpw $__KERNEL_CS,16(%esp)
	jne nmi_stack_correct
841 842
	cmpl $debug,(%esp)
	jb nmi_stack_correct
L
Linus Torvalds 已提交
843
	cmpl $debug_esp_fix_insn,(%esp)
844
	ja nmi_stack_correct
L
Linus Torvalds 已提交
845 846 847
	FIX_STACK(24,nmi_stack_correct, 1)
	jmp nmi_stack_correct

S
Stas Sergeev 已提交
848
nmi_espfix_stack:
849 850 851 852
	/* We have a RING0_INT_FRAME here.
	 *
	 * create the pointer to lss back
	 */
L
Linus Torvalds 已提交
853
	pushl %ss
854
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
855
	pushl %esp
856
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
857 858 859 860
	addw $4, (%esp)
	/* copy the iret frame of 12 bytes */
	.rept 3
	pushl 16(%esp)
861
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
862 863
	.endr
	pushl %eax
864
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
865 866 867 868 869
	SAVE_ALL
	FIXUP_ESPFIX_STACK		# %eax == %esp
	xorl %edx,%edx			# zero error code
	call do_nmi
	RESTORE_REGS
S
Stas Sergeev 已提交
870 871
	lss 12+4(%esp), %esp		# back to espfix stack
	CFI_ADJUST_CFA_OFFSET -24
872
1:	INTERRUPT_RETURN
873
	CFI_ENDPROC
L
Linus Torvalds 已提交
874 875 876 877
.section __ex_table,"a"
	.align 4
	.long 1b,iret_exc
.previous
878
KPROBE_END(nmi)
L
Linus Torvalds 已提交
879

880 881 882 883 884 885 886
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
1:	iret
.section __ex_table,"a"
	.align 4
	.long 1b,iret_exc
.previous
887
END(native_iret)
888 889 890 891

ENTRY(native_irq_enable_sysexit)
	sti
	sysexit
892
END(native_irq_enable_sysexit)
893 894
#endif

895
KPROBE_ENTRY(int3)
896
	RING0_INT_FRAME
L
Linus Torvalds 已提交
897
	pushl $-1			# mark this as an int
898
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
899 900 901 902 903
	SAVE_ALL
	xorl %edx,%edx		# zero error code
	movl %esp,%eax		# pt_regs pointer
	call do_int3
	jmp ret_from_exception
904
	CFI_ENDPROC
905
KPROBE_END(int3)
L
Linus Torvalds 已提交
906 907

ENTRY(overflow)
908
	RING0_INT_FRAME
L
Linus Torvalds 已提交
909
	pushl $0
910
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
911
	pushl $do_overflow
912
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
913
	jmp error_code
914
	CFI_ENDPROC
915
END(overflow)
L
Linus Torvalds 已提交
916 917

ENTRY(bounds)
918
	RING0_INT_FRAME
L
Linus Torvalds 已提交
919
	pushl $0
920
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
921
	pushl $do_bounds
922
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
923
	jmp error_code
924
	CFI_ENDPROC
925
END(bounds)
L
Linus Torvalds 已提交
926 927

ENTRY(invalid_op)
928
	RING0_INT_FRAME
L
Linus Torvalds 已提交
929
	pushl $0
930
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
931
	pushl $do_invalid_op
932
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
933
	jmp error_code
934
	CFI_ENDPROC
935
END(invalid_op)
L
Linus Torvalds 已提交
936 937

ENTRY(coprocessor_segment_overrun)
938
	RING0_INT_FRAME
L
Linus Torvalds 已提交
939
	pushl $0
940
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
941
	pushl $do_coprocessor_segment_overrun
942
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
943
	jmp error_code
944
	CFI_ENDPROC
945
END(coprocessor_segment_overrun)
L
Linus Torvalds 已提交
946 947

ENTRY(invalid_TSS)
948
	RING0_EC_FRAME
L
Linus Torvalds 已提交
949
	pushl $do_invalid_TSS
950
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
951
	jmp error_code
952
	CFI_ENDPROC
953
END(invalid_TSS)
L
Linus Torvalds 已提交
954 955

ENTRY(segment_not_present)
956
	RING0_EC_FRAME
L
Linus Torvalds 已提交
957
	pushl $do_segment_not_present
958
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
959
	jmp error_code
960
	CFI_ENDPROC
961
END(segment_not_present)
L
Linus Torvalds 已提交
962 963

ENTRY(stack_segment)
964
	RING0_EC_FRAME
L
Linus Torvalds 已提交
965
	pushl $do_stack_segment
966
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
967
	jmp error_code
968
	CFI_ENDPROC
969
END(stack_segment)
L
Linus Torvalds 已提交
970

971
KPROBE_ENTRY(general_protection)
972
	RING0_EC_FRAME
L
Linus Torvalds 已提交
973
	pushl $do_general_protection
974
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
975
	jmp error_code
976
	CFI_ENDPROC
977
KPROBE_END(general_protection)
L
Linus Torvalds 已提交
978 979

ENTRY(alignment_check)
980
	RING0_EC_FRAME
L
Linus Torvalds 已提交
981
	pushl $do_alignment_check
982
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
983
	jmp error_code
984
	CFI_ENDPROC
985
END(alignment_check)
L
Linus Torvalds 已提交
986

987 988 989 990 991
ENTRY(divide_error)
	RING0_INT_FRAME
	pushl $0			# no error code
	CFI_ADJUST_CFA_OFFSET 4
	pushl $do_divide_error
992
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
993
	jmp error_code
994
	CFI_ENDPROC
995
END(divide_error)
L
Linus Torvalds 已提交
996 997 998

#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
999
	RING0_INT_FRAME
L
Linus Torvalds 已提交
1000
	pushl $0
1001
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1002
	pushl machine_check_vector
1003
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1004
	jmp error_code
1005
	CFI_ENDPROC
1006
END(machine_check)
L
Linus Torvalds 已提交
1007 1008 1009
#endif

ENTRY(spurious_interrupt_bug)
1010
	RING0_INT_FRAME
L
Linus Torvalds 已提交
1011
	pushl $0
1012
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1013
	pushl $do_spurious_interrupt_bug
1014
	CFI_ADJUST_CFA_OFFSET 4
L
Linus Torvalds 已提交
1015
	jmp error_code
1016
	CFI_ENDPROC
1017
END(spurious_interrupt_bug)
L
Linus Torvalds 已提交
1018

1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
ENTRY(kernel_thread_helper)
	pushl $0		# fake return address for unwinder
	CFI_STARTPROC
	movl %edx,%eax
	push %edx
	CFI_ADJUST_CFA_OFFSET 4
	call *%ebx
	push %eax
	CFI_ADJUST_CFA_OFFSET 4
	call do_exit
	CFI_ENDPROC
ENDPROC(kernel_thread_helper)

1032
.section .rodata,"a"
1033
#include "syscall_table.S"
L
Linus Torvalds 已提交
1034 1035

syscall_table_size=(.-sys_call_table)