entry_64_compat.S 11.4 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Compatibility mode system call entry point for x86-64.
 *
L
Linus Torvalds 已提交
4
 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
5
 */
6
#include "calling.h"
7
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
8 9
#include <asm/current.h>
#include <asm/errno.h>
10 11
#include <asm/ia32_unistd.h>
#include <asm/thread_info.h>
L
Linus Torvalds 已提交
12
#include <asm/segment.h>
13
#include <asm/irqflags.h>
14
#include <asm/asm.h>
15
#include <asm/smap.h>
L
Linus Torvalds 已提交
16
#include <linux/linkage.h>
17
#include <linux/err.h>
L
Linus Torvalds 已提交
18

J
Jiri Olsa 已提交
19 20
	.section .entry.text, "ax"

L
Linus Torvalds 已提交
21
/*
22
 * 32-bit SYSENTER entry.
L
Linus Torvalds 已提交
23
 *
24 25 26 27 28 29 30 31 32 33 34
 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
 * on 64-bit kernels running on Intel CPUs.
 *
 * The SYSENTER instruction, in principle, should *only* occur in the
 * vDSO.  In practice, a small number of Android devices were shipped
 * with a copy of Bionic that inlined a SYSENTER instruction.  This
 * never happened in any of Google's Bionic versions -- it only happened
 * in a narrow range of Intel-provided versions.
 *
 * SYSENTER loads SS, RSP, CS, and RIP from previously programmed MSRs.
 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
35
 * SYSENTER does not save anything on the stack,
36
 * and does not save old RIP (!!!), RSP, or RFLAGS.
37
 *
L
Linus Torvalds 已提交
38
 * Arguments:
39 40 41 42 43 44 45 46 47
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * ebp  user stack
 * 0(%ebp) arg6
 */
48
ENTRY(entry_SYSENTER_compat)
49
	/* Interrupts are off on entry. */
50
	SWAPGS_UNSAFE_STACK
51
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
52

53 54 55 56 57 58 59
	/*
	 * User tracing code (ptrace or signal handlers) might assume that
	 * the saved RAX contains a 32-bit number when we're invoking a 32-bit
	 * syscall.  Just in case the high bits are nonzero, zero-extend
	 * the syscall number.  (This could almost certainly be deleted
	 * with no ill effects.)
	 */
60 61
	movl	%eax, %eax

62
	/* Construct struct pt_regs on stack */
63
	pushq	$__USER32_DS		/* pt_regs->ss */
64
	pushq	%rbp			/* pt_regs->sp (stashed in bp) */
65 66 67 68 69 70 71 72 73

	/*
	 * Push flags.  This is nasty.  First, interrupts are currently
	 * off, but we need pt_regs->flags to have IF set.  Second, even
	 * if TF was set when SYSENTER started, it's clear by now.  We fix
	 * that later using TIF_SINGLESTEP.
	 */
	pushfq				/* pt_regs->flags (except IF = 0) */
	orl	$X86_EFLAGS_IF, (%rsp)	/* Fix saved flags */
74
	pushq	$__USER32_CS		/* pt_regs->cs */
75
	pushq	$0			/* pt_regs->ip = 0 (placeholder) */
76 77 78 79
	pushq	%rax			/* pt_regs->orig_ax */
	pushq	%rdi			/* pt_regs->di */
	pushq	%rsi			/* pt_regs->si */
	pushq	%rdx			/* pt_regs->dx */
80
	pushq	%rcx			/* pt_regs->cx */
81
	pushq	$-ENOSYS		/* pt_regs->ax */
82 83 84 85
	pushq   $0			/* pt_regs->r8  = 0 */
	pushq   $0			/* pt_regs->r9  = 0 */
	pushq   $0			/* pt_regs->r10 = 0 */
	pushq   $0			/* pt_regs->r11 = 0 */
86
	pushq   %rbx                    /* pt_regs->rbx */
87
	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */
88 89 90 91
	pushq   $0			/* pt_regs->r12 = 0 */
	pushq   $0			/* pt_regs->r13 = 0 */
	pushq   $0			/* pt_regs->r14 = 0 */
	pushq   $0			/* pt_regs->r15 = 0 */
L
Linus Torvalds 已提交
92
	cld
93

94
	/*
95
	 * SYSENTER doesn't filter flags, so we need to clear NT and AC
96
	 * ourselves.  To save a few cycles, we can check whether
97
	 * either was set instead of doing an unconditional popfq.
98 99
	 * This needs to happen before enabling interrupts so that
	 * we don't get preempted with NT set.
100
	 *
101 102 103 104 105 106
	 * If TF is set, we will single-step all the way to here -- do_debug
	 * will ignore all the traps.  (Yes, this is slow, but so is
	 * single-stepping in general.  This allows us to avoid having
	 * a more complicated code to handle the case where a user program
	 * forces us to single-step through the SYSENTER entry code.)
	 *
107
	 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
108 109 110 111
	 * out-of-line as an optimization: NT is unlikely to be set in the
	 * majority of the cases and instead of polluting the I$ unnecessarily,
	 * we're keeping that code behind a branch which will predict as
	 * not-taken and therefore its instructions won't be fetched.
112
	 */
113
	testl	$X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, EFLAGS(%rsp)
114 115
	jnz	.Lsysenter_fix_flags
.Lsysenter_flags_fixed:
116

117 118 119 120 121
	/*
	 * User mode is traced as though IRQs are on, and SYSENTER
	 * turned them off.
	 */
	TRACE_IRQS_OFF
122

123 124
	movq	%rsp, %rdi
	call	do_fast_syscall_32
125 126 127
	/* XEN PV guests always use IRET path */
	ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
		    "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
128
	jmp	sysret32_from_system_call
L
Linus Torvalds 已提交
129

130
.Lsysenter_fix_flags:
131
	pushq	$X86_EFLAGS_FIXED
132
	popfq
133
	jmp	.Lsysenter_flags_fixed
134
GLOBAL(__end_entry_SYSENTER_compat)
135
ENDPROC(entry_SYSENTER_compat)
L
Linus Torvalds 已提交
136 137

/*
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
 * 32-bit SYSCALL entry.
 *
 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
 * on 64-bit kernels running on AMD CPUs.
 *
 * The SYSCALL instruction, in principle, should *only* occur in the
 * vDSO.  In practice, it appears that this really is the case.
 * As evidence:
 *
 *  - The calling convention for SYSCALL has changed several times without
 *    anyone noticing.
 *
 *  - Prior to the in-kernel X86_BUG_SYSRET_SS_ATTRS fixup, anything
 *    user task that did SYSCALL without immediately reloading SS
 *    would randomly crash.
L
Linus Torvalds 已提交
153
 *
154 155 156 157 158 159
 *  - Most programmers do not directly target AMD CPUs, and the 32-bit
 *    SYSCALL instruction does not exist on Intel CPUs.  Even on AMD
 *    CPUs, Linux disables the SYSCALL instruction on 32-bit kernels
 *    because the SYSCALL instruction in legacy/native 32-bit mode (as
 *    opposed to compat mode) is sufficiently poorly designed as to be
 *    essentially unusable.
160
 *
161 162 163 164 165 166 167
 * 32-bit SYSCALL saves RIP to RCX, clears RFLAGS.RF, then saves
 * RFLAGS to R11, then loads new SS, CS, and RIP from previously
 * programmed MSRs.  RFLAGS gets masked by a value from another MSR
 * (so CLD and CLAC are not needed).  SYSCALL does not save anything on
 * the stack and does not change RSP.
 *
 * Note: RFLAGS saving+masking-with-MSR happens only in Long mode
168
 * (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it).
169
 * Don't get confused: RFLAGS saving+masking depends on Long Mode Active bit
170 171 172
 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
 *
L
Linus Torvalds 已提交
173
 * Arguments:
174 175 176 177 178 179 180 181 182 183
 * eax  system call number
 * ecx  return address
 * ebx  arg1
 * ebp  arg2	(note: not saved in the stack frame, should not be touched)
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * esp  user stack
 * 0(%esp) arg6
 */
184
ENTRY(entry_SYSCALL_compat)
185
	/* Interrupts are off on entry. */
186
	swapgs
187

188
	/* Stash user ESP and switch to the kernel stack. */
189 190
	movl	%esp, %r8d
	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
191

192
	/* Construct struct pt_regs on stack */
193 194 195 196 197
	pushq	$__USER32_DS		/* pt_regs->ss */
	pushq	%r8			/* pt_regs->sp */
	pushq	%r11			/* pt_regs->flags */
	pushq	$__USER32_CS		/* pt_regs->cs */
	pushq	%rcx			/* pt_regs->ip */
198 199
GLOBAL(entry_SYSCALL_compat_after_hwframe)
	movl	%eax, %eax		/* discard orig_ax high bits */
200 201 202 203
	pushq	%rax			/* pt_regs->orig_ax */
	pushq	%rdi			/* pt_regs->di */
	pushq	%rsi			/* pt_regs->si */
	pushq	%rdx			/* pt_regs->dx */
204
	pushq	%rbp			/* pt_regs->cx (stashed in bp) */
205
	pushq	$-ENOSYS		/* pt_regs->ax */
206 207 208 209
	pushq   $0			/* pt_regs->r8  = 0 */
	pushq   $0			/* pt_regs->r9  = 0 */
	pushq   $0			/* pt_regs->r10 = 0 */
	pushq   $0			/* pt_regs->r11 = 0 */
210
	pushq   %rbx                    /* pt_regs->rbx */
211
	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */
212 213 214 215
	pushq   $0			/* pt_regs->r12 = 0 */
	pushq   $0			/* pt_regs->r13 = 0 */
	pushq   $0			/* pt_regs->r14 = 0 */
	pushq   $0			/* pt_regs->r15 = 0 */
216

217 218 219 220 221 222 223 224
	/*
	 * User mode is traced as though IRQs are on, and SYSENTER
	 * turned them off.
	 */
	TRACE_IRQS_OFF

	movq	%rsp, %rdi
	call	do_fast_syscall_32
225 226 227
	/* XEN PV guests always use IRET path */
	ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
		    "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261

	/* Opportunistic SYSRET */
sysret32_from_system_call:
	TRACE_IRQS_ON			/* User mode traces as IRQs on. */
	movq	RBX(%rsp), %rbx		/* pt_regs->rbx */
	movq	RBP(%rsp), %rbp		/* pt_regs->rbp */
	movq	EFLAGS(%rsp), %r11	/* pt_regs->flags (in r11) */
	movq	RIP(%rsp), %rcx		/* pt_regs->ip (in rcx) */
	addq	$RAX, %rsp		/* Skip r8-r15 */
	popq	%rax			/* pt_regs->rax */
	popq	%rdx			/* Skip pt_regs->cx */
	popq	%rdx			/* pt_regs->dx */
	popq	%rsi			/* pt_regs->si */
	popq	%rdi			/* pt_regs->di */

        /*
         * USERGS_SYSRET32 does:
         *  GSBASE = user's GS base
         *  EIP = ECX
         *  RFLAGS = R11
         *  CS = __USER32_CS
         *  SS = __USER_DS
         *
	 * ECX will not match pt_regs->cx, but we're returning to a vDSO
	 * trampoline that will fix up RCX, so this is okay.
	 *
	 * R12-R15 are callee-saved, so they contain whatever was in them
	 * when the system call started, which is already known to user
	 * code.  We zero R8-R10 to avoid info leaks.
         */
	xorq	%r8, %r8
	xorq	%r9, %r9
	xorq	%r10, %r10
	movq	RSP-ORIG_RAX(%rsp), %rsp
262 263
	swapgs
	sysretl
264
END(entry_SYSCALL_compat)
265

266
/*
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
 * 32-bit legacy system call entry.
 *
 * 32-bit x86 Linux system calls traditionally used the INT $0x80
 * instruction.  INT $0x80 lands here.
 *
 * This entry point can be used by 32-bit and 64-bit programs to perform
 * 32-bit system calls.  Instances of INT $0x80 can be found inline in
 * various programs and libraries.  It is also used by the vDSO's
 * __kernel_vsyscall fallback for hardware that doesn't support a faster
 * entry method.  Restarted 32-bit system calls also fall back to INT
 * $0x80 regardless of what instruction was originally used to do the
 * system call.
 *
 * This is considered a slow path.  It is not used by most libc
 * implementations on modern hardware except during process startup.
L
Linus Torvalds 已提交
282
 *
283 284 285 286 287 288 289
 * Arguments:
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
290
 * ebp  arg6
291
 */
292
ENTRY(entry_INT80_compat)
293
	/*
294
	 * Interrupts are off on entry.
295
	 */
296
	PARAVIRT_ADJUST_EXCEPTION_FRAME
297
	ASM_CLAC			/* Do this early to minimize exposure */
298 299
	SWAPGS

300 301 302 303 304 305 306
	/*
	 * User tracing code (ptrace or signal handlers) might assume that
	 * the saved RAX contains a 32-bit number when we're invoking a 32-bit
	 * syscall.  Just in case the high bits are nonzero, zero-extend
	 * the syscall number.  (This could almost certainly be deleted
	 * with no ill effects.)
	 */
307
	movl	%eax, %eax
308

309
	/* Construct struct pt_regs on stack (iret frame is already on stack) */
310 311 312 313 314 315
	pushq	%rax			/* pt_regs->orig_ax */
	pushq	%rdi			/* pt_regs->di */
	pushq	%rsi			/* pt_regs->si */
	pushq	%rdx			/* pt_regs->dx */
	pushq	%rcx			/* pt_regs->cx */
	pushq	$-ENOSYS		/* pt_regs->ax */
316 317 318 319
	pushq   $0			/* pt_regs->r8  = 0 */
	pushq   $0			/* pt_regs->r9  = 0 */
	pushq   $0			/* pt_regs->r10 = 0 */
	pushq   $0			/* pt_regs->r11 = 0 */
320 321 322 323 324 325
	pushq   %rbx                    /* pt_regs->rbx */
	pushq   %rbp                    /* pt_regs->rbp */
	pushq   %r12                    /* pt_regs->r12 */
	pushq   %r13                    /* pt_regs->r13 */
	pushq   %r14                    /* pt_regs->r14 */
	pushq   %r15                    /* pt_regs->r15 */
L
Linus Torvalds 已提交
326
	cld
327

328
	/*
329 330
	 * User mode is traced as though IRQs are on, and the interrupt
	 * gate turned them off.
331
	 */
332 333 334
	TRACE_IRQS_OFF

	movq	%rsp, %rdi
335
	call	do_int80_syscall_32
336
.Lsyscall_32_done:
337 338 339 340 341

	/* Go back to user mode. */
	TRACE_IRQS_ON
	SWAPGS
	jmp	restore_regs_and_iret
342
END(entry_INT80_compat)
L
Linus Torvalds 已提交
343

344 345
	ALIGN
GLOBAL(stub32_clone)
346
	/*
347 348 349 350 351
	 * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr).
	 * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val).
	 *
	 * The native 64-bit kernel's sys_clone() implements the latter,
	 * so we need to swap arguments here before calling it:
352
	 */
353
	xchg	%r8, %rcx
354
	jmp	sys_clone