uaccess.h 17.3 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_UACCESS_H
#define _ASM_X86_UACCESS_H
3 4 5 6 7 8 9 10 11
/*
 * User space memory access functions
 */
#include <linux/errno.h>
#include <linux/compiler.h>
#include <linux/thread_info.h>
#include <linux/string.h>
#include <asm/asm.h>
#include <asm/page.h>
12
#include <asm/smap.h>
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27

#define VERIFY_READ 0
#define VERIFY_WRITE 1

/*
 * The fs value determines whether argument validity checking should be
 * performed or not.  If get_fs() == USER_DS, checking is performed, with
 * get_fs() == KERNEL_DS, checking is bypassed.
 *
 * For historical reasons, these macros are grossly misnamed.
 */

#define MAKE_MM_SEG(s)	((mm_segment_t) { (s) })

#define KERNEL_DS	MAKE_MM_SEG(-1UL)
28
#define USER_DS 	MAKE_MM_SEG(TASK_SIZE_MAX)
29 30 31 32 33 34 35

#define get_ds()	(KERNEL_DS)
#define get_fs()	(current_thread_info()->addr_limit)
#define set_fs(x)	(current_thread_info()->addr_limit = (x))

#define segment_eq(a, b)	((a).seg == (b).seg)

36
#define user_addr_max() (current_thread_info()->addr_limit.seg)
37 38
#define __addr_ok(addr) 	\
	((unsigned long __force)(addr) < user_addr_max())
39

40 41 42 43 44
/*
 * Test whether a block of memory is a valid user space address.
 * Returns 0 if the range is valid, nonzero otherwise.
 *
 * This is equivalent to the following test:
45
 * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
46 47 48 49
 *
 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
 */

50
#define __range_not_ok(addr, size, limit)				\
51 52 53 54 55 56
({									\
	unsigned long flag, roksum;					\
	__chk_user_ptr(addr);						\
	asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0"		\
	    : "=&r" (flag), "=r" (roksum)				\
	    : "1" (addr), "g" ((long)(size)),				\
57
	      "rm" (limit));						\
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
	flag;								\
})

/**
 * access_ok: - Checks if a user space pointer is valid
 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 *        to write to a block, it is always safe to read from it.
 * @addr: User space pointer to start of block to check
 * @size: Size of block to check
 *
 * Context: User context only.  This function may sleep.
 *
 * Checks if a pointer to a block of memory in user space is valid.
 *
 * Returns true (nonzero) if the memory block may be valid, false (zero)
 * if it is definitely invalid.
 *
 * Note that, depending on architecture, this function probably just
 * checks that the pointer is in the user space range - after calling
 * this function, memory access functions may still return -EFAULT.
 */
80 81
#define access_ok(type, addr, size) \
	(likely(__range_not_ok(addr, size, user_addr_max()) == 0))
82 83

/*
84 85 86 87 88 89
 * The exception table consists of pairs of addresses relative to the
 * exception table enty itself: the first is the address of an
 * instruction that is allowed to fault, and the second is the address
 * at which the program should continue.  No registers are modified,
 * so it is entirely up to the continuation code to figure out what to
 * do.
90 91 92 93 94 95 96 97
 *
 * All the routines below use bits of fixup code that are out of line
 * with the main instruction path.  This means when everything is well,
 * we don't even have to jump over them.  Further, they do not intrude
 * on our cache or tlb entries.
 */

struct exception_table_entry {
98
	int insn, fixup;
99
};
100 101 102
/* This is not the generic standard exception_table_entry format */
#define ARCH_HAS_SORT_EXTABLE
#define ARCH_HAS_SEARCH_EXTABLE
103 104

extern int fixup_exception(struct pt_regs *regs);
105
extern int early_fixup_exception(unsigned long *ip);
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129

/*
 * These are the main single-value transfer routines.  They automatically
 * use the right size if we just have the right pointer type.
 *
 * This gets kind of ugly. We want to return _two_ values in "get_user()"
 * and yet we don't want to do any pointers, because that is too much
 * of a performance impact. Thus we have a few rather ugly macros here,
 * and hide all the ugliness from the user.
 *
 * The "__xxx" versions of the user access functions are versions that
 * do not verify the address space, that must have been done previously
 * with a separate "access_ok()" call (this is used when we do multiple
 * accesses to the same area of user memory).
 */

extern int __get_user_1(void);
extern int __get_user_2(void);
extern int __get_user_4(void);
extern int __get_user_8(void);
extern int __get_user_bad(void);

#define __get_user_x(size, ret, x, ptr)		      \
	asm volatile("call __get_user_" #size	      \
130
		     : "=a" (ret), "=d" (x)	      \
131 132
		     : "0" (ptr))		      \

G
Glauber Costa 已提交
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
/* Careful: we have to cast the result to the type of the pointer
 * for sign reasons */

/**
 * get_user: - Get a simple variable from user space.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
 * Context: User context only.  This function may sleep.
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Returns zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
 */
#ifdef CONFIG_X86_32
#define __get_user_8(__ret_gu, __val_gu, ptr)				\
		__get_user_x(X, __ret_gu, __val_gu, ptr)
#else
#define __get_user_8(__ret_gu, __val_gu, ptr)				\
		__get_user_x(8, __ret_gu, __val_gu, ptr)
#endif

#define get_user(x, ptr)						\
({									\
	int __ret_gu;							\
	unsigned long __val_gu;						\
	__chk_user_ptr(ptr);						\
166
	might_fault();							\
G
Glauber Costa 已提交
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	switch (sizeof(*(ptr))) {					\
	case 1:								\
		__get_user_x(1, __ret_gu, __val_gu, ptr);		\
		break;							\
	case 2:								\
		__get_user_x(2, __ret_gu, __val_gu, ptr);		\
		break;							\
	case 4:								\
		__get_user_x(4, __ret_gu, __val_gu, ptr);		\
		break;							\
	case 8:								\
		__get_user_8(__ret_gu, __val_gu, ptr);			\
		break;							\
	default:							\
		__get_user_x(X, __ret_gu, __val_gu, ptr);		\
		break;							\
	}								\
	(x) = (__typeof__(*(ptr)))__val_gu;				\
	__ret_gu;							\
})

G
Glauber Costa 已提交
188 189
#define __put_user_x(size, x, ptr, __ret_pu)			\
	asm volatile("call __put_user_" #size : "=a" (__ret_pu)	\
190
		     : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
G
Glauber Costa 已提交
191 192 193



194
#ifdef CONFIG_X86_32
195
#define __put_user_asm_u64(x, addr, err, errret)			\
196 197
	asm volatile(ASM_STAC "\n"					\
		     "1:	movl %%eax,0(%2)\n"			\
198
		     "2:	movl %%edx,4(%2)\n"			\
199
		     "3: " ASM_CLAC "\n"				\
200 201 202 203 204 205 206
		     ".section .fixup,\"ax\"\n"				\
		     "4:	movl %3,%0\n"				\
		     "	jmp 3b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 4b)				\
		     _ASM_EXTABLE(2b, 4b)				\
		     : "=r" (err)					\
207
		     : "A" (x), "r" (addr), "i" (errret), "0" (err))
G
Glauber Costa 已提交
208

209
#define __put_user_asm_ex_u64(x, addr)					\
210 211
	asm volatile(ASM_STAC "\n"					\
		     "1:	movl %%eax,0(%1)\n"			\
212
		     "2:	movl %%edx,4(%1)\n"			\
213
		     "3: " ASM_CLAC "\n"				\
214 215
		     _ASM_EXTABLE_EX(1b, 2b)				\
		     _ASM_EXTABLE_EX(2b, 3b)				\
216 217
		     : : "A" (x), "r" (addr))

G
Glauber Costa 已提交
218 219 220
#define __put_user_x8(x, ptr, __ret_pu)				\
	asm volatile("call __put_user_8" : "=a" (__ret_pu)	\
		     : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
221
#else
222
#define __put_user_asm_u64(x, ptr, retval, errret) \
223
	__put_user_asm(x, ptr, retval, "q", "", "er", errret)
224
#define __put_user_asm_ex_u64(x, addr)	\
225
	__put_user_asm_ex(x, addr, "q", "", "er")
G
Glauber Costa 已提交
226
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
227 228
#endif

G
Glauber Costa 已提交
229 230 231 232 233 234 235 236 237 238 239
extern void __put_user_bad(void);

/*
 * Strange magic calling convention: pointer in %ecx,
 * value in %eax(:%edx), return value in %eax. clobbers %rbx
 */
extern void __put_user_1(void);
extern void __put_user_2(void);
extern void __put_user_4(void);
extern void __put_user_8(void);

240 241
#ifdef CONFIG_X86_WP_WORKS_OK

G
Glauber Costa 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
/**
 * put_user: - Write a simple value into user space.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
 * Context: User context only.  This function may sleep.
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Returns zero on success, or -EFAULT on error.
 */
#define put_user(x, ptr)					\
({								\
	int __ret_pu;						\
	__typeof__(*(ptr)) __pu_val;				\
	__chk_user_ptr(ptr);					\
263
	might_fault();						\
G
Glauber Costa 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
	__pu_val = x;						\
	switch (sizeof(*(ptr))) {				\
	case 1:							\
		__put_user_x(1, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 2:							\
		__put_user_x(2, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 4:							\
		__put_user_x(4, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 8:							\
		__put_user_x8(__pu_val, ptr, __ret_pu);		\
		break;						\
	default:						\
		__put_user_x(X, __pu_val, ptr, __ret_pu);	\
		break;						\
	}							\
	__ret_pu;						\
})

285 286 287 288 289 290 291 292 293 294 295 296
#define __put_user_size(x, ptr, size, retval, errret)			\
do {									\
	retval = 0;							\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__put_user_asm(x, ptr, retval, "b", "b", "iq", errret);	\
		break;							\
	case 2:								\
		__put_user_asm(x, ptr, retval, "w", "w", "ir", errret);	\
		break;							\
	case 4:								\
297
		__put_user_asm(x, ptr, retval, "l", "k", "ir", errret);	\
298 299
		break;							\
	case 8:								\
300 301
		__put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval,	\
				   errret);				\
302 303 304 305 306 307
		break;							\
	default:							\
		__put_user_bad();					\
	}								\
} while (0)

308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
#define __put_user_size_ex(x, ptr, size)				\
do {									\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__put_user_asm_ex(x, ptr, "b", "b", "iq");		\
		break;							\
	case 2:								\
		__put_user_asm_ex(x, ptr, "w", "w", "ir");		\
		break;							\
	case 4:								\
		__put_user_asm_ex(x, ptr, "l", "k", "ir");		\
		break;							\
	case 8:								\
		__put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);	\
		break;							\
	default:							\
		__put_user_bad();					\
	}								\
} while (0)

329 330 331 332 333 334 335 336 337 338 339
#else

#define __put_user_size(x, ptr, size, retval, errret)			\
do {									\
	__typeof__(*(ptr))__pus_tmp = x;				\
	retval = 0;							\
									\
	if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0))	\
		retval = errret;					\
} while (0)

G
Glauber Costa 已提交
340 341 342 343 344 345 346 347 348 349
#define put_user(x, ptr)					\
({								\
	int __ret_pu;						\
	__typeof__(*(ptr))__pus_tmp = x;			\
	__ret_pu = 0;						\
	if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp,		\
				       sizeof(*(ptr))) != 0))	\
		__ret_pu = -EFAULT;				\
	__ret_pu;						\
})
350 351
#endif

352 353
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval, errret)	(x) = __get_user_bad()
354
#define __get_user_asm_ex_u64(x, ptr)			(x) = __get_user_bad()
355 356 357
#else
#define __get_user_asm_u64(x, ptr, retval, errret) \
	 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
358 359
#define __get_user_asm_ex_u64(x, ptr) \
	 __get_user_asm_ex(x, ptr, "q", "", "=r")
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
#endif

#define __get_user_size(x, ptr, size, retval, errret)			\
do {									\
	retval = 0;							\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__get_user_asm(x, ptr, retval, "b", "b", "=q", errret);	\
		break;							\
	case 2:								\
		__get_user_asm(x, ptr, retval, "w", "w", "=r", errret);	\
		break;							\
	case 4:								\
		__get_user_asm(x, ptr, retval, "l", "k", "=r", errret);	\
		break;							\
	case 8:								\
		__get_user_asm_u64(x, ptr, retval, errret);		\
		break;							\
	default:							\
		(x) = __get_user_bad();					\
	}								\
} while (0)

#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
385 386 387
	asm volatile(ASM_STAC "\n"					\
		     "1:	mov"itype" %2,%"rtype"1\n"		\
		     "2: " ASM_CLAC "\n"				\
388 389 390 391 392 393 394 395 396
		     ".section .fixup,\"ax\"\n"				\
		     "3:	mov %3,%0\n"				\
		     "	xor"itype" %"rtype"1,%"rtype"1\n"		\
		     "	jmp 2b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 3b)				\
		     : "=r" (err), ltype(x)				\
		     : "m" (__m(addr)), "i" (errret), "0" (err))

397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
#define __get_user_size_ex(x, ptr, size)				\
do {									\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__get_user_asm_ex(x, ptr, "b", "b", "=q");		\
		break;							\
	case 2:								\
		__get_user_asm_ex(x, ptr, "w", "w", "=r");		\
		break;							\
	case 4:								\
		__get_user_asm_ex(x, ptr, "l", "k", "=r");		\
		break;							\
	case 8:								\
		__get_user_asm_ex_u64(x, ptr);				\
		break;							\
	default:							\
		(x) = __get_user_bad();					\
	}								\
} while (0)

#define __get_user_asm_ex(x, addr, itype, rtype, ltype)			\
419 420
	asm volatile("1:	mov"itype" %1,%"rtype"0\n"		\
		     "2:\n"						\
421
		     _ASM_EXTABLE_EX(1b, 2b)				\
422 423
		     : ltype(x) : "m" (__m(addr)))

424 425
#define __put_user_nocheck(x, ptr, size)			\
({								\
426
	int __pu_err;						\
427 428 429 430
	__put_user_size((x), (ptr), (size), __pu_err, -EFAULT);	\
	__pu_err;						\
})

431 432
#define __get_user_nocheck(x, ptr, size)				\
({									\
433
	int __gu_err;							\
434 435 436 437 438
	unsigned long __gu_val;						\
	__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);	\
	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
	__gu_err;							\
})
439 440 441 442 443 444 445 446 447 448 449

/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x))

/*
 * Tell gcc we read from memory instead of writing: this is because
 * we do not write to any memory gcc knows about, so there are no
 * aliasing issues.
 */
#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
450 451 452
	asm volatile(ASM_STAC "\n"					\
		     "1:	mov"itype" %"rtype"1,%2\n"		\
		     "2: " ASM_CLAC "\n"				\
453 454 455 456 457 458 459
		     ".section .fixup,\"ax\"\n"				\
		     "3:	mov %3,%0\n"				\
		     "	jmp 2b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 3b)				\
		     : "=r"(err)					\
		     : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
460 461

#define __put_user_asm_ex(x, addr, itype, rtype, ltype)			\
462 463
	asm volatile("1:	mov"itype" %"rtype"0,%1\n"		\
		     "2:\n"						\
464
		     _ASM_EXTABLE_EX(1b, 2b)				\
465 466 467 468 469 470 471
		     : : ltype(x), "m" (__m(addr)))

/*
 * uaccess_try and catch
 */
#define uaccess_try	do {						\
	current_thread_info()->uaccess_err = 0;				\
472
	stac();								\
473 474 475
	barrier();

#define uaccess_catch(err)						\
476
	clac();								\
477
	(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0);	\
478 479
} while (0)

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
/**
 * __get_user: - Get a simple variable from user space, with less checking.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
 * Context: User context only.  This function may sleep.
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Returns zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
 */

#define __get_user(x, ptr)						\
	__get_user_nocheck((x), (ptr), sizeof(*(ptr)))
503

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
/**
 * __put_user: - Write a simple value into user space, with less checking.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
 * Context: User context only.  This function may sleep.
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Returns zero on success, or -EFAULT on error.
 */

#define __put_user(x, ptr)						\
	__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
526

527 528
#define __get_user_unaligned __get_user
#define __put_user_unaligned __put_user
G
Glauber Costa 已提交
529

530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
/*
 * {get|put}_user_try and catch
 *
 * get_user_try {
 *	get_user_ex(...);
 * } get_user_catch(err)
 */
#define get_user_try		uaccess_try
#define get_user_catch(err)	uaccess_catch(err)

#define get_user_ex(x, ptr)	do {					\
	unsigned long __gue_val;					\
	__get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));	\
	(x) = (__force __typeof__(*(ptr)))__gue_val;			\
} while (0)

546 547 548 549 550
#ifdef CONFIG_X86_WP_WORKS_OK

#define put_user_try		uaccess_try
#define put_user_catch(err)	uaccess_catch(err)

551 552 553
#define put_user_ex(x, ptr)						\
	__put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
#else /* !CONFIG_X86_WP_WORKS_OK */

#define put_user_try		do {		\
	int __uaccess_err = 0;

#define put_user_catch(err)			\
	(err) |= __uaccess_err;			\
} while (0)

#define put_user_ex(x, ptr)	do {		\
	__uaccess_err |= __put_user(x, ptr);	\
} while (0)

#endif /* CONFIG_X86_WP_WORKS_OK */

569 570
extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
571 572
extern __must_check long
strncpy_from_user(char *dst, const char __user *src, long count);
573

574 575 576
extern __must_check long strlen_user(const char __user *str);
extern __must_check long strnlen_user(const char __user *str, long n);

577 578 579
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);

580 581 582 583 584 585 586 587 588
/*
 * movsl can be slow when source and dest are not both 8-byte aligned
 */
#ifdef CONFIG_X86_INTEL_USERCOPY
extern struct movsl_mask {
	int mask;
} ____cacheline_aligned_in_smp movsl_mask;
#endif

589 590
#define ARCH_HAS_NOCACHE_UACCESS 1

591 592 593 594 595
#ifdef CONFIG_X86_32
# include "uaccess_32.h"
#else
# include "uaccess_64.h"
#endif
596

H
H. Peter Anvin 已提交
597
#endif /* _ASM_X86_UACCESS_H */
598