uaccess.h 22.3 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_UACCESS_H
#define _ASM_X86_UACCESS_H
3 4 5 6 7 8 9 10 11
/*
 * User space memory access functions
 */
#include <linux/errno.h>
#include <linux/compiler.h>
#include <linux/thread_info.h>
#include <linux/string.h>
#include <asm/asm.h>
#include <asm/page.h>
12
#include <asm/smap.h>
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27

#define VERIFY_READ 0
#define VERIFY_WRITE 1

/*
 * The fs value determines whether argument validity checking should be
 * performed or not.  If get_fs() == USER_DS, checking is performed, with
 * get_fs() == KERNEL_DS, checking is bypassed.
 *
 * For historical reasons, these macros are grossly misnamed.
 */

#define MAKE_MM_SEG(s)	((mm_segment_t) { (s) })

#define KERNEL_DS	MAKE_MM_SEG(-1UL)
28
#define USER_DS 	MAKE_MM_SEG(TASK_SIZE_MAX)
29 30 31 32 33 34 35

#define get_ds()	(KERNEL_DS)
#define get_fs()	(current_thread_info()->addr_limit)
#define set_fs(x)	(current_thread_info()->addr_limit = (x))

#define segment_eq(a, b)	((a).seg == (b).seg)

36
#define user_addr_max() (current_thread_info()->addr_limit.seg)
37 38
#define __addr_ok(addr) 	\
	((unsigned long __force)(addr) < user_addr_max())
39

40 41 42 43
/*
 * Test whether a block of memory is a valid user space address.
 * Returns 0 if the range is valid, nonzero otherwise.
 */
44
static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
45 46 47 48 49 50 51 52 53
{
	/*
	 * If we have used "sizeof()" for the size,
	 * we know it won't overflow the limit (but
	 * it might overflow the 'addr', so it's
	 * important to subtract the size from the
	 * limit, not add it to the address).
	 */
	if (__builtin_constant_p(size))
54
		return unlikely(addr > limit - size);
55 56 57

	/* Arbitrary sizes? Be careful about overflow */
	addr += size;
58
	if (unlikely(addr < size))
59
		return true;
60
	return unlikely(addr > limit);
61
}
62

63
#define __range_not_ok(addr, size, limit)				\
64 65
({									\
	__chk_user_ptr(addr);						\
66
	__chk_range_not_ok((unsigned long __force)(addr), size, limit); \
67 68 69 70 71 72 73 74 75 76
})

/**
 * access_ok: - Checks if a user space pointer is valid
 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 *        to write to a block, it is always safe to read from it.
 * @addr: User space pointer to start of block to check
 * @size: Size of block to check
 *
77 78
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
79 80 81 82 83 84 85 86 87 88
 *
 * Checks if a pointer to a block of memory in user space is valid.
 *
 * Returns true (nonzero) if the memory block may be valid, false (zero)
 * if it is definitely invalid.
 *
 * Note that, depending on architecture, this function probably just
 * checks that the pointer is in the user space range - after calling
 * this function, memory access functions may still return -EFAULT.
 */
89
#define access_ok(type, addr, size) \
90
	likely(!__range_not_ok(addr, size, user_addr_max()))
91 92

/*
93 94 95 96 97 98
 * The exception table consists of pairs of addresses relative to the
 * exception table enty itself: the first is the address of an
 * instruction that is allowed to fault, and the second is the address
 * at which the program should continue.  No registers are modified,
 * so it is entirely up to the continuation code to figure out what to
 * do.
99 100 101 102 103 104 105 106
 *
 * All the routines below use bits of fixup code that are out of line
 * with the main instruction path.  This means when everything is well,
 * we don't even have to jump over them.  Further, they do not intrude
 * on our cache or tlb entries.
 */

struct exception_table_entry {
107
	int insn, fixup;
108
};
109 110 111
/* This is not the generic standard exception_table_entry format */
#define ARCH_HAS_SORT_EXTABLE
#define ARCH_HAS_SEARCH_EXTABLE
112 113

extern int fixup_exception(struct pt_regs *regs);
114
extern int early_fixup_exception(unsigned long *ip);
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136

/*
 * These are the main single-value transfer routines.  They automatically
 * use the right size if we just have the right pointer type.
 *
 * This gets kind of ugly. We want to return _two_ values in "get_user()"
 * and yet we don't want to do any pointers, because that is too much
 * of a performance impact. Thus we have a few rather ugly macros here,
 * and hide all the ugliness from the user.
 *
 * The "__xxx" versions of the user access functions are versions that
 * do not verify the address space, that must have been done previously
 * with a separate "access_ok()" call (this is used when we do multiple
 * accesses to the same area of user memory).
 */

extern int __get_user_1(void);
extern int __get_user_2(void);
extern int __get_user_4(void);
extern int __get_user_8(void);
extern int __get_user_bad(void);

137 138 139 140 141 142
/*
 * This is a type: either unsigned long, if the argument fits into
 * that type, or otherwise unsigned long long.
 */
#define __inttype(x) \
__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
G
Glauber Costa 已提交
143 144 145 146 147 148

/**
 * get_user: - Get a simple variable from user space.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
149 150
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
G
Glauber Costa 已提交
151 152 153 154 155 156 157 158 159 160
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Returns zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
161 162
 */
/*
163 164
 * Careful: we have to cast the result to the type of the pointer
 * for sign reasons.
165
 *
166
 * The use of _ASM_DX as the register specifier is a bit of a
167 168 169 170
 * simplification, as gcc only cares about it as the starting point
 * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
 * (%ecx being the next register in gcc's x86 register sequence), and
 * %rdx on 64 bits.
171 172 173
 *
 * Clang/LLVM cares about the size of the register, but still wants
 * the base register for something that ends up being a pair.
G
Glauber Costa 已提交
174 175 176 177
 */
#define get_user(x, ptr)						\
({									\
	int __ret_gu;							\
178
	register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);		\
G
Glauber Costa 已提交
179
	__chk_user_ptr(ptr);						\
180
	might_fault();							\
181 182 183
	asm volatile("call __get_user_%P3"				\
		     : "=a" (__ret_gu), "=r" (__val_gu)			\
		     : "0" (ptr), "i" (sizeof(*(ptr))));		\
184
	(x) = (__force __typeof__(*(ptr))) __val_gu;			\
185
	__builtin_expect(__ret_gu, 0);					\
G
Glauber Costa 已提交
186 187
})

G
Glauber Costa 已提交
188 189
#define __put_user_x(size, x, ptr, __ret_pu)			\
	asm volatile("call __put_user_" #size : "=a" (__ret_pu)	\
190
		     : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
G
Glauber Costa 已提交
191 192 193



194
#ifdef CONFIG_X86_32
195
#define __put_user_asm_u64(x, addr, err, errret)			\
196 197
	asm volatile(ASM_STAC "\n"					\
		     "1:	movl %%eax,0(%2)\n"			\
198
		     "2:	movl %%edx,4(%2)\n"			\
199
		     "3: " ASM_CLAC "\n"				\
200 201 202 203 204 205 206
		     ".section .fixup,\"ax\"\n"				\
		     "4:	movl %3,%0\n"				\
		     "	jmp 3b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 4b)				\
		     _ASM_EXTABLE(2b, 4b)				\
		     : "=r" (err)					\
207
		     : "A" (x), "r" (addr), "i" (errret), "0" (err))
G
Glauber Costa 已提交
208

209
#define __put_user_asm_ex_u64(x, addr)					\
210 211
	asm volatile(ASM_STAC "\n"					\
		     "1:	movl %%eax,0(%1)\n"			\
212
		     "2:	movl %%edx,4(%1)\n"			\
213
		     "3: " ASM_CLAC "\n"				\
214 215
		     _ASM_EXTABLE_EX(1b, 2b)				\
		     _ASM_EXTABLE_EX(2b, 3b)				\
216 217
		     : : "A" (x), "r" (addr))

G
Glauber Costa 已提交
218 219 220
#define __put_user_x8(x, ptr, __ret_pu)				\
	asm volatile("call __put_user_8" : "=a" (__ret_pu)	\
		     : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
221
#else
222
#define __put_user_asm_u64(x, ptr, retval, errret) \
223
	__put_user_asm(x, ptr, retval, "q", "", "er", errret)
224
#define __put_user_asm_ex_u64(x, addr)	\
225
	__put_user_asm_ex(x, addr, "q", "", "er")
G
Glauber Costa 已提交
226
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
227 228
#endif

G
Glauber Costa 已提交
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
extern void __put_user_bad(void);

/*
 * Strange magic calling convention: pointer in %ecx,
 * value in %eax(:%edx), return value in %eax. clobbers %rbx
 */
extern void __put_user_1(void);
extern void __put_user_2(void);
extern void __put_user_4(void);
extern void __put_user_8(void);

/**
 * put_user: - Write a simple value into user space.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
245 246
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
G
Glauber Costa 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Returns zero on success, or -EFAULT on error.
 */
#define put_user(x, ptr)					\
({								\
	int __ret_pu;						\
	__typeof__(*(ptr)) __pu_val;				\
	__chk_user_ptr(ptr);					\
262
	might_fault();						\
G
Glauber Costa 已提交
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
	__pu_val = x;						\
	switch (sizeof(*(ptr))) {				\
	case 1:							\
		__put_user_x(1, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 2:							\
		__put_user_x(2, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 4:							\
		__put_user_x(4, __pu_val, ptr, __ret_pu);	\
		break;						\
	case 8:							\
		__put_user_x8(__pu_val, ptr, __ret_pu);		\
		break;						\
	default:						\
		__put_user_x(X, __pu_val, ptr, __ret_pu);	\
		break;						\
	}							\
281
	__builtin_expect(__ret_pu, 0);				\
G
Glauber Costa 已提交
282 283
})

284 285 286 287 288 289 290 291 292 293 294 295
#define __put_user_size(x, ptr, size, retval, errret)			\
do {									\
	retval = 0;							\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__put_user_asm(x, ptr, retval, "b", "b", "iq", errret);	\
		break;							\
	case 2:								\
		__put_user_asm(x, ptr, retval, "w", "w", "ir", errret);	\
		break;							\
	case 4:								\
296
		__put_user_asm(x, ptr, retval, "l", "k", "ir", errret);	\
297 298
		break;							\
	case 8:								\
299 300
		__put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval,	\
				   errret);				\
301 302 303 304 305 306
		break;							\
	default:							\
		__put_user_bad();					\
	}								\
} while (0)

307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
#define __put_user_size_ex(x, ptr, size)				\
do {									\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__put_user_asm_ex(x, ptr, "b", "b", "iq");		\
		break;							\
	case 2:								\
		__put_user_asm_ex(x, ptr, "w", "w", "ir");		\
		break;							\
	case 4:								\
		__put_user_asm_ex(x, ptr, "l", "k", "ir");		\
		break;							\
	case 8:								\
		__put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);	\
		break;							\
	default:							\
		__put_user_bad();					\
	}								\
} while (0)

328 329
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval, errret)	(x) = __get_user_bad()
330
#define __get_user_asm_ex_u64(x, ptr)			(x) = __get_user_bad()
331 332 333
#else
#define __get_user_asm_u64(x, ptr, retval, errret) \
	 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
334 335
#define __get_user_asm_ex_u64(x, ptr) \
	 __get_user_asm_ex(x, ptr, "q", "", "=r")
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
#endif

#define __get_user_size(x, ptr, size, retval, errret)			\
do {									\
	retval = 0;							\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__get_user_asm(x, ptr, retval, "b", "b", "=q", errret);	\
		break;							\
	case 2:								\
		__get_user_asm(x, ptr, retval, "w", "w", "=r", errret);	\
		break;							\
	case 4:								\
		__get_user_asm(x, ptr, retval, "l", "k", "=r", errret);	\
		break;							\
	case 8:								\
		__get_user_asm_u64(x, ptr, retval, errret);		\
		break;							\
	default:							\
		(x) = __get_user_bad();					\
	}								\
} while (0)

#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
361 362 363
	asm volatile(ASM_STAC "\n"					\
		     "1:	mov"itype" %2,%"rtype"1\n"		\
		     "2: " ASM_CLAC "\n"				\
364 365 366 367 368 369 370 371 372
		     ".section .fixup,\"ax\"\n"				\
		     "3:	mov %3,%0\n"				\
		     "	xor"itype" %"rtype"1,%"rtype"1\n"		\
		     "	jmp 2b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 3b)				\
		     : "=r" (err), ltype(x)				\
		     : "m" (__m(addr)), "i" (errret), "0" (err))

373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
#define __get_user_size_ex(x, ptr, size)				\
do {									\
	__chk_user_ptr(ptr);						\
	switch (size) {							\
	case 1:								\
		__get_user_asm_ex(x, ptr, "b", "b", "=q");		\
		break;							\
	case 2:								\
		__get_user_asm_ex(x, ptr, "w", "w", "=r");		\
		break;							\
	case 4:								\
		__get_user_asm_ex(x, ptr, "l", "k", "=r");		\
		break;							\
	case 8:								\
		__get_user_asm_ex_u64(x, ptr);				\
		break;							\
	default:							\
		(x) = __get_user_bad();					\
	}								\
} while (0)

#define __get_user_asm_ex(x, addr, itype, rtype, ltype)			\
395 396
	asm volatile("1:	mov"itype" %1,%"rtype"0\n"		\
		     "2:\n"						\
397
		     _ASM_EXTABLE_EX(1b, 2b)				\
398 399
		     : ltype(x) : "m" (__m(addr)))

400 401
#define __put_user_nocheck(x, ptr, size)			\
({								\
402
	int __pu_err;						\
403
	__put_user_size((x), (ptr), (size), __pu_err, -EFAULT);	\
404
	__builtin_expect(__pu_err, 0);				\
405 406
})

407 408
#define __get_user_nocheck(x, ptr, size)				\
({									\
409
	int __gu_err;							\
410 411 412
	unsigned long __gu_val;						\
	__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);	\
	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
413
	__builtin_expect(__gu_err, 0);					\
414
})
415 416 417 418 419 420 421 422 423 424 425

/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x))

/*
 * Tell gcc we read from memory instead of writing: this is because
 * we do not write to any memory gcc knows about, so there are no
 * aliasing issues.
 */
#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
426 427 428
	asm volatile(ASM_STAC "\n"					\
		     "1:	mov"itype" %"rtype"1,%2\n"		\
		     "2: " ASM_CLAC "\n"				\
429 430 431 432 433 434 435
		     ".section .fixup,\"ax\"\n"				\
		     "3:	mov %3,%0\n"				\
		     "	jmp 2b\n"					\
		     ".previous\n"					\
		     _ASM_EXTABLE(1b, 3b)				\
		     : "=r"(err)					\
		     : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
436 437

#define __put_user_asm_ex(x, addr, itype, rtype, ltype)			\
438 439
	asm volatile("1:	mov"itype" %"rtype"0,%1\n"		\
		     "2:\n"						\
440
		     _ASM_EXTABLE_EX(1b, 2b)				\
441 442 443 444 445 446 447
		     : : ltype(x), "m" (__m(addr)))

/*
 * uaccess_try and catch
 */
#define uaccess_try	do {						\
	current_thread_info()->uaccess_err = 0;				\
448
	stac();								\
449 450 451
	barrier();

#define uaccess_catch(err)						\
452
	clac();								\
453
	(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0);	\
454 455
} while (0)

456 457 458 459 460
/**
 * __get_user: - Get a simple variable from user space, with less checking.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
461 462
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Returns zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
 */

#define __get_user(x, ptr)						\
	__get_user_nocheck((x), (ptr), sizeof(*(ptr)))
480

481 482 483 484 485
/**
 * __put_user: - Write a simple value into user space, with less checking.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
486 487
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Returns zero on success, or -EFAULT on error.
 */

#define __put_user(x, ptr)						\
	__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
504

505 506
#define __get_user_unaligned __get_user
#define __put_user_unaligned __put_user
G
Glauber Costa 已提交
507

508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
/*
 * {get|put}_user_try and catch
 *
 * get_user_try {
 *	get_user_ex(...);
 * } get_user_catch(err)
 */
#define get_user_try		uaccess_try
#define get_user_catch(err)	uaccess_catch(err)

#define get_user_ex(x, ptr)	do {					\
	unsigned long __gue_val;					\
	__get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));	\
	(x) = (__force __typeof__(*(ptr)))__gue_val;			\
} while (0)

524 525 526
#define put_user_try		uaccess_try
#define put_user_catch(err)	uaccess_catch(err)

527 528 529
#define put_user_ex(x, ptr)						\
	__put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))

530 531
extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
532 533
extern __must_check long
strncpy_from_user(char *dst, const char __user *src, long count);
534

535 536 537
extern __must_check long strlen_user(const char __user *str);
extern __must_check long strnlen_user(const char __user *str, long n);

538 539 540
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);

541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
extern void __cmpxchg_wrong_size(void)
	__compiletime_error("Bad argument size for cmpxchg");

#define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size)	\
({									\
	int __ret = 0;							\
	__typeof__(ptr) __uval = (uval);				\
	__typeof__(*(ptr)) __old = (old);				\
	__typeof__(*(ptr)) __new = (new);				\
	switch (size) {							\
	case 1:								\
	{								\
		asm volatile("\t" ASM_STAC "\n"				\
			"1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"		\
			"2:\t" ASM_CLAC "\n"				\
			"\t.section .fixup, \"ax\"\n"			\
			"3:\tmov     %3, %0\n"				\
			"\tjmp     2b\n"				\
			"\t.previous\n"					\
			_ASM_EXTABLE(1b, 3b)				\
			: "+r" (__ret), "=a" (__old), "+m" (*(ptr))	\
			: "i" (-EFAULT), "q" (__new), "1" (__old)	\
			: "memory"					\
		);							\
		break;							\
	}								\
	case 2:								\
	{								\
		asm volatile("\t" ASM_STAC "\n"				\
			"1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"		\
			"2:\t" ASM_CLAC "\n"				\
			"\t.section .fixup, \"ax\"\n"			\
			"3:\tmov     %3, %0\n"				\
			"\tjmp     2b\n"				\
			"\t.previous\n"					\
			_ASM_EXTABLE(1b, 3b)				\
			: "+r" (__ret), "=a" (__old), "+m" (*(ptr))	\
			: "i" (-EFAULT), "r" (__new), "1" (__old)	\
			: "memory"					\
		);							\
		break;							\
	}								\
	case 4:								\
	{								\
		asm volatile("\t" ASM_STAC "\n"				\
			"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"		\
			"2:\t" ASM_CLAC "\n"				\
			"\t.section .fixup, \"ax\"\n"			\
			"3:\tmov     %3, %0\n"				\
			"\tjmp     2b\n"				\
			"\t.previous\n"					\
			_ASM_EXTABLE(1b, 3b)				\
			: "+r" (__ret), "=a" (__old), "+m" (*(ptr))	\
			: "i" (-EFAULT), "r" (__new), "1" (__old)	\
			: "memory"					\
		);							\
		break;							\
	}								\
	case 8:								\
	{								\
		if (!IS_ENABLED(CONFIG_X86_64))				\
			__cmpxchg_wrong_size();				\
									\
		asm volatile("\t" ASM_STAC "\n"				\
			"1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"		\
			"2:\t" ASM_CLAC "\n"				\
			"\t.section .fixup, \"ax\"\n"			\
			"3:\tmov     %3, %0\n"				\
			"\tjmp     2b\n"				\
			"\t.previous\n"					\
			_ASM_EXTABLE(1b, 3b)				\
			: "+r" (__ret), "=a" (__old), "+m" (*(ptr))	\
			: "i" (-EFAULT), "r" (__new), "1" (__old)	\
			: "memory"					\
		);							\
		break;							\
	}								\
	default:							\
		__cmpxchg_wrong_size();					\
	}								\
	*__uval = __old;						\
	__ret;								\
})

#define user_atomic_cmpxchg_inatomic(uval, ptr, old, new)		\
({									\
	access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ?		\
		__user_atomic_cmpxchg_inatomic((uval), (ptr),		\
				(old), (new), sizeof(*(ptr))) :		\
		-EFAULT;						\
})

633 634 635 636 637 638 639 640 641
/*
 * movsl can be slow when source and dest are not both 8-byte aligned
 */
#ifdef CONFIG_X86_INTEL_USERCOPY
extern struct movsl_mask {
	int mask;
} ____cacheline_aligned_in_smp movsl_mask;
#endif

642 643
#define ARCH_HAS_NOCACHE_UACCESS 1

644
#ifdef CONFIG_X86_32
645
# include <asm/uaccess_32.h>
646
#else
647
# include <asm/uaccess_64.h>
648
#endif
649

650 651
unsigned long __must_check _copy_from_user(void *to, const void __user *from,
					   unsigned n);
652 653
unsigned long __must_check _copy_to_user(void __user *to, const void *from,
					 unsigned n);
654 655 656 657 658 659 660 661 662

#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
# define copy_user_diag __compiletime_error
#else
# define copy_user_diag __compiletime_warning
#endif

extern void copy_user_diag("copy_from_user() buffer size is too small")
copy_from_user_overflow(void);
663 664
extern void copy_user_diag("copy_to_user() buffer size is too small")
copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
665 666 667 668 669 670 671 672 673 674

#undef copy_user_diag

#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS

extern void
__compiletime_warning("copy_from_user() buffer size is not provably correct")
__copy_from_user_overflow(void) __asm__("copy_from_user_overflow");
#define __copy_from_user_overflow(size, count) __copy_from_user_overflow()

675 676 677 678 679
extern void
__compiletime_warning("copy_to_user() buffer size is not provably correct")
__copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
#define __copy_to_user_overflow(size, count) __copy_to_user_overflow()

680 681 682 683 684 685 686 687
#else

static inline void
__copy_from_user_overflow(int size, unsigned long count)
{
	WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
}

688 689
#define __copy_to_user_overflow __copy_from_user_overflow

690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
#endif

static inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long n)
{
	int sz = __compiletime_object_size(to);

	might_fault();

	/*
	 * While we would like to have the compiler do the checking for us
	 * even in the non-constant size case, any false positives there are
	 * a problem (especially when DEBUG_STRICT_USER_COPY_CHECKS, but even
	 * without - the [hopefully] dangerous looking nature of the warning
	 * would make people go look at the respecitive call sites over and
	 * over again just to find that there's no problem).
	 *
	 * And there are cases where it's just not realistic for the compiler
	 * to prove the count to be in range. For example when multiple call
	 * sites of a helper function - perhaps in different source files -
	 * all doing proper range checking, yet the helper function not doing
	 * so again.
	 *
	 * Therefore limit the compile time checking to the constant size
	 * case, and do only runtime checking for non-constant sizes.
	 */

	if (likely(sz < 0 || sz >= n))
		n = _copy_from_user(to, from, n);
	else if(__builtin_constant_p(n))
		copy_from_user_overflow();
	else
		__copy_from_user_overflow(sz, n);

	return n;
}

727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
static inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long n)
{
	int sz = __compiletime_object_size(from);

	might_fault();

	/* See the comment in copy_from_user() above. */
	if (likely(sz < 0 || sz >= n))
		n = _copy_to_user(to, from, n);
	else if(__builtin_constant_p(n))
		copy_to_user_overflow();
	else
		__copy_to_user_overflow(sz, n);

	return n;
}

745
#undef __copy_from_user_overflow
746
#undef __copy_to_user_overflow
747

H
H. Peter Anvin 已提交
748
#endif /* _ASM_X86_UACCESS_H */
749