read_write.c 19.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  linux/fs/read_write.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

#include <linux/slab.h> 
#include <linux/stat.h>
#include <linux/fcntl.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/smp_lock.h>
R
Robert Love 已提交
13
#include <linux/fsnotify.h>
L
Linus Torvalds 已提交
14 15 16
#include <linux/security.h>
#include <linux/module.h>
#include <linux/syscalls.h>
17
#include <linux/pagemap.h>
18
#include <linux/splice.h>
19
#include "read_write.h"
L
Linus Torvalds 已提交
20 21 22 23

#include <asm/uaccess.h>
#include <asm/unistd.h>

24
const struct file_operations generic_ro_fops = {
L
Linus Torvalds 已提交
25
	.llseek		= generic_file_llseek,
26 27
	.read		= do_sync_read,
	.aio_read	= generic_file_aio_read,
L
Linus Torvalds 已提交
28
	.mmap		= generic_file_readonly_mmap,
29
	.splice_read	= generic_file_splice_read,
L
Linus Torvalds 已提交
30 31 32 33
};

EXPORT_SYMBOL(generic_ro_fops);

34 35 36 37 38 39 40 41 42
/**
 * generic_file_llseek_unlocked - lockless generic llseek implementation
 * @file:	file structure to seek on
 * @offset:	file offset to seek to
 * @origin:	type of seek
 *
 * Updates the file offset to the value specified by @offset and @origin.
 * Locking must be provided by the caller.
 */
A
Andi Kleen 已提交
43 44
loff_t
generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
L
Linus Torvalds 已提交
45 46 47 48
{
	struct inode *inode = file->f_mapping->host;

	switch (origin) {
49 50 51 52
	case SEEK_END:
		offset += inode->i_size;
		break;
	case SEEK_CUR:
53 54 55 56 57 58 59 60
		/*
		 * Here we special-case the lseek(fd, 0, SEEK_CUR)
		 * position-querying operation.  Avoid rewriting the "same"
		 * f_pos value back to the file because a concurrent read(),
		 * write() or lseek() might have altered it
		 */
		if (offset == 0)
			return file->f_pos;
61 62
		offset += file->f_pos;
		break;
L
Linus Torvalds 已提交
63
	}
64 65 66 67 68 69 70 71

	if (offset < 0 || offset > inode->i_sb->s_maxbytes)
		return -EINVAL;

	/* Special lock needed here? */
	if (offset != file->f_pos) {
		file->f_pos = offset;
		file->f_version = 0;
L
Linus Torvalds 已提交
72
	}
73 74

	return offset;
L
Linus Torvalds 已提交
75
}
A
Andi Kleen 已提交
76
EXPORT_SYMBOL(generic_file_llseek_unlocked);
L
Linus Torvalds 已提交
77

78 79 80 81 82 83 84 85 86 87
/**
 * generic_file_llseek - generic llseek implementation for regular files
 * @file:	file structure to seek on
 * @offset:	file offset to seek to
 * @origin:	type of seek
 *
 * This is a generic implemenation of ->llseek useable for all normal local
 * filesystems.  It just updates the file offset to the value specified by
 * @offset and @origin under i_mutex.
 */
A
Andi Kleen 已提交
88
loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
L
Linus Torvalds 已提交
89
{
90 91
	loff_t rval;

A
Andi Kleen 已提交
92
	mutex_lock(&file->f_dentry->d_inode->i_mutex);
93
	rval = generic_file_llseek_unlocked(file, offset, origin);
A
Andi Kleen 已提交
94
	mutex_unlock(&file->f_dentry->d_inode->i_mutex);
95 96

	return rval;
L
Linus Torvalds 已提交
97
}
A
Andi Kleen 已提交
98
EXPORT_SYMBOL(generic_file_llseek);
L
Linus Torvalds 已提交
99 100 101 102 103 104 105 106 107

loff_t no_llseek(struct file *file, loff_t offset, int origin)
{
	return -ESPIPE;
}
EXPORT_SYMBOL(no_llseek);

loff_t default_llseek(struct file *file, loff_t offset, int origin)
{
108
	loff_t retval;
L
Linus Torvalds 已提交
109 110 111

	lock_kernel();
	switch (origin) {
112
		case SEEK_END:
113
			offset += i_size_read(file->f_path.dentry->d_inode);
L
Linus Torvalds 已提交
114
			break;
115
		case SEEK_CUR:
116 117 118 119
			if (offset == 0) {
				retval = file->f_pos;
				goto out;
			}
L
Linus Torvalds 已提交
120 121 122 123 124 125 126 127 128 129
			offset += file->f_pos;
	}
	retval = -EINVAL;
	if (offset >= 0) {
		if (offset != file->f_pos) {
			file->f_pos = offset;
			file->f_version = 0;
		}
		retval = offset;
	}
130
out:
L
Linus Torvalds 已提交
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
	unlock_kernel();
	return retval;
}
EXPORT_SYMBOL(default_llseek);

loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
{
	loff_t (*fn)(struct file *, loff_t, int);

	fn = no_llseek;
	if (file->f_mode & FMODE_LSEEK) {
		fn = default_llseek;
		if (file->f_op && file->f_op->llseek)
			fn = file->f_op->llseek;
	}
	return fn(file, offset, origin);
}
EXPORT_SYMBOL(vfs_llseek);

150
SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
L
Linus Torvalds 已提交
151 152 153 154 155 156 157 158 159 160 161
{
	off_t retval;
	struct file * file;
	int fput_needed;

	retval = -EBADF;
	file = fget_light(fd, &fput_needed);
	if (!file)
		goto bad;

	retval = -EINVAL;
162
	if (origin <= SEEK_MAX) {
L
Linus Torvalds 已提交
163 164 165 166 167 168 169 170 171 172 173
		loff_t res = vfs_llseek(file, offset, origin);
		retval = res;
		if (res != (loff_t)retval)
			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
	}
	fput_light(file, fput_needed);
bad:
	return retval;
}

#ifdef __ARCH_WANT_SYS_LLSEEK
174 175 176
SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
		unsigned long, offset_low, loff_t __user *, result,
		unsigned int, origin)
L
Linus Torvalds 已提交
177 178 179 180 181 182 183 184 185 186 187 188
{
	int retval;
	struct file * file;
	loff_t offset;
	int fput_needed;

	retval = -EBADF;
	file = fget_light(fd, &fput_needed);
	if (!file)
		goto bad;

	retval = -EINVAL;
189
	if (origin > SEEK_MAX)
L
Linus Torvalds 已提交
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
		goto out_putf;

	offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
			origin);

	retval = (int)offset;
	if (offset >= 0) {
		retval = -EFAULT;
		if (!copy_to_user(result, &offset, sizeof(offset)))
			retval = 0;
	}
out_putf:
	fput_light(file, fput_needed);
bad:
	return retval;
}
#endif

208 209 210 211 212 213
/*
 * rw_verify_area doesn't like huge counts. We limit
 * them to something that fits in "int" so that others
 * won't have to do range checks all the time.
 */
#define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
L
Linus Torvalds 已提交
214 215 216 217 218

int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
{
	struct inode *inode;
	loff_t pos;
219
	int retval = -EINVAL;
L
Linus Torvalds 已提交
220

221
	inode = file->f_path.dentry->d_inode;
222
	if (unlikely((ssize_t) count < 0))
223
		return retval;
L
Linus Torvalds 已提交
224 225
	pos = *ppos;
	if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
226
		return retval;
L
Linus Torvalds 已提交
227

228
	if (unlikely(inode->i_flock && mandatory_lock(inode))) {
229
		retval = locks_mandatory_area(
230 231 232 233 234
			read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
			inode, file, pos, count);
		if (retval < 0)
			return retval;
	}
235 236 237 238
	retval = security_file_permission(file,
				read_write == READ ? MAY_READ : MAY_WRITE);
	if (retval)
		return retval;
239
	return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
L
Linus Torvalds 已提交
240 241
}

242 243 244 245 246 247 248 249 250 251
static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
{
	set_current_state(TASK_UNINTERRUPTIBLE);
	if (!kiocbIsKicked(iocb))
		schedule();
	else
		kiocbClearKicked(iocb);
	__set_current_state(TASK_RUNNING);
}

L
Linus Torvalds 已提交
252 253
ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
{
254
	struct iovec iov = { .iov_base = buf, .iov_len = len };
L
Linus Torvalds 已提交
255 256 257 258 259
	struct kiocb kiocb;
	ssize_t ret;

	init_sync_kiocb(&kiocb, filp);
	kiocb.ki_pos = *ppos;
260 261 262 263 264 265
	kiocb.ki_left = len;

	for (;;) {
		ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
		if (ret != -EIOCBRETRY)
			break;
266
		wait_on_retry_sync_kiocb(&kiocb);
267
	}
268

L
Linus Torvalds 已提交
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
	if (-EIOCBQUEUED == ret)
		ret = wait_on_sync_kiocb(&kiocb);
	*ppos = kiocb.ki_pos;
	return ret;
}

EXPORT_SYMBOL(do_sync_read);

ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
{
	ssize_t ret;

	if (!(file->f_mode & FMODE_READ))
		return -EBADF;
	if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
		return -EINVAL;
	if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
		return -EFAULT;

	ret = rw_verify_area(READ, file, pos, count);
289 290
	if (ret >= 0) {
		count = ret;
291 292 293 294 295 296 297
		if (file->f_op->read)
			ret = file->f_op->read(file, buf, count, pos);
		else
			ret = do_sync_read(file, buf, count, pos);
		if (ret > 0) {
			fsnotify_access(file->f_path.dentry);
			add_rchar(current, ret);
L
Linus Torvalds 已提交
298
		}
299
		inc_syscr(current);
L
Linus Torvalds 已提交
300 301 302 303 304 305 306 307 308
	}

	return ret;
}

EXPORT_SYMBOL(vfs_read);

ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
{
309
	struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
L
Linus Torvalds 已提交
310 311 312 313 314
	struct kiocb kiocb;
	ssize_t ret;

	init_sync_kiocb(&kiocb, filp);
	kiocb.ki_pos = *ppos;
315 316 317 318 319 320
	kiocb.ki_left = len;

	for (;;) {
		ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
		if (ret != -EIOCBRETRY)
			break;
321
		wait_on_retry_sync_kiocb(&kiocb);
322
	}
323

L
Linus Torvalds 已提交
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
	if (-EIOCBQUEUED == ret)
		ret = wait_on_sync_kiocb(&kiocb);
	*ppos = kiocb.ki_pos;
	return ret;
}

EXPORT_SYMBOL(do_sync_write);

ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
{
	ssize_t ret;

	if (!(file->f_mode & FMODE_WRITE))
		return -EBADF;
	if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
		return -EINVAL;
	if (unlikely(!access_ok(VERIFY_READ, buf, count)))
		return -EFAULT;

	ret = rw_verify_area(WRITE, file, pos, count);
344 345
	if (ret >= 0) {
		count = ret;
346 347 348 349 350 351 352
		if (file->f_op->write)
			ret = file->f_op->write(file, buf, count, pos);
		else
			ret = do_sync_write(file, buf, count, pos);
		if (ret > 0) {
			fsnotify_modify(file->f_path.dentry);
			add_wchar(current, ret);
L
Linus Torvalds 已提交
353
		}
354
		inc_syscw(current);
L
Linus Torvalds 已提交
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
	}

	return ret;
}

EXPORT_SYMBOL(vfs_write);

static inline loff_t file_pos_read(struct file *file)
{
	return file->f_pos;
}

static inline void file_pos_write(struct file *file, loff_t pos)
{
	file->f_pos = pos;
}

372
SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
L
Linus Torvalds 已提交
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	file = fget_light(fd, &fput_needed);
	if (file) {
		loff_t pos = file_pos_read(file);
		ret = vfs_read(file, buf, count, &pos);
		file_pos_write(file, pos);
		fput_light(file, fput_needed);
	}

	return ret;
}

389 390
SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
		size_t, count)
L
Linus Torvalds 已提交
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	file = fget_light(fd, &fput_needed);
	if (file) {
		loff_t pos = file_pos_read(file);
		ret = vfs_write(file, buf, count, &pos);
		file_pos_write(file, pos);
		fput_light(file, fput_needed);
	}

	return ret;
}

407 408
SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
			size_t count, loff_t pos)
L
Linus Torvalds 已提交
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	if (pos < 0)
		return -EINVAL;

	file = fget_light(fd, &fput_needed);
	if (file) {
		ret = -ESPIPE;
		if (file->f_mode & FMODE_PREAD)
			ret = vfs_read(file, buf, count, &pos);
		fput_light(file, fput_needed);
	}

	return ret;
}
427 428 429 430 431 432 433 434
#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
{
	return SYSC_pread64((unsigned int) fd, (char __user *) buf,
			    (size_t) count, pos);
}
SYSCALL_ALIAS(sys_pread64, SyS_pread64);
#endif
L
Linus Torvalds 已提交
435

436 437
SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
			 size_t count, loff_t pos)
L
Linus Torvalds 已提交
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	if (pos < 0)
		return -EINVAL;

	file = fget_light(fd, &fput_needed);
	if (file) {
		ret = -ESPIPE;
		if (file->f_mode & FMODE_PWRITE)  
			ret = vfs_write(file, buf, count, &pos);
		fput_light(file, fput_needed);
	}

	return ret;
}
456 457 458 459 460 461 462 463
#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
{
	return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
			     (size_t) count, pos);
}
SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
#endif
L
Linus Torvalds 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483

/*
 * Reduce an iovec's length in-place.  Return the resulting number of segments
 */
unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
{
	unsigned long seg = 0;
	size_t len = 0;

	while (seg < nr_segs) {
		seg++;
		if (len + iov->iov_len >= to) {
			iov->iov_len = to - len;
			break;
		}
		len += iov->iov_len;
		iov++;
	}
	return seg;
}
484
EXPORT_SYMBOL(iov_shorten);
L
Linus Torvalds 已提交
485

486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
		unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
{
	struct kiocb kiocb;
	ssize_t ret;

	init_sync_kiocb(&kiocb, filp);
	kiocb.ki_pos = *ppos;
	kiocb.ki_left = len;
	kiocb.ki_nbytes = len;

	for (;;) {
		ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
		if (ret != -EIOCBRETRY)
			break;
		wait_on_retry_sync_kiocb(&kiocb);
	}

	if (ret == -EIOCBQUEUED)
		ret = wait_on_sync_kiocb(&kiocb);
	*ppos = kiocb.ki_pos;
	return ret;
}

/* Do it by hand, with file-ops */
ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
		unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
{
	struct iovec *vector = iov;
	ssize_t ret = 0;

	while (nr_segs > 0) {
		void __user *base;
		size_t len;
		ssize_t nr;

		base = vector->iov_base;
		len = vector->iov_len;
		vector++;
		nr_segs--;

		nr = fn(filp, base, len, ppos);

		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != len)
			break;
	}

	return ret;
}

L
Linus Torvalds 已提交
542 543 544
/* A write operation does a read from user space and vice versa */
#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)

B
Badari Pulavarty 已提交
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
			      unsigned long nr_segs, unsigned long fast_segs,
			      struct iovec *fast_pointer,
			      struct iovec **ret_pointer)
  {
	unsigned long seg;
  	ssize_t ret;
	struct iovec *iov = fast_pointer;

  	/*
  	 * SuS says "The readv() function *may* fail if the iovcnt argument
  	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
  	 * traditionally returned zero for zero segments, so...
  	 */
	if (nr_segs == 0) {
		ret = 0;
  		goto out;
	}

  	/*
  	 * First get the "struct iovec" from user memory and
  	 * verify all the pointers
  	 */
	if (nr_segs > UIO_MAXIOV) {
		ret = -EINVAL;
  		goto out;
	}
	if (nr_segs > fast_segs) {
  		iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
		if (iov == NULL) {
			ret = -ENOMEM;
  			goto out;
		}
  	}
	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
		ret = -EFAULT;
  		goto out;
	}

  	/*
	 * According to the Single Unix Specification we should return EINVAL
	 * if an element length is < 0 when cast to ssize_t or if the
	 * total length would overflow the ssize_t return value of the
	 * system call.
  	 */
	ret = 0;
  	for (seg = 0; seg < nr_segs; seg++) {
  		void __user *buf = iov[seg].iov_base;
  		ssize_t len = (ssize_t)iov[seg].iov_len;

		/* see if we we're about to use an invalid len or if
		 * it's about to overflow ssize_t */
		if (len < 0 || (ret + len < ret)) {
			ret = -EINVAL;
  			goto out;
		}
		if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
			ret = -EFAULT;
  			goto out;
		}

		ret += len;
  	}
out:
	*ret_pointer = iov;
	return ret;
}

L
Linus Torvalds 已提交
613 614 615 616 617 618
static ssize_t do_readv_writev(int type, struct file *file,
			       const struct iovec __user * uvector,
			       unsigned long nr_segs, loff_t *pos)
{
	size_t tot_len;
	struct iovec iovstack[UIO_FASTIOV];
619
	struct iovec *iov = iovstack;
L
Linus Torvalds 已提交
620 621 622 623
	ssize_t ret;
	io_fn_t fn;
	iov_fn_t fnv;

B
Badari Pulavarty 已提交
624 625
	if (!file->f_op) {
		ret = -EINVAL;
L
Linus Torvalds 已提交
626 627 628
		goto out;
	}

B
Badari Pulavarty 已提交
629 630 631
	ret = rw_copy_check_uvector(type, uvector, nr_segs,
			ARRAY_SIZE(iovstack), iovstack, &iov);
	if (ret <= 0)
L
Linus Torvalds 已提交
632 633
		goto out;

B
Badari Pulavarty 已提交
634
	tot_len = ret;
L
Linus Torvalds 已提交
635
	ret = rw_verify_area(type, file, pos, tot_len);
636
	if (ret < 0)
637
		goto out;
L
Linus Torvalds 已提交
638 639 640 641

	fnv = NULL;
	if (type == READ) {
		fn = file->f_op->read;
642
		fnv = file->f_op->aio_read;
L
Linus Torvalds 已提交
643 644
	} else {
		fn = (io_fn_t)file->f_op->write;
645
		fnv = file->f_op->aio_write;
L
Linus Torvalds 已提交
646 647
	}

648 649 650 651 652
	if (fnv)
		ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
						pos, fnv);
	else
		ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
L
Linus Torvalds 已提交
653 654 655 656

out:
	if (iov != iovstack)
		kfree(iov);
R
Robert Love 已提交
657 658
	if ((ret + (type == READ)) > 0) {
		if (type == READ)
659
			fsnotify_access(file->f_path.dentry);
R
Robert Love 已提交
660
		else
661
			fsnotify_modify(file->f_path.dentry);
R
Robert Love 已提交
662
	}
L
Linus Torvalds 已提交
663 664 665 666 667 668 669 670
	return ret;
}

ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
		  unsigned long vlen, loff_t *pos)
{
	if (!(file->f_mode & FMODE_READ))
		return -EBADF;
671
	if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
L
Linus Torvalds 已提交
672 673 674 675 676 677 678 679 680 681 682 683
		return -EINVAL;

	return do_readv_writev(READ, file, vec, vlen, pos);
}

EXPORT_SYMBOL(vfs_readv);

ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
		   unsigned long vlen, loff_t *pos)
{
	if (!(file->f_mode & FMODE_WRITE))
		return -EBADF;
684
	if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
L
Linus Torvalds 已提交
685 686 687 688 689 690 691
		return -EINVAL;

	return do_readv_writev(WRITE, file, vec, vlen, pos);
}

EXPORT_SYMBOL(vfs_writev);

692 693
SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
		unsigned long, vlen)
L
Linus Torvalds 已提交
694 695 696 697 698 699 700 701 702 703 704 705 706 707
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	file = fget_light(fd, &fput_needed);
	if (file) {
		loff_t pos = file_pos_read(file);
		ret = vfs_readv(file, vec, vlen, &pos);
		file_pos_write(file, pos);
		fput_light(file, fput_needed);
	}

	if (ret > 0)
708 709
		add_rchar(current, ret);
	inc_syscr(current);
L
Linus Torvalds 已提交
710 711 712
	return ret;
}

713 714
SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
		unsigned long, vlen)
L
Linus Torvalds 已提交
715 716 717 718 719 720 721 722 723 724 725 726 727 728
{
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	file = fget_light(fd, &fput_needed);
	if (file) {
		loff_t pos = file_pos_read(file);
		ret = vfs_writev(file, vec, vlen, &pos);
		file_pos_write(file, pos);
		fput_light(file, fput_needed);
	}

	if (ret > 0)
729 730
		add_wchar(current, ret);
	inc_syscw(current);
L
Linus Torvalds 已提交
731 732 733
	return ret;
}

734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
		unsigned long, vlen, u32, pos_high, u32, pos_low)
{
	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	if (pos < 0)
		return -EINVAL;

	file = fget_light(fd, &fput_needed);
	if (file) {
		ret = -ESPIPE;
		if (file->f_mode & FMODE_PREAD)
			ret = vfs_readv(file, vec, vlen, &pos);
		fput_light(file, fput_needed);
	}

	if (ret > 0)
		add_rchar(current, ret);
	inc_syscr(current);
	return ret;
}

SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
		unsigned long, vlen, u32, pos_high, u32, pos_low)
{
	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
	struct file *file;
	ssize_t ret = -EBADF;
	int fput_needed;

	if (pos < 0)
		return -EINVAL;

	file = fget_light(fd, &fput_needed);
	if (file) {
		ret = -ESPIPE;
		if (file->f_mode & FMODE_PWRITE)
			ret = vfs_writev(file, vec, vlen, &pos);
		fput_light(file, fput_needed);
	}

	if (ret > 0)
		add_wchar(current, ret);
	inc_syscw(current);
	return ret;
}

L
Linus Torvalds 已提交
784 785 786 787 788 789 790
static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
			   size_t count, loff_t max)
{
	struct file * in_file, * out_file;
	struct inode * in_inode, * out_inode;
	loff_t pos;
	ssize_t retval;
791
	int fput_needed_in, fput_needed_out, fl;
L
Linus Torvalds 已提交
792 793 794 795 796 797 798 799 800 801 802

	/*
	 * Get input file, and verify that it is ok..
	 */
	retval = -EBADF;
	in_file = fget_light(in_fd, &fput_needed_in);
	if (!in_file)
		goto out;
	if (!(in_file->f_mode & FMODE_READ))
		goto fput_in;
	retval = -EINVAL;
803
	in_inode = in_file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
804 805
	if (!in_inode)
		goto fput_in;
J
Jens Axboe 已提交
806
	if (!in_file->f_op || !in_file->f_op->splice_read)
L
Linus Torvalds 已提交
807 808 809 810 811 812 813 814
		goto fput_in;
	retval = -ESPIPE;
	if (!ppos)
		ppos = &in_file->f_pos;
	else
		if (!(in_file->f_mode & FMODE_PREAD))
			goto fput_in;
	retval = rw_verify_area(READ, in_file, ppos, count);
815
	if (retval < 0)
L
Linus Torvalds 已提交
816
		goto fput_in;
817
	count = retval;
L
Linus Torvalds 已提交
818 819 820 821 822 823 824 825 826 827 828 829 830

	/*
	 * Get output file, and verify that it is ok..
	 */
	retval = -EBADF;
	out_file = fget_light(out_fd, &fput_needed_out);
	if (!out_file)
		goto fput_in;
	if (!(out_file->f_mode & FMODE_WRITE))
		goto fput_out;
	retval = -EINVAL;
	if (!out_file->f_op || !out_file->f_op->sendpage)
		goto fput_out;
831
	out_inode = out_file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
832
	retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
833
	if (retval < 0)
L
Linus Torvalds 已提交
834
		goto fput_out;
835
	count = retval;
L
Linus Torvalds 已提交
836 837 838 839 840 841 842 843 844 845 846 847 848 849 850

	if (!max)
		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);

	pos = *ppos;
	retval = -EINVAL;
	if (unlikely(pos < 0))
		goto fput_out;
	if (unlikely(pos + count > max)) {
		retval = -EOVERFLOW;
		if (pos >= max)
			goto fput_out;
		count = max - pos;
	}

J
Jens Axboe 已提交
851
	fl = 0;
852
#if 0
J
Jens Axboe 已提交
853 854 855 856 857 858 859 860
	/*
	 * We need to debate whether we can enable this or not. The
	 * man page documents EAGAIN return for the output at least,
	 * and the application is arguably buggy if it doesn't expect
	 * EAGAIN on a non-blocking file descriptor.
	 */
	if (in_file->f_flags & O_NONBLOCK)
		fl = SPLICE_F_NONBLOCK;
861
#endif
J
Jens Axboe 已提交
862
	retval = do_splice_direct(in_file, ppos, out_file, count, fl);
L
Linus Torvalds 已提交
863 864

	if (retval > 0) {
865 866
		add_rchar(current, retval);
		add_wchar(current, retval);
L
Linus Torvalds 已提交
867 868
	}

869 870
	inc_syscr(current);
	inc_syscw(current);
L
Linus Torvalds 已提交
871 872 873 874 875 876 877 878 879 880 881
	if (*ppos > max)
		retval = -EOVERFLOW;

fput_out:
	fput_light(out_file, fput_needed_out);
fput_in:
	fput_light(in_file, fput_needed_in);
out:
	return retval;
}

882
SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
L
Linus Torvalds 已提交
883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
{
	loff_t pos;
	off_t off;
	ssize_t ret;

	if (offset) {
		if (unlikely(get_user(off, offset)))
			return -EFAULT;
		pos = off;
		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
		if (unlikely(put_user(pos, offset)))
			return -EFAULT;
		return ret;
	}

	return do_sendfile(out_fd, in_fd, NULL, count, 0);
}

901
SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
L
Linus Torvalds 已提交
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916
{
	loff_t pos;
	ssize_t ret;

	if (offset) {
		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
			return -EFAULT;
		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
		if (unlikely(put_user(pos, offset)))
			return -EFAULT;
		return ret;
	}

	return do_sendfile(out_fd, in_fd, NULL, count, 0);
}