splice.c 21.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * "splice": joining two ropes together by interweaving their strands.
 *
 * This is the "extended pipe" functionality, where a pipe is used as
 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
 * buffer that you can use to transfer data from one end to the other.
 *
 * The traditional unix read/write is extended with a "splice()" operation
 * that transfers data buffers to or from a pipe buffer.
 *
 * Named by Larry McVoy, original implementation from Linus, extended by
 * Jens to support splicing to files and fixing the initial implementation
 * bugs.
 *
 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
 * Copyright (C) 2005 Linus Torvalds <torvalds@osdl.org>
 *
 */
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>
#include <linux/pipe_fs_i.h>
#include <linux/mm_inline.h>
24
#include <linux/swap.h>
25 26
#include <linux/writeback.h>
#include <linux/buffer_head.h>
J
Jeff Garzik 已提交
27
#include <linux/module.h>
28
#include <linux/syscalls.h>
29 30 31 32 33 34 35 36 37 38 39

/*
 * Passed to the actors
 */
struct splice_desc {
	unsigned int len, total_len;	/* current and remaining length */
	unsigned int flags;		/* splice flags */
	struct file *file;		/* file to read/write */
	loff_t pos;			/* file position */
};

40 41 42 43 44 45
/*
 * Attempt to steal a page from a pipe buffer. This should perhaps go into
 * a vm helper function, it's already simplified quite a bit by the
 * addition of remove_mapping(). If success is returned, the caller may
 * attempt to reuse this page for another destination.
 */
46 47 48 49
static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
50
	struct address_space *mapping = page_mapping(page);
51 52 53 54

	WARN_ON(!PageLocked(page));
	WARN_ON(!PageUptodate(page));

55 56 57 58 59 60 61 62 63
	/*
	 * At least for ext2 with nobh option, we need to wait on writeback
	 * completing on this page, since we'll remove it from the pagecache.
	 * Otherwise truncate wont wait on the page, allowing the disk
	 * blocks to be reused by someone else before we actually wrote our
	 * data to them. fs corruption ensues.
	 */
	wait_on_page_writeback(page);

64 65 66 67
	if (PagePrivate(page))
		try_to_release_page(page, mapping_gfp_mask(mapping));

	if (!remove_mapping(mapping, page))
68 69
		return 1;

70
	buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
71 72 73
	return 0;
}

74 75 76 77 78
static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
					struct pipe_buffer *buf)
{
	page_cache_release(buf->page);
	buf->page = NULL;
79
	buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
80 81 82 83 84 85 86
}

static void *page_cache_pipe_buf_map(struct file *file,
				     struct pipe_inode_info *info,
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
87
	int err;
88 89

	if (!PageUptodate(page)) {
90 91 92 93 94 95 96 97 98 99
		lock_page(page);

		/*
		 * Page got truncated/unhashed. This will cause a 0-byte
		 * splice, if this is the first page
		 */
		if (!page->mapping) {
			err = -ENODATA;
			goto error;
		}
100

101 102 103 104 105 106 107 108 109 110 111
		/*
		 * uh oh, read-error from disk
		 */
		if (!PageUptodate(page)) {
			err = -EIO;
			goto error;
		}

		/*
		 * page is ok afterall, fall through to mapping
		 */
112 113 114
		unlock_page(page);
	}

115 116 117 118
	return kmap(page);
error:
	unlock_page(page);
	return ERR_PTR(err);
119 120 121 122 123 124 125 126 127 128 129 130 131
}

static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
				      struct pipe_buffer *buf)
{
	kunmap(buf->page);
}

static struct pipe_buf_operations page_cache_pipe_buf_ops = {
	.can_merge = 0,
	.map = page_cache_pipe_buf_map,
	.unmap = page_cache_pipe_buf_unmap,
	.release = page_cache_pipe_buf_release,
132
	.steal = page_cache_pipe_buf_steal,
133 134
};

135 136 137 138
/*
 * Pipe output worker. This sets up our pipe format with the page cache
 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
 */
139
static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
140
			    int nr_pages, unsigned long offset,
141
			    unsigned long len, unsigned int flags)
142 143 144 145 146 147 148
{
	int ret, do_wakeup, i;

	ret = 0;
	do_wakeup = 0;
	i = 0;

149 150
	if (pipe->inode)
		mutex_lock(&pipe->inode->i_mutex);
151 152

	for (;;) {
153
		if (!pipe->readers) {
154 155 156 157 158 159
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

160 161
		if (pipe->nrbufs < PIPE_BUFFERS) {
			int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
162
			struct pipe_buffer *buf = pipe->bufs + newbuf;
163 164 165 166 167 168 169 170 171 172 173
			struct page *page = pages[i++];
			unsigned long this_len;

			this_len = PAGE_CACHE_SIZE - offset;
			if (this_len > len)
				this_len = len;

			buf->page = page;
			buf->offset = offset;
			buf->len = this_len;
			buf->ops = &page_cache_pipe_buf_ops;
174 175 176
			pipe->nrbufs++;
			if (pipe->inode)
				do_wakeup = 1;
177 178 179 180 181 182 183 184

			ret += this_len;
			len -= this_len;
			offset = 0;
			if (!--nr_pages)
				break;
			if (!len)
				break;
185
			if (pipe->nrbufs < PIPE_BUFFERS)
186 187 188 189 190
				continue;

			break;
		}

191 192 193 194 195 196
		if (flags & SPLICE_F_NONBLOCK) {
			if (!ret)
				ret = -EAGAIN;
			break;
		}

197 198 199 200 201 202 203
		if (signal_pending(current)) {
			if (!ret)
				ret = -ERESTARTSYS;
			break;
		}

		if (do_wakeup) {
204
			smp_mb();
205 206 207
			if (waitqueue_active(&pipe->wait))
				wake_up_interruptible_sync(&pipe->wait);
			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
208 209 210
			do_wakeup = 0;
		}

211 212 213
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
214 215
	}

216 217
	if (pipe->inode)
		mutex_unlock(&pipe->inode->i_mutex);
218 219

	if (do_wakeup) {
220
		smp_mb();
221 222 223
		if (waitqueue_active(&pipe->wait))
			wake_up_interruptible(&pipe->wait);
		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
224 225 226 227 228 229 230 231
	}

	while (i < nr_pages)
		page_cache_release(pages[i++]);

	return ret;
}

232 233 234
static int
__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
			   size_t len, unsigned int flags)
235 236 237
{
	struct address_space *mapping = in->f_mapping;
	unsigned int offset, nr_pages;
238
	struct page *pages[PIPE_BUFFERS];
239
	struct page *page;
240
	pgoff_t index;
241
	int i, error;
242 243 244 245 246 247 248 249 250

	index = in->f_pos >> PAGE_CACHE_SHIFT;
	offset = in->f_pos & ~PAGE_CACHE_MASK;
	nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;

	if (nr_pages > PIPE_BUFFERS)
		nr_pages = PIPE_BUFFERS;

	/*
251 252 253
	 * initiate read-ahead on this page range. however, don't call into
	 * read-ahead if this is a non-zero offset (we are likely doing small
	 * chunk splice and the page is already there) for a single page.
254
	 */
255 256
	if (!offset || nr_pages > 1)
		do_page_cache_readahead(mapping, in, index, nr_pages);
257 258 259 260

	/*
	 * now fill in the holes
	 */
261
	error = 0;
262
	for (i = 0; i < nr_pages; i++, index++) {
263
find_page:
264
		/*
265
		 * lookup the page for this index
266
		 */
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
		page = find_get_page(mapping, index);
		if (!page) {
			/*
			 * If in nonblock mode then dont block on
			 * readpage (we've kicked readahead so there
			 * will be asynchronous progress):
			 */
			if (flags & SPLICE_F_NONBLOCK)
				break;

			/*
			 * page didn't exist, allocate one
			 */
			page = page_cache_alloc_cold(mapping);
			if (!page)
				break;

			error = add_to_page_cache_lru(page, mapping, index,
						mapping_gfp_mask(mapping));
			if (unlikely(error)) {
				page_cache_release(page);
				break;
			}

			goto readpage;
		}

		/*
		 * If the page isn't uptodate, we may need to start io on it
		 */
		if (!PageUptodate(page)) {
			lock_page(page);

			/*
			 * page was truncated, stop here. if this isn't the
			 * first page, we'll just complete what we already
			 * added
			 */
			if (!page->mapping) {
				unlock_page(page);
				page_cache_release(page);
				break;
			}
			/*
			 * page was already under io and is now done, great
			 */
			if (PageUptodate(page)) {
				unlock_page(page);
				goto fill_it;
			}
317

318 319 320 321 322
readpage:
			/*
			 * need to read in the page
			 */
			error = mapping->a_ops->readpage(in, page);
323 324 325

			if (unlikely(error)) {
				page_cache_release(page);
326 327
				if (error == AOP_TRUNCATED_PAGE)
					goto find_page;
328 329 330
				break;
			}
		}
331
fill_it:
332
		pages[i] = page;
333 334
	}

335 336
	if (i)
		return move_to_pipe(pipe, pages, i, offset, len, flags);
337

338
	return error;
339 340
}

341 342 343 344 345 346 347 348 349 350
/**
 * generic_file_splice_read - splice data from file to a pipe
 * @in:		file to splice from
 * @pipe:	pipe to splice to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Will read pages from given file and fill them into a pipe.
 *
 */
351
ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
352 353 354 355 356 357 358
				 size_t len, unsigned int flags)
{
	ssize_t spliced;
	int ret;

	ret = 0;
	spliced = 0;
359

360
	while (len) {
361
		ret = __generic_file_splice_read(in, pipe, len, flags);
362 363 364 365 366 367 368

		if (ret <= 0)
			break;

		in->f_pos += ret;
		len -= ret;
		spliced += ret;
369 370 371 372 373

		if (!(flags & SPLICE_F_NONBLOCK))
			continue;
		ret = -EAGAIN;
		break;
374 375 376 377 378 379 380 381
	}

	if (spliced)
		return spliced;

	return ret;
}

382 383
EXPORT_SYMBOL(generic_file_splice_read);

384
/*
385 386
 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
 * using sendpage().
387 388 389 390 391 392 393 394 395
 */
static int pipe_to_sendpage(struct pipe_inode_info *info,
			    struct pipe_buffer *buf, struct splice_desc *sd)
{
	struct file *file = sd->file;
	loff_t pos = sd->pos;
	unsigned int offset;
	ssize_t ret;
	void *ptr;
396
	int more;
397 398 399 400 401 402 403 404 405 406 407 408

	/*
	 * sub-optimal, but we are limited by the pipe ->map. we don't
	 * need a kmap'ed buffer here, we just want to make sure we
	 * have the page pinned if the pipe page originates from the
	 * page cache
	 */
	ptr = buf->ops->map(file, info, buf);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	offset = pos & ~PAGE_CACHE_MASK;
409
	more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
410

411
	ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more);
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432

	buf->ops->unmap(info, buf);
	if (ret == sd->len)
		return 0;

	return -EIO;
}

/*
 * This is a little more tricky than the file -> pipe splicing. There are
 * basically three cases:
 *
 *	- Destination page already exists in the address space and there
 *	  are users of it. For that case we have no other option that
 *	  copying the data. Tough luck.
 *	- Destination page already exists in the address space, but there
 *	  are no users of it. Make sure it's uptodate, then drop it. Fall
 *	  through to last case.
 *	- Destination page does not exist, we can add the pipe page to
 *	  the page cache and avoid the copy.
 *
433 434 435 436 437 438
 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
 * sd->flags), we attempt to migrate pages from the pipe to the output
 * file address space page cache. This is possible if no one else has
 * the pipe page referenced outside of the pipe and page cache. If
 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
 * a new page in the output file page cache and fill/dirty that.
439 440 441 442 443 444
 */
static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
			struct splice_desc *sd)
{
	struct file *file = sd->file;
	struct address_space *mapping = file->f_mapping;
445
	gfp_t gfp_mask = mapping_gfp_mask(mapping);
446 447 448
	unsigned int offset;
	struct page *page;
	pgoff_t index;
449
	char *src;
450
	int ret;
451 452

	/*
453
	 * make sure the data in this buffer is uptodate
454 455 456 457 458 459 460 461 462
	 */
	src = buf->ops->map(file, info, buf);
	if (IS_ERR(src))
		return PTR_ERR(src);

	index = sd->pos >> PAGE_CACHE_SHIFT;
	offset = sd->pos & ~PAGE_CACHE_MASK;

	/*
463
	 * reuse buf page, if SPLICE_F_MOVE is set
464
	 */
465
	if (sd->flags & SPLICE_F_MOVE) {
466 467 468 469
		/*
		 * If steal succeeds, buf->page is now pruned from the vm
		 * side (LRU and page cache) and we can reuse it.
		 */
470 471 472
		if (buf->ops->steal(info, buf))
			goto find_page;

473 474 475
		/*
		 * this will also set the page locked
		 */
476
		page = buf->page;
477
		if (add_to_page_cache(page, mapping, index, gfp_mask))
478
			goto find_page;
479 480 481

		if (!(buf->flags & PIPE_BUF_FLAG_LRU))
			lru_cache_add(page);
482 483 484
	} else {
find_page:
		ret = -ENOMEM;
485
		page = find_or_create_page(mapping, index, gfp_mask);
486
		if (!page)
487
			goto out_nomem;
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512

		/*
		 * If the page is uptodate, it is also locked. If it isn't
		 * uptodate, we can mark it uptodate if we are filling the
		 * full page. Otherwise we need to read it in first...
		 */
		if (!PageUptodate(page)) {
			if (sd->len < PAGE_CACHE_SIZE) {
				ret = mapping->a_ops->readpage(file, page);
				if (unlikely(ret))
					goto out;

				lock_page(page);

				if (!PageUptodate(page)) {
					/*
					 * page got invalidated, repeat
					 */
					if (!page->mapping) {
						unlock_page(page);
						page_cache_release(page);
						goto find_page;
					}
					ret = -EIO;
					goto out;
513
				}
514 515 516
			} else {
				WARN_ON(!PageLocked(page));
				SetPageUptodate(page);
517 518 519 520 521
			}
		}
	}

	ret = mapping->a_ops->prepare_write(file, page, 0, sd->len);
522 523 524 525
	if (ret == AOP_TRUNCATED_PAGE) {
		page_cache_release(page);
		goto find_page;
	} else if (ret)
526 527
		goto out;

528
	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
529 530 531 532 533 534
		char *dst = kmap_atomic(page, KM_USER0);

		memcpy(dst + offset, src + buf->offset, sd->len);
		flush_dcache_page(page);
		kunmap_atomic(dst, KM_USER0);
	}
535 536

	ret = mapping->a_ops->commit_write(file, page, 0, sd->len);
537 538 539 540
	if (ret == AOP_TRUNCATED_PAGE) {
		page_cache_release(page);
		goto find_page;
	} else if (ret)
541 542
		goto out;

543
	mark_page_accessed(page);
544
	balance_dirty_pages_ratelimited(mapping);
545
out:
546
	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
547
		page_cache_release(page);
548 549
		unlock_page(page);
	}
550
out_nomem:
551 552 553 554 555 556 557
	buf->ops->unmap(info, buf);
	return ret;
}

typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
			   struct splice_desc *);

558 559 560 561 562
/*
 * Pipe input worker. Most of this logic works like a regular pipe, the
 * key here is the 'actor' worker passed in that actually moves the data
 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
 */
563
static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
564 565 566 567 568 569 570 571 572 573 574 575 576 577
			      size_t len, unsigned int flags,
			      splice_actor *actor)
{
	int ret, do_wakeup, err;
	struct splice_desc sd;

	ret = 0;
	do_wakeup = 0;

	sd.total_len = len;
	sd.flags = flags;
	sd.file = out;
	sd.pos = out->f_pos;

578 579
	if (pipe->inode)
		mutex_lock(&pipe->inode->i_mutex);
580 581

	for (;;) {
582 583
		if (pipe->nrbufs) {
			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
584 585 586 587 588 589
			struct pipe_buf_operations *ops = buf->ops;

			sd.len = buf->len;
			if (sd.len > sd.total_len)
				sd.len = sd.total_len;

590
			err = actor(pipe, buf, &sd);
591 592 593 594 595 596 597 598 599 600 601 602
			if (err) {
				if (!ret && err != -ENODATA)
					ret = err;

				break;
			}

			ret += sd.len;
			buf->offset += sd.len;
			buf->len -= sd.len;
			if (!buf->len) {
				buf->ops = NULL;
603
				ops->release(pipe, buf);
604 605 606 607
				pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
				pipe->nrbufs--;
				if (pipe->inode)
					do_wakeup = 1;
608 609 610 611 612 613 614 615
			}

			sd.pos += sd.len;
			sd.total_len -= sd.len;
			if (!sd.total_len)
				break;
		}

616
		if (pipe->nrbufs)
617
			continue;
618
		if (!pipe->writers)
619
			break;
620
		if (!pipe->waiting_writers) {
621 622 623 624
			if (ret)
				break;
		}

625 626 627 628 629 630
		if (flags & SPLICE_F_NONBLOCK) {
			if (!ret)
				ret = -EAGAIN;
			break;
		}

631 632 633 634 635 636 637
		if (signal_pending(current)) {
			if (!ret)
				ret = -ERESTARTSYS;
			break;
		}

		if (do_wakeup) {
638
			smp_mb();
639 640 641
			if (waitqueue_active(&pipe->wait))
				wake_up_interruptible_sync(&pipe->wait);
			kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
642 643 644
			do_wakeup = 0;
		}

645
		pipe_wait(pipe);
646 647
	}

648 649
	if (pipe->inode)
		mutex_unlock(&pipe->inode->i_mutex);
650 651

	if (do_wakeup) {
652
		smp_mb();
653 654 655
		if (waitqueue_active(&pipe->wait))
			wake_up_interruptible(&pipe->wait);
		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
656 657 658 659 660 661 662
	}

	out->f_pos = sd.pos;
	return ret;

}

663 664
/**
 * generic_file_splice_write - splice data from a pipe to a file
665
 * @pipe:	pipe info
666 667 668 669 670 671 672 673
 * @out:	file to write to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Will either move or copy pages (determined by @flags options) from
 * the given pipe inode to the given file.
 *
 */
674 675 676
ssize_t
generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
			  size_t len, unsigned int flags)
677
{
678
	struct address_space *mapping = out->f_mapping;
679 680 681
	ssize_t ret;

	ret = move_from_pipe(pipe, out, len, flags, pipe_to_file);
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700

	/*
	 * if file or inode is SYNC and we actually wrote some data, sync it
	 */
	if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
	    && ret > 0) {
		struct inode *inode = mapping->host;
		int err;

		mutex_lock(&inode->i_mutex);
		err = generic_osync_inode(mapping->host, mapping,
						OSYNC_METADATA|OSYNC_DATA);
		mutex_unlock(&inode->i_mutex);

		if (err)
			ret = err;
	}

	return ret;
701 702
}

703 704
EXPORT_SYMBOL(generic_file_splice_write);

705 706 707 708 709 710 711 712 713 714 715
/**
 * generic_splice_sendpage - splice data from a pipe to a socket
 * @inode:	pipe inode
 * @out:	socket to write to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Will send @len bytes from the pipe to a network socket. No data copying
 * is involved.
 *
 */
716
ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
717 718
				size_t len, unsigned int flags)
{
719
	return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage);
720 721
}

722
EXPORT_SYMBOL(generic_splice_sendpage);
J
Jeff Garzik 已提交
723

724 725 726
/*
 * Attempt to initiate a splice from pipe to file.
 */
727
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
728
			   size_t len, unsigned int flags)
729 730 731 732 733 734 735 736 737 738 739
{
	loff_t pos;
	int ret;

	if (!out->f_op || !out->f_op->splice_write)
		return -EINVAL;

	if (!(out->f_mode & FMODE_WRITE))
		return -EBADF;

	pos = out->f_pos;
740

741 742 743 744 745 746 747
	ret = rw_verify_area(WRITE, out, &pos, len);
	if (unlikely(ret < 0))
		return ret;

	return out->f_op->splice_write(pipe, out, len, flags);
}

748 749 750
/*
 * Attempt to initiate a splice from a file to a pipe.
 */
751 752
static long do_splice_to(struct file *in, struct pipe_inode_info *pipe,
			 size_t len, unsigned int flags)
753 754 755 756 757 758 759 760 761 762 763
{
	loff_t pos, isize, left;
	int ret;

	if (!in->f_op || !in->f_op->splice_read)
		return -EINVAL;

	if (!(in->f_mode & FMODE_READ))
		return -EBADF;

	pos = in->f_pos;
764

765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
	ret = rw_verify_area(READ, in, &pos, len);
	if (unlikely(ret < 0))
		return ret;

	isize = i_size_read(in->f_mapping->host);
	if (unlikely(in->f_pos >= isize))
		return 0;
	
	left = isize - in->f_pos;
	if (left < len)
		len = left;

	return in->f_op->splice_read(in, pipe, len, flags);
}

780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
long do_splice_direct(struct file *in, struct file *out, size_t len,
		      unsigned int flags)
{
	struct pipe_inode_info *pipe;
	long ret, bytes;
	umode_t i_mode;
	int i;

	/*
	 * We require the input being a regular file, as we don't want to
	 * randomly drop data for eg socket -> socket splicing. Use the
	 * piped splicing for that!
	 */
	i_mode = in->f_dentry->d_inode->i_mode;
	if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
		return -EINVAL;

	/*
	 * neither in nor out is a pipe, setup an internal pipe attached to
	 * 'out' and transfer the wanted data from 'in' to 'out' through that
	 */
	pipe = current->splice_pipe;
	if (!pipe) {
		pipe = alloc_pipe_info(NULL);
		if (!pipe)
			return -ENOMEM;

		/*
		 * We don't have an immediate reader, but we'll read the stuff
		 * out of the pipe right after the move_to_pipe(). So set
		 * PIPE_READERS appropriately.
		 */
		pipe->readers = 1;

		current->splice_pipe = pipe;
	}

	/*
	 * do the splice
	 */
	ret = 0;
	bytes = 0;

	while (len) {
		size_t read_len, max_read_len;

		/*
		 * Do at most PIPE_BUFFERS pages worth of transfer:
		 */
		max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));

		ret = do_splice_to(in, pipe, max_read_len, flags);
		if (unlikely(ret < 0))
			goto out_release;

		read_len = ret;

		/*
		 * NOTE: nonblocking mode only applies to the input. We
		 * must not do the output in nonblocking mode as then we
		 * could get stuck data in the internal pipe:
		 */
		ret = do_splice_from(pipe, out, read_len,
				     flags & ~SPLICE_F_NONBLOCK);
		if (unlikely(ret < 0))
			goto out_release;

		bytes += ret;
		len -= ret;

		/*
		 * In nonblocking mode, if we got back a short read then
		 * that was due to either an IO error or due to the
		 * pagecache entry not being there. In the IO error case
		 * the _next_ splice attempt will produce a clean IO error
		 * return value (not a short read), so in both cases it's
		 * correct to break out of the loop here:
		 */
		if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
			break;
	}

	pipe->nrbufs = pipe->curbuf = 0;

	return bytes;

out_release:
	/*
	 * If we did an incomplete transfer we must release
	 * the pipe buffers in question:
	 */
	for (i = 0; i < PIPE_BUFFERS; i++) {
		struct pipe_buffer *buf = pipe->bufs + i;

		if (buf->ops) {
			buf->ops->release(pipe, buf);
			buf->ops = NULL;
		}
	}
	pipe->nrbufs = pipe->curbuf = 0;

	/*
	 * If we transferred some data, return the number of bytes:
	 */
	if (bytes > 0)
		return bytes;

	return ret;
}

EXPORT_SYMBOL(do_splice_direct);

892 893 894
/*
 * Determine where to splice to/from.
 */
895 896 897
static long do_splice(struct file *in, loff_t __user *off_in,
		      struct file *out, loff_t __user *off_out,
		      size_t len, unsigned int flags)
898
{
899
	struct pipe_inode_info *pipe;
900

901
	pipe = in->f_dentry->d_inode->i_pipe;
902 903 904
	if (pipe) {
		if (off_in)
			return -ESPIPE;
905 906 907 908 909 910 911
		if (off_out) {
			if (out->f_op->llseek == no_llseek)
				return -EINVAL;
			if (copy_from_user(&out->f_pos, off_out,
					   sizeof(loff_t)))
				return -EFAULT;
		}
912

913
		return do_splice_from(pipe, out, len, flags);
914
	}
915

916
	pipe = out->f_dentry->d_inode->i_pipe;
917 918 919
	if (pipe) {
		if (off_out)
			return -ESPIPE;
920 921 922 923 924 925
		if (off_in) {
			if (in->f_op->llseek == no_llseek)
				return -EINVAL;
			if (copy_from_user(&in->f_pos, off_in, sizeof(loff_t)))
				return -EFAULT;
		}
926

927
		return do_splice_to(in, pipe, len, flags);
928
	}
929 930 931 932

	return -EINVAL;
}

933 934 935
asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
			   int fd_out, loff_t __user *off_out,
			   size_t len, unsigned int flags)
936 937 938 939 940 941 942 943 944
{
	long error;
	struct file *in, *out;
	int fput_in, fput_out;

	if (unlikely(!len))
		return 0;

	error = -EBADF;
945
	in = fget_light(fd_in, &fput_in);
946 947
	if (in) {
		if (in->f_mode & FMODE_READ) {
948
			out = fget_light(fd_out, &fput_out);
949 950
			if (out) {
				if (out->f_mode & FMODE_WRITE)
951 952 953
					error = do_splice(in, off_in,
							  out, off_out,
							  len, flags);
954 955 956 957 958 959 960 961 962
				fput_light(out, fput_out);
			}
		}

		fput_light(in, fput_in);
	}

	return error;
}