splice.c 44.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * "splice": joining two ropes together by interweaving their strands.
 *
 * This is the "extended pipe" functionality, where a pipe is used as
 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
 * buffer that you can use to transfer data from one end to the other.
 *
 * The traditional unix read/write is extended with a "splice()" operation
 * that transfers data buffers to or from a pipe buffer.
 *
 * Named by Larry McVoy, original implementation from Linus, extended by
12 13
 * Jens to support splicing to files, network, direct splicing, etc and
 * fixing lots of bugs.
14
 *
15
 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
16 17
 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
18 19 20 21 22
 *
 */
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>
23
#include <linux/splice.h>
K
KAMEZAWA Hiroyuki 已提交
24
#include <linux/memcontrol.h>
25
#include <linux/mm_inline.h>
26
#include <linux/swap.h>
27
#include <linux/writeback.h>
28
#include <linux/export.h>
29
#include <linux/syscalls.h>
30
#include <linux/uio.h>
31
#include <linux/security.h>
32
#include <linux/gfp.h>
33
#include <linux/socket.h>
34
#include <linux/compat.h>
35
#include "internal.h"
36

37 38 39 40 41 42
/*
 * Attempt to steal a page from a pipe buffer. This should perhaps go into
 * a vm helper function, it's already simplified quite a bit by the
 * addition of remove_mapping(). If success is returned, the caller may
 * attempt to reuse this page for another destination.
 */
43
static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
44 45 46
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
47
	struct address_space *mapping;
48

49 50
	lock_page(page);

51 52 53
	mapping = page_mapping(page);
	if (mapping) {
		WARN_ON(!PageUptodate(page));
54

55 56 57 58 59 60 61 62 63
		/*
		 * At least for ext2 with nobh option, we need to wait on
		 * writeback completing on this page, since we'll remove it
		 * from the pagecache.  Otherwise truncate wont wait on the
		 * page, allowing the disk blocks to be reused by someone else
		 * before we actually wrote our data to them. fs corruption
		 * ensues.
		 */
		wait_on_page_writeback(page);
64

65 66
		if (page_has_private(page) &&
		    !try_to_release_page(page, GFP_KERNEL))
67
			goto out_unlock;
68

69 70 71 72 73 74 75 76
		/*
		 * If we succeeded in removing the mapping, set LRU flag
		 * and return good.
		 */
		if (remove_mapping(mapping, page)) {
			buf->flags |= PIPE_BUF_FLAG_LRU;
			return 0;
		}
77
	}
78

79 80 81 82
	/*
	 * Raced with truncate or failed to remove page from current
	 * address space, unlock and return failure.
	 */
83
out_unlock:
84 85
	unlock_page(page);
	return 1;
86 87
}

88
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
89 90
					struct pipe_buffer *buf)
{
91
	put_page(buf->page);
J
Jens Axboe 已提交
92
	buf->flags &= ~PIPE_BUF_FLAG_LRU;
93 94
}

95 96 97 98
/*
 * Check whether the contents of buf is OK to access. Since the content
 * is a page cache page, IO may be in flight.
 */
99 100
static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
				       struct pipe_buffer *buf)
101 102
{
	struct page *page = buf->page;
103
	int err;
104 105

	if (!PageUptodate(page)) {
106 107 108 109
		lock_page(page);

		/*
		 * Page got truncated/unhashed. This will cause a 0-byte
I
Ingo Molnar 已提交
110
		 * splice, if this is the first page.
111 112 113 114 115
		 */
		if (!page->mapping) {
			err = -ENODATA;
			goto error;
		}
116

117
		/*
I
Ingo Molnar 已提交
118
		 * Uh oh, read-error from disk.
119 120 121 122 123 124 125
		 */
		if (!PageUptodate(page)) {
			err = -EIO;
			goto error;
		}

		/*
126
		 * Page is ok afterall, we are done.
127
		 */
128 129 130
		unlock_page(page);
	}

131
	return 0;
132 133
error:
	unlock_page(page);
134
	return err;
135 136
}

137
const struct pipe_buf_operations page_cache_pipe_buf_ops = {
138
	.can_merge = 0,
139
	.confirm = page_cache_pipe_buf_confirm,
140
	.release = page_cache_pipe_buf_release,
141
	.steal = page_cache_pipe_buf_steal,
142
	.get = generic_pipe_buf_get,
143 144
};

145 146 147
static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
148 149 150
	if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
		return 1;

J
Jens Axboe 已提交
151
	buf->flags |= PIPE_BUF_FLAG_LRU;
152
	return generic_pipe_buf_steal(pipe, buf);
153 154
}

155
static const struct pipe_buf_operations user_page_pipe_buf_ops = {
156
	.can_merge = 0,
157
	.confirm = generic_pipe_buf_confirm,
158 159
	.release = page_cache_pipe_buf_release,
	.steal = user_page_pipe_buf_steal,
160
	.get = generic_pipe_buf_get,
161 162
};

N
Namhyung Kim 已提交
163 164 165 166 167 168 169 170
static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
}

171 172 173 174 175 176
/**
 * splice_to_pipe - fill passed data into a pipe
 * @pipe:	pipe to fill
 * @spd:	data to fill
 *
 * Description:
R
Randy Dunlap 已提交
177
 *    @spd contains a map of pages and len/offset tuples, along with
178 179 180
 *    the struct pipe_buf_operations associated with these pages. This
 *    function will link that data to the pipe.
 *
181
 */
182 183
ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
		       struct splice_pipe_desc *spd)
184
{
185
	unsigned int spd_pages = spd->nr_pages;
186
	int ret = 0, page_nr = 0;
187

188 189 190
	if (!spd_pages)
		return 0;

191 192 193 194 195
	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
		goto out;
	}
196

197 198 199
	while (pipe->nrbufs < pipe->buffers) {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		struct pipe_buffer *buf = pipe->bufs + newbuf;
200

201 202 203 204 205
		buf->page = spd->pages[page_nr];
		buf->offset = spd->partial[page_nr].offset;
		buf->len = spd->partial[page_nr].len;
		buf->private = spd->partial[page_nr].private;
		buf->ops = spd->ops;
206

207 208 209
		pipe->nrbufs++;
		page_nr++;
		ret += buf->len;
210

211
		if (!--spd->nr_pages)
212 213 214
			break;
	}

215 216
	if (!ret)
		ret = -EAGAIN;
217

218
out:
219
	while (page_nr < spd_pages)
220
		spd->spd_release(spd, page_nr++);
221 222 223

	return ret;
}
224
EXPORT_SYMBOL_GPL(splice_to_pipe);
225

A
Al Viro 已提交
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
{
	int ret;

	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
	} else if (pipe->nrbufs == pipe->buffers) {
		ret = -EAGAIN;
	} else {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		pipe->bufs[newbuf] = *buf;
		pipe->nrbufs++;
		return buf->len;
	}
	buf->ops->release(pipe, buf);
	buf->ops = NULL;
	return ret;
}
EXPORT_SYMBOL(add_to_pipe);

247
void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
248
{
249
	put_page(spd->pages[i]);
250 251
}

252 253 254 255
/*
 * Check if we need to grow the arrays holding pages and partial page
 * descriptions.
 */
256
int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
257
{
258 259 260 261
	unsigned int buffers = ACCESS_ONCE(pipe->buffers);

	spd->nr_pages_max = buffers;
	if (buffers <= PIPE_DEF_BUFFERS)
262 263
		return 0;

264 265
	spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
	spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
266 267 268 269 270 271 272 273 274

	if (spd->pages && spd->partial)
		return 0;

	kfree(spd->pages);
	kfree(spd->partial);
	return -ENOMEM;
}

275
void splice_shrink_spd(struct splice_pipe_desc *spd)
276
{
277
	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
278 279 280 281 282 283
		return;

	kfree(spd->pages);
	kfree(spd->partial);
}

284
static int
285 286 287
__generic_file_splice_read(struct file *in, loff_t *ppos,
			   struct pipe_inode_info *pipe, size_t len,
			   unsigned int flags)
288 289
{
	struct address_space *mapping = in->f_mapping;
F
Fengguang Wu 已提交
290
	unsigned int loff, nr_pages, req_pages;
291 292
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
293
	struct page *page;
294 295
	pgoff_t index, end_index;
	loff_t isize;
296
	int error, page_nr;
297 298 299
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
300
		.nr_pages_max = PIPE_DEF_BUFFERS,
301 302
		.flags = flags,
		.ops = &page_cache_pipe_buf_ops,
303
		.spd_release = spd_release_page,
304
	};
305

306 307 308
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

309 310 311
	index = *ppos >> PAGE_SHIFT;
	loff = *ppos & ~PAGE_MASK;
	req_pages = (len + loff + PAGE_SIZE - 1) >> PAGE_SHIFT;
312
	nr_pages = min(req_pages, spd.nr_pages_max);
313

314 315 316
	/*
	 * Lookup the (hopefully) full range of pages we need.
	 */
317
	spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
318
	index += spd.nr_pages;
319

320 321
	/*
	 * If find_get_pages_contig() returned fewer pages than we needed,
322
	 * readahead/allocate the rest and fill in the holes.
323
	 */
324
	if (spd.nr_pages < nr_pages)
325 326
		page_cache_sync_readahead(mapping, &in->f_ra, in,
				index, req_pages - spd.nr_pages);
327

328
	error = 0;
329
	while (spd.nr_pages < nr_pages) {
330
		/*
331 332
		 * Page could be there, find_get_pages_contig() breaks on
		 * the first hole.
333
		 */
334 335 336
		page = find_get_page(mapping, index);
		if (!page) {
			/*
337
			 * page didn't exist, allocate one.
338 339 340 341 342 343
			 */
			page = page_cache_alloc_cold(mapping);
			if (!page)
				break;

			error = add_to_page_cache_lru(page, mapping, index,
344
				   mapping_gfp_constraint(mapping, GFP_KERNEL));
345
			if (unlikely(error)) {
346
				put_page(page);
347 348
				if (error == -EEXIST)
					continue;
349 350
				break;
			}
351 352 353 354 355
			/*
			 * add_to_page_cache() locks the page, unlock it
			 * to avoid convoluting the logic below even more.
			 */
			unlock_page(page);
356 357
		}

358
		spd.pages[spd.nr_pages++] = page;
359 360 361 362 363 364 365
		index++;
	}

	/*
	 * Now loop over the map and see if we need to start IO on any
	 * pages, fill in the partial map, etc.
	 */
366
	index = *ppos >> PAGE_SHIFT;
367 368 369 370 371 372 373 374 375 376 377
	nr_pages = spd.nr_pages;
	spd.nr_pages = 0;
	for (page_nr = 0; page_nr < nr_pages; page_nr++) {
		unsigned int this_len;

		if (!len)
			break;

		/*
		 * this_len is the max we'll use from this page
		 */
378
		this_len = min_t(unsigned long, len, PAGE_SIZE - loff);
379
		page = spd.pages[page_nr];
380

381
		if (PageReadahead(page))
382
			page_cache_async_readahead(mapping, &in->f_ra, in,
F
Fengguang Wu 已提交
383
					page, index, req_pages - page_nr);
384

385 386 387 388
		/*
		 * If the page isn't uptodate, we may need to start io on it
		 */
		if (!PageUptodate(page)) {
389
			lock_page(page);
390 391

			/*
392 393 394 395
			 * Page was truncated, or invalidated by the
			 * filesystem.  Redo the find/create, but this time the
			 * page is kept locked, so there's no chance of another
			 * race with truncate/invalidate.
396 397 398
			 */
			if (!page->mapping) {
				unlock_page(page);
399
retry_lookup:
400 401 402 403 404 405 406
				page = find_or_create_page(mapping, index,
						mapping_gfp_mask(mapping));

				if (!page) {
					error = -ENOMEM;
					break;
				}
407
				put_page(spd.pages[page_nr]);
408
				spd.pages[page_nr] = page;
409 410 411 412 413 414 415 416
			}
			/*
			 * page was already under io and is now done, great
			 */
			if (PageUptodate(page)) {
				unlock_page(page);
				goto fill_it;
			}
417

418 419 420 421
			/*
			 * need to read in the page
			 */
			error = mapping->a_ops->readpage(in, page);
422
			if (unlikely(error)) {
423
				/*
424
				 * Re-lookup the page
425
				 */
426
				if (error == AOP_TRUNCATED_PAGE)
427
					goto retry_lookup;
428

429 430
				break;
			}
431 432 433 434 435 436
		}
fill_it:
		/*
		 * i_size must be checked after PageUptodate.
		 */
		isize = i_size_read(mapping->host);
437
		end_index = (isize - 1) >> PAGE_SHIFT;
438 439 440 441 442 443 444 445 446
		if (unlikely(!isize || index > end_index))
			break;

		/*
		 * if this is the last page, see if we need to shrink
		 * the length and stop
		 */
		if (end_index == index) {
			unsigned int plen;
447 448

			/*
449
			 * max good bytes in this page
450
			 */
451
			plen = ((isize - 1) & ~PAGE_MASK) + 1;
452
			if (plen <= loff)
453 454 455
				break;

			/*
456
			 * force quit after adding this page
457
			 */
458 459
			this_len = min(this_len, plen - loff);
			len = this_len;
460
		}
461

462 463
		spd.partial[page_nr].offset = loff;
		spd.partial[page_nr].len = this_len;
464
		len -= this_len;
465
		loff = 0;
466 467
		spd.nr_pages++;
		index++;
468 469
	}

470
	/*
471
	 * Release any pages at the end, if we quit early. 'page_nr' is how far
472 473 474
	 * we got, 'nr_pages' is how many pages are in the map.
	 */
	while (page_nr < nr_pages)
475 476
		put_page(spd.pages[page_nr++]);
	in->f_ra.prev_pos = (loff_t)index << PAGE_SHIFT;
477

478
	if (spd.nr_pages)
479
		error = splice_to_pipe(pipe, &spd);
480

481
	splice_shrink_spd(&spd);
482
	return error;
483 484
}

485 486 487
/**
 * generic_file_splice_read - splice data from file to a pipe
 * @in:		file to splice from
488
 * @ppos:	position in @in
489 490 491 492
 * @pipe:	pipe to splice to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
493 494 495 496 497
 * Description:
 *    Will read pages from given file and fill them into a pipe. Can be
 *    used as long as the address_space operations for the source implements
 *    a readpage() hook.
 *
498
 */
499 500 501
ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
502
{
503
	loff_t isize, left;
504
	int ret;
505

506 507 508
	if (IS_DAX(in->f_mapping->host))
		return default_file_splice_read(in, ppos, pipe, len, flags);

509 510 511 512 513 514 515
	isize = i_size_read(in->f_mapping->host);
	if (unlikely(*ppos >= isize))
		return 0;

	left = isize - *ppos;
	if (unlikely(left < len))
		len = left;
516

517
	ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
518
	if (ret > 0) {
519
		*ppos += ret;
520 521
		file_accessed(in);
	}
522 523 524

	return ret;
}
525 526
EXPORT_SYMBOL(generic_file_splice_read);

527 528 529 530 531 532 533 534
static const struct pipe_buf_operations default_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};

M
Miklos Szeredi 已提交
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	return 1;
}

/* Pipe buffer operations for a socket and similar. */
const struct pipe_buf_operations nosteal_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_nosteal,
	.get = generic_pipe_buf_get,
};
EXPORT_SYMBOL(nosteal_pipe_buf_ops);

551 552 553 554 555 556 557 558 559 560
static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
			    unsigned long vlen, loff_t offset)
{
	mm_segment_t old_fs;
	loff_t pos = offset;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
561
	res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0);
562 563 564 565 566
	set_fs(old_fs);

	return res;
}

567
ssize_t kernel_write(struct file *file, const char *buf, size_t count,
568
			    loff_t pos)
569 570 571 572 573 574 575
{
	mm_segment_t old_fs;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
576
	res = vfs_write(file, (__force const char __user *)buf, count, &pos);
577 578 579 580
	set_fs(old_fs);

	return res;
}
581
EXPORT_SYMBOL(kernel_write);
582

583 584 585 586 587 588 589
ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
{
	unsigned int nr_pages;
	unsigned int nr_freed;
	size_t offset;
590 591 592
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
	struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
593 594 595 596 597 598 599
	ssize_t res;
	size_t this_len;
	int error;
	int i;
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
600
		.nr_pages_max = PIPE_DEF_BUFFERS,
601 602 603 604 605
		.flags = flags,
		.ops = &default_pipe_buf_ops,
		.spd_release = spd_release_page,
	};

606 607 608 609 610
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

	res = -ENOMEM;
	vec = __vec;
611 612
	if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
		vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
613 614 615 616
		if (!vec)
			goto shrink_ret;
	}

617 618
	offset = *ppos & ~PAGE_MASK;
	nr_pages = (len + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
619

620
	for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
621 622
		struct page *page;

623
		page = alloc_page(GFP_USER);
624 625 626 627
		error = -ENOMEM;
		if (!page)
			goto err;

628
		this_len = min_t(size_t, len, PAGE_SIZE - offset);
629
		vec[i].iov_base = (void __user *) page_address(page);
630
		vec[i].iov_len = this_len;
631
		spd.pages[i] = page;
632 633 634 635 636 637
		spd.nr_pages++;
		len -= this_len;
		offset = 0;
	}

	res = kernel_readv(in, vec, spd.nr_pages, *ppos);
A
Andrew Morton 已提交
638 639
	if (res < 0) {
		error = res;
640
		goto err;
A
Andrew Morton 已提交
641
	}
642 643 644 645 646 647 648 649

	error = 0;
	if (!res)
		goto err;

	nr_freed = 0;
	for (i = 0; i < spd.nr_pages; i++) {
		this_len = min_t(size_t, vec[i].iov_len, res);
650 651
		spd.partial[i].offset = 0;
		spd.partial[i].len = this_len;
652
		if (!this_len) {
653 654
			__free_page(spd.pages[i]);
			spd.pages[i] = NULL;
655 656 657 658 659 660 661 662 663 664
			nr_freed++;
		}
		res -= this_len;
	}
	spd.nr_pages -= nr_freed;

	res = splice_to_pipe(pipe, &spd);
	if (res > 0)
		*ppos += res;

665 666 667
shrink_ret:
	if (vec != __vec)
		kfree(vec);
668
	splice_shrink_spd(&spd);
669 670 671
	return res;

err:
672
	for (i = 0; i < spd.nr_pages; i++)
673
		__free_page(spd.pages[i]);
674

675 676
	res = error;
	goto shrink_ret;
677 678 679
}
EXPORT_SYMBOL(default_file_splice_read);

680
/*
681
 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
J
Jens Axboe 已提交
682
 * using sendpage(). Return the number of bytes sent.
683
 */
684
static int pipe_to_sendpage(struct pipe_inode_info *pipe,
685 686
			    struct pipe_buffer *buf, struct splice_desc *sd)
{
687
	struct file *file = sd->u.file;
688
	loff_t pos = sd->pos;
689
	int more;
690

A
Al Viro 已提交
691
	if (!likely(file->f_op->sendpage))
692 693
		return -EINVAL;

694
	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
695 696

	if (sd->len < sd->total_len && pipe->nrbufs > 1)
697
		more |= MSG_SENDPAGE_NOTLAST;
698

699 700
	return file->f_op->sendpage(file, buf->page, buf->offset,
				    sd->len, &pos, more);
701 702
}

703 704 705 706 707 708 709 710
static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
}

711
/**
712
 * splice_from_pipe_feed - feed available data from a pipe to a file
713 714 715 716 717
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
718 719 720 721 722 723 724
 *    This function loops over the pipe and calls @actor to do the
 *    actual moving of a single struct pipe_buffer to the desired
 *    destination.  It returns when there's no more buffers left in
 *    the pipe or if the requested number of bytes (@sd->total_len)
 *    have been copied.  It returns a positive number (one) if the
 *    pipe needs to be filled with more data, zero if the required
 *    number of bytes have been copied and -errno on error.
725
 *
726 727 728 729
 *    This, together with splice_from_pipe_{begin,end,next}, may be
 *    used to implement the functionality of __splice_from_pipe() when
 *    locking is required around copying the pipe buffers to the
 *    destination.
730
 */
A
Al Viro 已提交
731
static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
732
			  splice_actor *actor)
733
{
734
	int ret;
735

736 737 738
	while (pipe->nrbufs) {
		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
		const struct pipe_buf_operations *ops = buf->ops;
739

740 741 742
		sd->len = buf->len;
		if (sd->len > sd->total_len)
			sd->len = sd->total_len;
743

744 745
		ret = buf->ops->confirm(pipe, buf);
		if (unlikely(ret)) {
746 747 748 749
			if (ret == -ENODATA)
				ret = 0;
			return ret;
		}
750 751 752 753 754

		ret = actor(pipe, buf, sd);
		if (ret <= 0)
			return ret;

755 756 757 758 759 760 761 762 763 764 765
		buf->offset += ret;
		buf->len -= ret;

		sd->num_spliced += ret;
		sd->len -= ret;
		sd->pos += ret;
		sd->total_len -= ret;

		if (!buf->len) {
			buf->ops = NULL;
			ops->release(pipe, buf);
766
			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
767
			pipe->nrbufs--;
A
Al Viro 已提交
768
			if (pipe->files)
769 770
				sd->need_wakeup = true;
		}
771

772 773 774
		if (!sd->total_len)
			return 0;
	}
775

776 777
	return 1;
}
778

779 780 781 782 783 784 785 786 787 788
/**
 * splice_from_pipe_next - wait for some data to splice from
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wait for some data and return a positive
 *    value (one) if pipe buffers are available.  It will return zero
 *    or -errno if no more data needs to be spliced.
 */
A
Al Viro 已提交
789
static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
790
{
791 792 793 794 795 796 797
	/*
	 * Check for signal early to make process killable when there are
	 * always buffers available
	 */
	if (signal_pending(current))
		return -ERESTARTSYS;

798 799 800
	while (!pipe->nrbufs) {
		if (!pipe->writers)
			return 0;
J
Jens Axboe 已提交
801

802 803
		if (!pipe->waiting_writers && sd->num_spliced)
			return 0;
I
Ingo Molnar 已提交
804

805 806
		if (sd->flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;
807

808 809
		if (signal_pending(current))
			return -ERESTARTSYS;
810

811 812 813
		if (sd->need_wakeup) {
			wakeup_pipe_writers(pipe);
			sd->need_wakeup = false;
814 815
		}

816 817
		pipe_wait(pipe);
	}
818

819 820
	return 1;
}
821

822 823
/**
 * splice_from_pipe_begin - start splicing from pipe
824
 * @sd:		information about the splice operation
825 826 827 828 829 830
 *
 * Description:
 *    This function should be called before a loop containing
 *    splice_from_pipe_next() and splice_from_pipe_feed() to
 *    initialize the necessary fields of @sd.
 */
A
Al Viro 已提交
831
static void splice_from_pipe_begin(struct splice_desc *sd)
832 833 834 835
{
	sd->num_spliced = 0;
	sd->need_wakeup = false;
}
836

837 838 839 840 841 842 843 844 845 846
/**
 * splice_from_pipe_end - finish splicing from pipe
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wake up pipe writers if necessary.  It should
 *    be called after a loop containing splice_from_pipe_next() and
 *    splice_from_pipe_feed().
 */
A
Al Viro 已提交
847
static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
848 849 850 851
{
	if (sd->need_wakeup)
		wakeup_pipe_writers(pipe);
}
852

853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
/**
 * __splice_from_pipe - splice data from a pipe to given actor
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
 *    This function does little more than loop over the pipe and call
 *    @actor to do the actual moving of a single struct pipe_buffer to
 *    the desired destination. See pipe_to_file, pipe_to_sendpage, or
 *    pipe_to_user.
 *
 */
ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
			   splice_actor *actor)
{
	int ret;
870

871 872
	splice_from_pipe_begin(sd);
	do {
J
Jan Kara 已提交
873
		cond_resched();
874 875 876 877 878 879 880
		ret = splice_from_pipe_next(pipe, sd);
		if (ret > 0)
			ret = splice_from_pipe_feed(pipe, sd, actor);
	} while (ret > 0);
	splice_from_pipe_end(pipe, sd);

	return sd->num_spliced ? sd->num_spliced : ret;
881
}
M
Mark Fasheh 已提交
882
EXPORT_SYMBOL(__splice_from_pipe);
883

884 885 886 887 888 889 890 891 892 893
/**
 * splice_from_pipe - splice data from a pipe to a file
 * @pipe:	pipe to splice from
 * @out:	file to splice to
 * @ppos:	position in @out
 * @len:	how many bytes to splice
 * @flags:	splice modifier flags
 * @actor:	handler that splices the data
 *
 * Description:
894
 *    See __splice_from_pipe. This function locks the pipe inode,
895 896 897
 *    otherwise it's identical to __splice_from_pipe().
 *
 */
898 899 900 901 902
ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
			 loff_t *ppos, size_t len, unsigned int flags,
			 splice_actor *actor)
{
	ssize_t ret;
J
Jens Axboe 已提交
903 904 905 906
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
907
		.u.file = out,
J
Jens Axboe 已提交
908
	};
909

910
	pipe_lock(pipe);
J
Jens Axboe 已提交
911
	ret = __splice_from_pipe(pipe, &sd, actor);
912
	pipe_unlock(pipe);
913 914 915 916

	return ret;
}

A
Al Viro 已提交
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
/**
 * iter_file_splice_write - splice data from a pipe to a file
 * @pipe:	pipe info
 * @out:	file to write to
 * @ppos:	position in @out
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    Will either move or copy pages (determined by @flags options) from
 *    the given pipe inode to the given file.
 *    This one is ->write_iter-based.
 *
 */
ssize_t
iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
			  loff_t *ppos, size_t len, unsigned int flags)
{
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
	};
	int nbufs = pipe->buffers;
	struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
	ssize_t ret;

	if (unlikely(!array))
		return -ENOMEM;

	pipe_lock(pipe);

	splice_from_pipe_begin(&sd);
	while (sd.total_len) {
		struct iov_iter from;
		size_t left;
		int n, idx;

		ret = splice_from_pipe_next(pipe, &sd);
		if (ret <= 0)
			break;

		if (unlikely(nbufs < pipe->buffers)) {
			kfree(array);
			nbufs = pipe->buffers;
			array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
			if (!array) {
				ret = -ENOMEM;
				break;
			}
		}

		/* build the vector */
		left = sd.total_len;
		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {
			struct pipe_buffer *buf = pipe->bufs + idx;
			size_t this_len = buf->len;

			if (this_len > left)
				this_len = left;

			if (idx == pipe->buffers - 1)
				idx = -1;

			ret = buf->ops->confirm(pipe, buf);
			if (unlikely(ret)) {
				if (ret == -ENODATA)
					ret = 0;
				goto done;
			}

			array[n].bv_page = buf->page;
			array[n].bv_len = this_len;
			array[n].bv_offset = buf->offset;
			left -= this_len;
		}

A
Al Viro 已提交
997 998
		iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n,
			      sd.total_len - left);
999
		ret = vfs_iter_write(out, &from, &sd.pos);
A
Al Viro 已提交
1000 1001 1002 1003 1004
		if (ret <= 0)
			break;

		sd.num_spliced += ret;
		sd.total_len -= ret;
1005
		*ppos = sd.pos;
A
Al Viro 已提交
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040

		/* dismiss the fully eaten buffers, adjust the partial one */
		while (ret) {
			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
			if (ret >= buf->len) {
				const struct pipe_buf_operations *ops = buf->ops;
				ret -= buf->len;
				buf->len = 0;
				buf->ops = NULL;
				ops->release(pipe, buf);
				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
				pipe->nrbufs--;
				if (pipe->files)
					sd.need_wakeup = true;
			} else {
				buf->offset += ret;
				buf->len -= ret;
				ret = 0;
			}
		}
	}
done:
	kfree(array);
	splice_from_pipe_end(pipe, &sd);

	pipe_unlock(pipe);

	if (sd.num_spliced)
		ret = sd.num_spliced;

	return ret;
}

EXPORT_SYMBOL(iter_file_splice_write);

1041 1042
static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			  struct splice_desc *sd)
1043
{
1044 1045
	int ret;
	void *data;
1046
	loff_t tmp = sd->pos;
1047

A
Al Viro 已提交
1048
	data = kmap(buf->page);
1049
	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
A
Al Viro 已提交
1050
	kunmap(buf->page);
1051 1052

	return ret;
1053 1054 1055 1056 1057 1058
}

static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
					 struct file *out, loff_t *ppos,
					 size_t len, unsigned int flags)
{
1059
	ssize_t ret;
1060

1061 1062 1063
	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
	if (ret > 0)
		*ppos += ret;
1064

1065
	return ret;
1066 1067
}

1068 1069
/**
 * generic_splice_sendpage - splice data from a pipe to a socket
1070
 * @pipe:	pipe to splice from
1071
 * @out:	socket to write to
1072
 * @ppos:	position in @out
1073 1074 1075
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
1076 1077 1078
 * Description:
 *    Will send @len bytes from the pipe to a network socket. No data copying
 *    is involved.
1079 1080
 *
 */
1081
ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1082
				loff_t *ppos, size_t len, unsigned int flags)
1083
{
1084
	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1085 1086
}

1087
EXPORT_SYMBOL(generic_splice_sendpage);
J
Jeff Garzik 已提交
1088

1089 1090 1091
/*
 * Attempt to initiate a splice from pipe to file.
 */
1092
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1093
			   loff_t *ppos, size_t len, unsigned int flags)
1094
{
1095 1096
	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
				loff_t *, size_t, unsigned int);
1097

A
Al Viro 已提交
1098
	if (out->f_op->splice_write)
1099 1100
		splice_write = out->f_op->splice_write;
	else
1101 1102
		splice_write = default_file_splice_write;

1103
	return splice_write(pipe, out, ppos, len, flags);
1104 1105
}

1106 1107 1108
/*
 * Attempt to initiate a splice from a file to a pipe.
 */
1109 1110 1111
static long do_splice_to(struct file *in, loff_t *ppos,
			 struct pipe_inode_info *pipe, size_t len,
			 unsigned int flags)
1112
{
1113 1114
	ssize_t (*splice_read)(struct file *, loff_t *,
			       struct pipe_inode_info *, size_t, unsigned int);
1115 1116
	int ret;

1117
	if (unlikely(!(in->f_mode & FMODE_READ)))
1118 1119
		return -EBADF;

1120
	ret = rw_verify_area(READ, in, ppos, len);
1121 1122 1123
	if (unlikely(ret < 0))
		return ret;

1124 1125 1126
	if (unlikely(len > MAX_RW_COUNT))
		len = MAX_RW_COUNT;

A
Al Viro 已提交
1127
	if (in->f_op->splice_read)
1128 1129
		splice_read = in->f_op->splice_read;
	else
1130 1131 1132
		splice_read = default_file_splice_read;

	return splice_read(in, ppos, pipe, len, flags);
1133 1134
}

1135 1136 1137 1138 1139 1140 1141 1142 1143
/**
 * splice_direct_to_actor - splices data directly between two non-pipes
 * @in:		file to splice from
 * @sd:		actor information on where to splice to
 * @actor:	handles the data splicing
 *
 * Description:
 *    This is a special case helper to splice directly between two
 *    points, without requiring an explicit pipe. Internally an allocated
R
Randy Dunlap 已提交
1144
 *    pipe is cached in the process, and reused during the lifetime of
1145 1146
 *    that process.
 *
J
Jens Axboe 已提交
1147 1148 1149
 */
ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
			       splice_direct_actor *actor)
1150 1151 1152 1153
{
	struct pipe_inode_info *pipe;
	long ret, bytes;
	umode_t i_mode;
J
Jens Axboe 已提交
1154
	size_t len;
1155
	int i, flags, more;
1156 1157 1158 1159 1160 1161

	/*
	 * We require the input being a regular file, as we don't want to
	 * randomly drop data for eg socket -> socket splicing. Use the
	 * piped splicing for that!
	 */
A
Al Viro 已提交
1162
	i_mode = file_inode(in)->i_mode;
1163 1164 1165 1166 1167 1168 1169 1170
	if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
		return -EINVAL;

	/*
	 * neither in nor out is a pipe, setup an internal pipe attached to
	 * 'out' and transfer the wanted data from 'in' to 'out' through that
	 */
	pipe = current->splice_pipe;
1171
	if (unlikely(!pipe)) {
A
Al Viro 已提交
1172
		pipe = alloc_pipe_info();
1173 1174 1175 1176 1177
		if (!pipe)
			return -ENOMEM;

		/*
		 * We don't have an immediate reader, but we'll read the stuff
1178
		 * out of the pipe right after the splice_to_pipe(). So set
1179 1180 1181 1182 1183 1184 1185 1186
		 * PIPE_READERS appropriately.
		 */
		pipe->readers = 1;

		current->splice_pipe = pipe;
	}

	/*
I
Ingo Molnar 已提交
1187
	 * Do the splice.
1188 1189 1190
	 */
	ret = 0;
	bytes = 0;
J
Jens Axboe 已提交
1191 1192 1193 1194 1195 1196 1197
	len = sd->total_len;
	flags = sd->flags;

	/*
	 * Don't block on output, we have to drain the direct pipe.
	 */
	sd->flags &= ~SPLICE_F_NONBLOCK;
1198
	more = sd->flags & SPLICE_F_MORE;
1199 1200

	while (len) {
1201
		size_t read_len;
1202
		loff_t pos = sd->pos, prev_pos = pos;
1203

1204
		ret = do_splice_to(in, &pos, pipe, len, flags);
1205
		if (unlikely(ret <= 0))
1206 1207 1208
			goto out_release;

		read_len = ret;
J
Jens Axboe 已提交
1209
		sd->total_len = read_len;
1210

1211 1212 1213 1214 1215 1216 1217 1218 1219
		/*
		 * If more data is pending, set SPLICE_F_MORE
		 * If this is the last data and SPLICE_F_MORE was not set
		 * initially, clears it.
		 */
		if (read_len < len)
			sd->flags |= SPLICE_F_MORE;
		else if (!more)
			sd->flags &= ~SPLICE_F_MORE;
1220 1221 1222 1223 1224
		/*
		 * NOTE: nonblocking mode only applies to the input. We
		 * must not do the output in nonblocking mode as then we
		 * could get stuck data in the internal pipe:
		 */
J
Jens Axboe 已提交
1225
		ret = actor(pipe, sd);
1226 1227
		if (unlikely(ret <= 0)) {
			sd->pos = prev_pos;
1228
			goto out_release;
1229
		}
1230 1231 1232

		bytes += ret;
		len -= ret;
1233
		sd->pos = pos;
1234

1235 1236
		if (ret < read_len) {
			sd->pos = prev_pos + ret;
1237
			goto out_release;
1238
		}
1239 1240
	}

1241
done:
1242
	pipe->nrbufs = pipe->curbuf = 0;
1243
	file_accessed(in);
1244 1245 1246 1247 1248 1249 1250
	return bytes;

out_release:
	/*
	 * If we did an incomplete transfer we must release
	 * the pipe buffers in question:
	 */
1251
	for (i = 0; i < pipe->buffers; i++) {
1252 1253 1254 1255 1256 1257 1258 1259
		struct pipe_buffer *buf = pipe->bufs + i;

		if (buf->ops) {
			buf->ops->release(pipe, buf);
			buf->ops = NULL;
		}
	}

1260 1261
	if (!bytes)
		bytes = ret;
J
Jens Axboe 已提交
1262

1263
	goto done;
J
Jens Axboe 已提交
1264 1265 1266 1267 1268 1269
}
EXPORT_SYMBOL(splice_direct_to_actor);

static int direct_splice_actor(struct pipe_inode_info *pipe,
			       struct splice_desc *sd)
{
1270
	struct file *file = sd->u.file;
J
Jens Axboe 已提交
1271

1272
	return do_splice_from(pipe, file, sd->opos, sd->total_len,
1273
			      sd->flags);
J
Jens Axboe 已提交
1274 1275
}

1276 1277 1278 1279 1280
/**
 * do_splice_direct - splices data directly between two files
 * @in:		file to splice from
 * @ppos:	input file offset
 * @out:	file to splice to
1281
 * @opos:	output file offset
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    For use by do_sendfile(). splice can easily emulate sendfile, but
 *    doing it in the application would incur an extra system call
 *    (splice in + splice out, as compared to just sendfile()). So this helper
 *    can splice directly through a process-private pipe.
 *
 */
J
Jens Axboe 已提交
1292
long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1293
		      loff_t *opos, size_t len, unsigned int flags)
J
Jens Axboe 已提交
1294 1295 1296 1297 1298 1299
{
	struct splice_desc sd = {
		.len		= len,
		.total_len	= len,
		.flags		= flags,
		.pos		= *ppos,
1300
		.u.file		= out,
1301
		.opos		= opos,
J
Jens Axboe 已提交
1302
	};
1303
	long ret;
J
Jens Axboe 已提交
1304

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
	if (unlikely(!(out->f_mode & FMODE_WRITE)))
		return -EBADF;

	if (unlikely(out->f_flags & O_APPEND))
		return -EINVAL;

	ret = rw_verify_area(WRITE, out, opos, len);
	if (unlikely(ret < 0))
		return ret;

J
Jens Axboe 已提交
1315
	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1316
	if (ret > 0)
1317
		*ppos = sd.pos;
1318

J
Jens Axboe 已提交
1319
	return ret;
1320
}
1321
EXPORT_SYMBOL(do_splice_direct);
1322

1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
{
	while (pipe->nrbufs == pipe->buffers) {
		if (flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;
		if (signal_pending(current))
			return -ERESTARTSYS;
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
	}
	return 0;
}

1337 1338 1339
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags);
1340

1341 1342 1343
/*
 * Determine where to splice to/from.
 */
1344 1345 1346
static long do_splice(struct file *in, loff_t __user *off_in,
		      struct file *out, loff_t __user *off_out,
		      size_t len, unsigned int flags)
1347
{
1348 1349
	struct pipe_inode_info *ipipe;
	struct pipe_inode_info *opipe;
1350
	loff_t offset;
J
Jens Axboe 已提交
1351
	long ret;
1352

1353 1354
	ipipe = get_pipe_info(in);
	opipe = get_pipe_info(out);
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373

	if (ipipe && opipe) {
		if (off_in || off_out)
			return -ESPIPE;

		if (!(in->f_mode & FMODE_READ))
			return -EBADF;

		if (!(out->f_mode & FMODE_WRITE))
			return -EBADF;

		/* Splicing to self would be fun, but... */
		if (ipipe == opipe)
			return -EINVAL;

		return splice_pipe_to_pipe(ipipe, opipe, len, flags);
	}

	if (ipipe) {
1374 1375
		if (off_in)
			return -ESPIPE;
1376
		if (off_out) {
1377
			if (!(out->f_mode & FMODE_PWRITE))
1378
				return -EINVAL;
1379
			if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1380
				return -EFAULT;
1381 1382 1383
		} else {
			offset = out->f_pos;
		}
1384

1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
		if (unlikely(!(out->f_mode & FMODE_WRITE)))
			return -EBADF;

		if (unlikely(out->f_flags & O_APPEND))
			return -EINVAL;

		ret = rw_verify_area(WRITE, out, &offset, len);
		if (unlikely(ret < 0))
			return ret;

1395
		file_start_write(out);
1396
		ret = do_splice_from(ipipe, out, &offset, len, flags);
1397
		file_end_write(out);
J
Jens Axboe 已提交
1398

1399 1400 1401
		if (!off_out)
			out->f_pos = offset;
		else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1402 1403 1404
			ret = -EFAULT;

		return ret;
1405
	}
1406

1407
	if (opipe) {
1408 1409
		if (off_out)
			return -ESPIPE;
1410
		if (off_in) {
1411
			if (!(in->f_mode & FMODE_PREAD))
1412
				return -EINVAL;
1413
			if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1414
				return -EFAULT;
1415 1416 1417
		} else {
			offset = in->f_pos;
		}
1418

1419 1420 1421 1422 1423 1424 1425
		pipe_lock(opipe);
		ret = wait_for_space(opipe, flags);
		if (!ret)
			ret = do_splice_to(in, &offset, opipe, len, flags);
		pipe_unlock(opipe);
		if (ret > 0)
			wakeup_pipe_readers(opipe);
1426 1427 1428
		if (!off_in)
			in->f_pos = offset;
		else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1429 1430 1431
			ret = -EFAULT;

		return ret;
1432
	}
1433 1434 1435 1436

	return -EINVAL;
}

A
Al Viro 已提交
1437 1438 1439
static int iter_to_pipe(struct iov_iter *from,
			struct pipe_inode_info *pipe,
			unsigned flags)
1440
{
A
Al Viro 已提交
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
	struct pipe_buffer buf = {
		.ops = &user_page_pipe_buf_ops,
		.flags = flags
	};
	size_t total = 0;
	int ret = 0;
	bool failed = false;

	while (iov_iter_count(from) && !failed) {
		struct page *pages[16];
1451 1452
		ssize_t copied;
		size_t start;
A
Al Viro 已提交
1453
		int n;
1454

A
Al Viro 已提交
1455 1456 1457 1458 1459
		copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
		if (copied <= 0) {
			ret = copied;
			break;
		}
1460

A
Al Viro 已提交
1461
		for (n = 0; copied; n++, start = 0) {
1462
			int size = min_t(int, copied, PAGE_SIZE - start);
A
Al Viro 已提交
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
			if (!failed) {
				buf.page = pages[n];
				buf.offset = start;
				buf.len = size;
				ret = add_to_pipe(pipe, &buf);
				if (unlikely(ret < 0)) {
					failed = true;
				} else {
					iov_iter_advance(from, ret);
					total += ret;
				}
			} else {
				put_page(pages[n]);
			}
1477
			copied -= size;
1478 1479
		}
	}
A
Al Viro 已提交
1480
	return total ? total : ret;
1481 1482
}

1483 1484 1485
static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			struct splice_desc *sd)
{
1486 1487
	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
	return n == sd->len ? n : -EFAULT;
1488 1489 1490 1491 1492 1493
}

/*
 * For lack of a better implementation, implement vmsplice() to userspace
 * as a simple copy of the pipes pages to the user iov.
 */
1494
static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov,
1495 1496 1497 1498 1499
			     unsigned long nr_segs, unsigned int flags)
{
	struct pipe_inode_info *pipe;
	struct splice_desc sd;
	long ret;
1500 1501 1502
	struct iovec iovstack[UIO_FASTIOV];
	struct iovec *iov = iovstack;
	struct iov_iter iter;
1503

1504
	pipe = get_pipe_info(file);
1505 1506 1507
	if (!pipe)
		return -EBADF;

1508 1509 1510 1511
	ret = import_iovec(READ, uiov, nr_segs,
			   ARRAY_SIZE(iovstack), &iov, &iter);
	if (ret < 0)
		return ret;
1512

1513
	sd.total_len = iov_iter_count(&iter);
1514 1515 1516 1517
	sd.len = 0;
	sd.flags = flags;
	sd.u.data = &iter;
	sd.pos = 0;
1518

1519 1520 1521 1522 1523
	if (sd.total_len) {
		pipe_lock(pipe);
		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
		pipe_unlock(pipe);
	}
1524

1525
	kfree(iov);
1526 1527 1528
	return ret;
}

1529 1530 1531 1532 1533
/*
 * vmsplice splices a user address range into a pipe. It can be thought of
 * as splice-from-memory, where the regular splice is splice-from-file (or
 * to file). In both cases the output is a pipe, naturally.
 */
1534
static long vmsplice_to_pipe(struct file *file, const struct iovec __user *uiov,
1535
			     unsigned long nr_segs, unsigned int flags)
1536
{
1537
	struct pipe_inode_info *pipe;
1538 1539 1540
	struct iovec iovstack[UIO_FASTIOV];
	struct iovec *iov = iovstack;
	struct iov_iter from;
1541
	long ret;
A
Al Viro 已提交
1542 1543 1544 1545
	unsigned buf_flag = 0;

	if (flags & SPLICE_F_GIFT)
		buf_flag = PIPE_BUF_FLAG_GIFT;
1546

1547
	pipe = get_pipe_info(file);
1548
	if (!pipe)
1549 1550
		return -EBADF;

1551 1552 1553 1554 1555
	ret = import_iovec(WRITE, uiov, nr_segs,
			   ARRAY_SIZE(iovstack), &iov, &from);
	if (ret < 0)
		return ret;

1556 1557
	pipe_lock(pipe);
	ret = wait_for_space(pipe, flags);
A
Al Viro 已提交
1558 1559
	if (!ret)
		ret = iter_to_pipe(&from, pipe, buf_flag);
1560 1561 1562
	pipe_unlock(pipe);
	if (ret > 0)
		wakeup_pipe_readers(pipe);
1563
	kfree(iov);
1564
	return ret;
1565 1566
}

1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
/*
 * Note that vmsplice only really supports true splicing _from_ user memory
 * to a pipe, not the other way around. Splicing from user memory is a simple
 * operation that can be supported without any funky alignment restrictions
 * or nasty vm tricks. We simply map in the user memory and fill them into
 * a pipe. The reverse isn't quite as easy, though. There are two possible
 * solutions for that:
 *
 *	- memcpy() the data internally, at which point we might as well just
 *	  do a regular read() on the buffer anyway.
 *	- Lots of nasty vm tricks, that are neither fast nor flexible (it
 *	  has restriction limitations on both ends of the pipe).
 *
 * Currently we punt and implement it as a normal copy, see pipe_to_user().
 *
 */
1583 1584
SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
		unsigned long, nr_segs, unsigned int, flags)
1585
{
1586
	struct fd f;
1587 1588
	long error;

1589 1590 1591 1592 1593
	if (unlikely(nr_segs > UIO_MAXIOV))
		return -EINVAL;
	else if (unlikely(!nr_segs))
		return 0;

1594
	error = -EBADF;
1595 1596 1597 1598 1599 1600 1601 1602
	f = fdget(fd);
	if (f.file) {
		if (f.file->f_mode & FMODE_WRITE)
			error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
		else if (f.file->f_mode & FMODE_READ)
			error = vmsplice_to_user(f.file, iov, nr_segs, flags);

		fdput(f);
1603 1604 1605 1606 1607
	}

	return error;
}

1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
		    unsigned int, nr_segs, unsigned int, flags)
{
	unsigned i;
	struct iovec __user *iov;
	if (nr_segs > UIO_MAXIOV)
		return -EINVAL;
	iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec));
	for (i = 0; i < nr_segs; i++) {
		struct compat_iovec v;
		if (get_user(v.iov_base, &iov32[i].iov_base) ||
		    get_user(v.iov_len, &iov32[i].iov_len) ||
		    put_user(compat_ptr(v.iov_base), &iov[i].iov_base) ||
		    put_user(v.iov_len, &iov[i].iov_len))
			return -EFAULT;
	}
	return sys_vmsplice(fd, iov, nr_segs, flags);
}
#endif

1629 1630 1631
SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
		int, fd_out, loff_t __user *, off_out,
		size_t, len, unsigned int, flags)
1632
{
1633
	struct fd in, out;
1634 1635 1636 1637 1638 1639
	long error;

	if (unlikely(!len))
		return 0;

	error = -EBADF;
1640 1641 1642 1643 1644 1645 1646 1647
	in = fdget(fd_in);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			out = fdget(fd_out);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_splice(in.file, off_in,
							  out.file, off_out,
1648
							  len, flags);
1649
				fdput(out);
1650 1651
			}
		}
1652
		fdput(in);
1653 1654 1655
	}
	return error;
}
1656

1657 1658 1659 1660
/*
 * Make sure there's data to read. Wait for input if we can, otherwise
 * return an appropriate error.
 */
1661
static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
	if (pipe->nrbufs)
		return 0;

	ret = 0;
1673
	pipe_lock(pipe);
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690

	while (!pipe->nrbufs) {
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		if (!pipe->writers)
			break;
		if (!pipe->waiting_writers) {
			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}
		}
		pipe_wait(pipe);
	}

1691
	pipe_unlock(pipe);
1692 1693 1694 1695 1696 1697 1698
	return ret;
}

/*
 * Make sure there's writeable room. Wait for room if we can, otherwise
 * return an appropriate error.
 */
1699
static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1700 1701 1702 1703 1704 1705 1706
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
1707
	if (pipe->nrbufs < pipe->buffers)
1708 1709 1710
		return 0;

	ret = 0;
1711
	pipe_lock(pipe);
1712

1713
	while (pipe->nrbufs >= pipe->buffers) {
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
		if (!pipe->readers) {
			send_sig(SIGPIPE, current, 0);
			ret = -EPIPE;
			break;
		}
		if (flags & SPLICE_F_NONBLOCK) {
			ret = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
	}

1732
	pipe_unlock(pipe);
1733 1734 1735
	return ret;
}

1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
/*
 * Splice contents of ipipe to opipe.
 */
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
	int ret = 0, nbuf;
	bool input_wakeup = false;


retry:
	ret = ipipe_prep(ipipe, flags);
	if (ret)
		return ret;

	ret = opipe_prep(opipe, flags);
	if (ret)
		return ret;

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
	 * grabbing by pipe info address. Otherwise two different processes
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
	pipe_double_lock(ipipe, opipe);

	do {
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

		if (!ipipe->nrbufs && !ipipe->writers)
			break;

		/*
		 * Cannot make any progress, because either the input
		 * pipe is empty or the output pipe is full.
		 */
1779
		if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
			/* Already processed some buffers, break */
			if (ret)
				break;

			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}

			/*
			 * We raced with another reader/writer and haven't
			 * managed to process any buffers.  A zero return
			 * value means EOF, so retry instead.
			 */
			pipe_unlock(ipipe);
			pipe_unlock(opipe);
			goto retry;
		}

		ibuf = ipipe->bufs + ipipe->curbuf;
1800
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1801 1802 1803 1804 1805 1806 1807 1808 1809
		obuf = opipe->bufs + nbuf;

		if (len >= ibuf->len) {
			/*
			 * Simply move the whole buffer from ipipe to opipe
			 */
			*obuf = *ibuf;
			ibuf->ops = NULL;
			opipe->nrbufs++;
1810
			ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
			ipipe->nrbufs--;
			input_wakeup = true;
		} else {
			/*
			 * Get a reference to this pipe buffer,
			 * so we can copy the contents over.
			 */
			ibuf->ops->get(ipipe, ibuf);
			*obuf = *ibuf;

			/*
			 * Don't inherit the gift flag, we need to
			 * prevent multiple steals of this page.
			 */
			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;

			obuf->len = len;
			opipe->nrbufs++;
			ibuf->offset += obuf->len;
			ibuf->len -= obuf->len;
		}
		ret += obuf->len;
		len -= obuf->len;
	} while (len);

	pipe_unlock(ipipe);
	pipe_unlock(opipe);

	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
N
Namhyung Kim 已提交
1842 1843 1844
	if (ret > 0)
		wakeup_pipe_readers(opipe);

1845 1846 1847 1848 1849 1850
	if (input_wakeup)
		wakeup_pipe_writers(ipipe);

	return ret;
}

1851 1852 1853 1854 1855 1856 1857 1858
/*
 * Link contents of ipipe to opipe.
 */
static int link_pipe(struct pipe_inode_info *ipipe,
		     struct pipe_inode_info *opipe,
		     size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
1859
	int ret = 0, i = 0, nbuf;
1860 1861 1862

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
1863
	 * grabbing by pipe info address. Otherwise two different processes
1864 1865
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
1866
	pipe_double_lock(ipipe, opipe);
1867

1868
	do {
1869 1870 1871 1872 1873 1874 1875
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

1876 1877 1878 1879
		/*
		 * If we have iterated all input buffers or ran out of
		 * output room, break.
		 */
1880
		if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
1881
			break;
1882

1883 1884
		ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1885 1886

		/*
1887 1888
		 * Get a reference to this pipe buffer,
		 * so we can copy the contents over.
1889
		 */
1890 1891 1892 1893 1894
		ibuf->ops->get(ipipe, ibuf);

		obuf = opipe->bufs + nbuf;
		*obuf = *ibuf;

1895
		/*
1896 1897
		 * Don't inherit the gift flag, we need to
		 * prevent multiple steals of this page.
1898
		 */
1899
		obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1900

1901 1902
		if (obuf->len > len)
			obuf->len = len;
1903

1904 1905 1906 1907 1908
		opipe->nrbufs++;
		ret += obuf->len;
		len -= obuf->len;
		i++;
	} while (len);
1909

1910 1911 1912 1913 1914 1915 1916
	/*
	 * return EAGAIN if we have the potential of some data in the
	 * future, otherwise just return 0
	 */
	if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
		ret = -EAGAIN;

1917 1918
	pipe_unlock(ipipe);
	pipe_unlock(opipe);
1919

1920 1921 1922
	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
N
Namhyung Kim 已提交
1923 1924
	if (ret > 0)
		wakeup_pipe_readers(opipe);
1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937

	return ret;
}

/*
 * This is a tee(1) implementation that works on pipes. It doesn't copy
 * any data, it simply references the 'in' pages on the 'out' pipe.
 * The 'flags' used are the SPLICE_F_* variants, currently the only
 * applicable one is SPLICE_F_NONBLOCK.
 */
static long do_tee(struct file *in, struct file *out, size_t len,
		   unsigned int flags)
{
1938 1939
	struct pipe_inode_info *ipipe = get_pipe_info(in);
	struct pipe_inode_info *opipe = get_pipe_info(out);
1940
	int ret = -EINVAL;
1941 1942

	/*
1943 1944
	 * Duplicate the contents of ipipe to opipe without actually
	 * copying the data.
1945
	 */
1946 1947 1948 1949 1950
	if (ipipe && opipe && ipipe != opipe) {
		/*
		 * Keep going, unless we encounter an error. The ipipe/opipe
		 * ordering doesn't really matter.
		 */
1951
		ret = ipipe_prep(ipipe, flags);
1952
		if (!ret) {
1953
			ret = opipe_prep(opipe, flags);
1954
			if (!ret)
1955 1956 1957
				ret = link_pipe(ipipe, opipe, len, flags);
		}
	}
1958

1959
	return ret;
1960 1961
}

1962
SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1963
{
1964 1965
	struct fd in;
	int error;
1966 1967 1968 1969 1970

	if (unlikely(!len))
		return 0;

	error = -EBADF;
1971 1972 1973 1974 1975 1976 1977 1978 1979
	in = fdget(fdin);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			struct fd out = fdget(fdout);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_tee(in.file, out.file,
							len, flags);
				fdput(out);
1980 1981
			}
		}
1982
 		fdput(in);
1983 1984 1985 1986
 	}

	return error;
}