splice.c 39.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * "splice": joining two ropes together by interweaving their strands.
 *
 * This is the "extended pipe" functionality, where a pipe is used as
 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
 * buffer that you can use to transfer data from one end to the other.
 *
 * The traditional unix read/write is extended with a "splice()" operation
 * that transfers data buffers to or from a pipe buffer.
 *
 * Named by Larry McVoy, original implementation from Linus, extended by
12 13
 * Jens to support splicing to files, network, direct splicing, etc and
 * fixing lots of bugs.
14
 *
15
 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
16 17
 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
18 19
 *
 */
20
#include <linux/bvec.h>
21 22 23
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>
24
#include <linux/splice.h>
K
KAMEZAWA Hiroyuki 已提交
25
#include <linux/memcontrol.h>
26
#include <linux/mm_inline.h>
27
#include <linux/swap.h>
28
#include <linux/writeback.h>
29
#include <linux/export.h>
30
#include <linux/syscalls.h>
31
#include <linux/uio.h>
32
#include <linux/security.h>
33
#include <linux/gfp.h>
34
#include <linux/socket.h>
35
#include <linux/compat.h>
36
#include "internal.h"
37

38 39 40 41 42 43
/*
 * Attempt to steal a page from a pipe buffer. This should perhaps go into
 * a vm helper function, it's already simplified quite a bit by the
 * addition of remove_mapping(). If success is returned, the caller may
 * attempt to reuse this page for another destination.
 */
44
static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
45 46 47
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
48
	struct address_space *mapping;
49

50 51
	lock_page(page);

52 53 54
	mapping = page_mapping(page);
	if (mapping) {
		WARN_ON(!PageUptodate(page));
55

56 57 58 59 60 61 62 63 64
		/*
		 * At least for ext2 with nobh option, we need to wait on
		 * writeback completing on this page, since we'll remove it
		 * from the pagecache.  Otherwise truncate wont wait on the
		 * page, allowing the disk blocks to be reused by someone else
		 * before we actually wrote our data to them. fs corruption
		 * ensues.
		 */
		wait_on_page_writeback(page);
65

66 67
		if (page_has_private(page) &&
		    !try_to_release_page(page, GFP_KERNEL))
68
			goto out_unlock;
69

70 71 72 73 74 75 76 77
		/*
		 * If we succeeded in removing the mapping, set LRU flag
		 * and return good.
		 */
		if (remove_mapping(mapping, page)) {
			buf->flags |= PIPE_BUF_FLAG_LRU;
			return 0;
		}
78
	}
79

80 81 82 83
	/*
	 * Raced with truncate or failed to remove page from current
	 * address space, unlock and return failure.
	 */
84
out_unlock:
85 86
	unlock_page(page);
	return 1;
87 88
}

89
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
90 91
					struct pipe_buffer *buf)
{
92
	put_page(buf->page);
J
Jens Axboe 已提交
93
	buf->flags &= ~PIPE_BUF_FLAG_LRU;
94 95
}

96 97 98 99
/*
 * Check whether the contents of buf is OK to access. Since the content
 * is a page cache page, IO may be in flight.
 */
100 101
static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
				       struct pipe_buffer *buf)
102 103
{
	struct page *page = buf->page;
104
	int err;
105 106

	if (!PageUptodate(page)) {
107 108 109 110
		lock_page(page);

		/*
		 * Page got truncated/unhashed. This will cause a 0-byte
I
Ingo Molnar 已提交
111
		 * splice, if this is the first page.
112 113 114 115 116
		 */
		if (!page->mapping) {
			err = -ENODATA;
			goto error;
		}
117

118
		/*
I
Ingo Molnar 已提交
119
		 * Uh oh, read-error from disk.
120 121 122 123 124 125 126
		 */
		if (!PageUptodate(page)) {
			err = -EIO;
			goto error;
		}

		/*
127
		 * Page is ok afterall, we are done.
128
		 */
129 130 131
		unlock_page(page);
	}

132
	return 0;
133 134
error:
	unlock_page(page);
135
	return err;
136 137
}

138
const struct pipe_buf_operations page_cache_pipe_buf_ops = {
139
	.can_merge = 0,
140
	.confirm = page_cache_pipe_buf_confirm,
141
	.release = page_cache_pipe_buf_release,
142
	.steal = page_cache_pipe_buf_steal,
143
	.get = generic_pipe_buf_get,
144 145
};

146 147 148
static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
149 150 151
	if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
		return 1;

J
Jens Axboe 已提交
152
	buf->flags |= PIPE_BUF_FLAG_LRU;
153
	return generic_pipe_buf_steal(pipe, buf);
154 155
}

156
static const struct pipe_buf_operations user_page_pipe_buf_ops = {
157
	.can_merge = 0,
158
	.confirm = generic_pipe_buf_confirm,
159 160
	.release = page_cache_pipe_buf_release,
	.steal = user_page_pipe_buf_steal,
161
	.get = generic_pipe_buf_get,
162 163
};

N
Namhyung Kim 已提交
164 165 166 167 168 169 170 171
static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
}

172 173 174 175 176 177
/**
 * splice_to_pipe - fill passed data into a pipe
 * @pipe:	pipe to fill
 * @spd:	data to fill
 *
 * Description:
R
Randy Dunlap 已提交
178
 *    @spd contains a map of pages and len/offset tuples, along with
179 180 181
 *    the struct pipe_buf_operations associated with these pages. This
 *    function will link that data to the pipe.
 *
182
 */
183 184
ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
		       struct splice_pipe_desc *spd)
185
{
186
	unsigned int spd_pages = spd->nr_pages;
187
	int ret = 0, page_nr = 0;
188

189 190 191
	if (!spd_pages)
		return 0;

192 193 194 195 196
	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
		goto out;
	}
197

198 199 200
	while (pipe->nrbufs < pipe->buffers) {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		struct pipe_buffer *buf = pipe->bufs + newbuf;
201

202 203 204 205 206
		buf->page = spd->pages[page_nr];
		buf->offset = spd->partial[page_nr].offset;
		buf->len = spd->partial[page_nr].len;
		buf->private = spd->partial[page_nr].private;
		buf->ops = spd->ops;
207

208 209 210
		pipe->nrbufs++;
		page_nr++;
		ret += buf->len;
211

212
		if (!--spd->nr_pages)
213 214 215
			break;
	}

216 217
	if (!ret)
		ret = -EAGAIN;
218

219
out:
220
	while (page_nr < spd_pages)
221
		spd->spd_release(spd, page_nr++);
222 223 224

	return ret;
}
225
EXPORT_SYMBOL_GPL(splice_to_pipe);
226

A
Al Viro 已提交
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
{
	int ret;

	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
	} else if (pipe->nrbufs == pipe->buffers) {
		ret = -EAGAIN;
	} else {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		pipe->bufs[newbuf] = *buf;
		pipe->nrbufs++;
		return buf->len;
	}
242
	pipe_buf_release(pipe, buf);
A
Al Viro 已提交
243 244 245 246
	return ret;
}
EXPORT_SYMBOL(add_to_pipe);

247
void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
248
{
249
	put_page(spd->pages[i]);
250 251
}

252 253 254 255
/*
 * Check if we need to grow the arrays holding pages and partial page
 * descriptions.
 */
256
int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
257
{
258 259 260 261
	unsigned int buffers = ACCESS_ONCE(pipe->buffers);

	spd->nr_pages_max = buffers;
	if (buffers <= PIPE_DEF_BUFFERS)
262 263
		return 0;

264 265
	spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
	spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
266 267 268 269 270 271 272 273 274

	if (spd->pages && spd->partial)
		return 0;

	kfree(spd->pages);
	kfree(spd->partial);
	return -ENOMEM;
}

275
void splice_shrink_spd(struct splice_pipe_desc *spd)
276
{
277
	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
278 279 280 281 282 283
		return;

	kfree(spd->pages);
	kfree(spd->partial);
}

284 285 286
/**
 * generic_file_splice_read - splice data from file to a pipe
 * @in:		file to splice from
287
 * @ppos:	position in @in
288 289 290 291
 * @pipe:	pipe to splice to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
292 293
 * Description:
 *    Will read pages from given file and fill them into a pipe. Can be
294
 *    used as long as it has more or less sane ->read_iter().
295
 *
296
 */
297 298 299
ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
300
{
301 302 303
	struct iov_iter to;
	struct kiocb kiocb;
	int idx, ret;
304

305 306 307 308
	iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len);
	idx = to.idx;
	init_sync_kiocb(&kiocb, in);
	kiocb.ki_pos = *ppos;
309
	ret = call_read_iter(in, &kiocb, &to);
310
	if (ret > 0) {
311
		*ppos = kiocb.ki_pos;
312
		file_accessed(in);
313
	} else if (ret < 0) {
A
Al Viro 已提交
314 315 316
		to.idx = idx;
		to.iov_offset = 0;
		iov_iter_advance(&to, 0); /* to free what was emitted */
317 318 319 320 321 322
		/*
		 * callers of ->splice_read() expect -EAGAIN on
		 * "can't put anything in there", rather than -EFAULT.
		 */
		if (ret == -EFAULT)
			ret = -EAGAIN;
323
	}
324 325 326

	return ret;
}
327 328
EXPORT_SYMBOL(generic_file_splice_read);

A
Al Viro 已提交
329
const struct pipe_buf_operations default_pipe_buf_ops = {
330 331 332 333 334 335 336
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};

M
Miklos Szeredi 已提交
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	return 1;
}

/* Pipe buffer operations for a socket and similar. */
const struct pipe_buf_operations nosteal_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_nosteal,
	.get = generic_pipe_buf_get,
};
EXPORT_SYMBOL(nosteal_pipe_buf_ops);

353
static ssize_t kernel_readv(struct file *file, const struct kvec *vec,
354 355 356 357 358 359 360 361 362
			    unsigned long vlen, loff_t offset)
{
	mm_segment_t old_fs;
	loff_t pos = offset;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
363
	res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0);
364 365 366 367 368
	set_fs(old_fs);

	return res;
}

369
ssize_t kernel_write(struct file *file, const char *buf, size_t count,
370
			    loff_t pos)
371 372 373 374 375 376 377
{
	mm_segment_t old_fs;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
378
	res = vfs_write(file, (__force const char __user *)buf, count, &pos);
379 380 381 382
	set_fs(old_fs);

	return res;
}
383
EXPORT_SYMBOL(kernel_write);
384

385
static ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
386 387 388
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
{
389 390 391
	struct kvec *vec, __vec[PIPE_DEF_BUFFERS];
	struct iov_iter to;
	struct page **pages;
392
	unsigned int nr_pages;
393
	size_t offset, dummy, copied = 0;
394 395
	ssize_t res;
	int i;
396

397 398
	if (pipe->nrbufs == pipe->buffers)
		return -EAGAIN;
399

400 401 402 403
	/*
	 * Try to keep page boundaries matching to source pagecache ones -
	 * it probably won't be much help, but...
	 */
404
	offset = *ppos & ~PAGE_MASK;
405

406
	iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset);
407

408 409 410
	res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy);
	if (res <= 0)
		return -ENOMEM;
411

A
Al Viro 已提交
412 413
	BUG_ON(dummy);
	nr_pages = DIV_ROUND_UP(res, PAGE_SIZE);
414

415 416 417 418 419 420 421
	vec = __vec;
	if (nr_pages > PIPE_DEF_BUFFERS) {
		vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL);
		if (unlikely(!vec)) {
			res = -ENOMEM;
			goto out;
		}
A
Andrew Morton 已提交
422
	}
423

424 425 426 427 428 429 430 431 432
	pipe->bufs[to.idx].offset = offset;
	pipe->bufs[to.idx].len -= offset;

	for (i = 0; i < nr_pages; i++) {
		size_t this_len = min_t(size_t, len, PAGE_SIZE - offset);
		vec[i].iov_base = page_address(pages[i]) + offset;
		vec[i].iov_len = this_len;
		len -= this_len;
		offset = 0;
433 434
	}

435 436 437
	res = kernel_readv(in, vec, nr_pages, *ppos);
	if (res > 0) {
		copied = res;
438
		*ppos += res;
439
	}
440

441 442
	if (vec != __vec)
		kfree(vec);
443 444 445 446 447
out:
	for (i = 0; i < nr_pages; i++)
		put_page(pages[i]);
	kvfree(pages);
	iov_iter_advance(&to, copied);	/* truncates and discards */
448 449 450
	return res;
}

451
/*
452
 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
J
Jens Axboe 已提交
453
 * using sendpage(). Return the number of bytes sent.
454
 */
455
static int pipe_to_sendpage(struct pipe_inode_info *pipe,
456 457
			    struct pipe_buffer *buf, struct splice_desc *sd)
{
458
	struct file *file = sd->u.file;
459
	loff_t pos = sd->pos;
460
	int more;
461

A
Al Viro 已提交
462
	if (!likely(file->f_op->sendpage))
463 464
		return -EINVAL;

465
	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
466 467

	if (sd->len < sd->total_len && pipe->nrbufs > 1)
468
		more |= MSG_SENDPAGE_NOTLAST;
469

470 471
	return file->f_op->sendpage(file, buf->page, buf->offset,
				    sd->len, &pos, more);
472 473
}

474 475 476 477 478 479 480 481
static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
}

482
/**
483
 * splice_from_pipe_feed - feed available data from a pipe to a file
484 485 486 487 488
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
489 490 491 492 493 494 495
 *    This function loops over the pipe and calls @actor to do the
 *    actual moving of a single struct pipe_buffer to the desired
 *    destination.  It returns when there's no more buffers left in
 *    the pipe or if the requested number of bytes (@sd->total_len)
 *    have been copied.  It returns a positive number (one) if the
 *    pipe needs to be filled with more data, zero if the required
 *    number of bytes have been copied and -errno on error.
496
 *
497 498 499 500
 *    This, together with splice_from_pipe_{begin,end,next}, may be
 *    used to implement the functionality of __splice_from_pipe() when
 *    locking is required around copying the pipe buffers to the
 *    destination.
501
 */
A
Al Viro 已提交
502
static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
503
			  splice_actor *actor)
504
{
505
	int ret;
506

507 508
	while (pipe->nrbufs) {
		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
509

510 511 512
		sd->len = buf->len;
		if (sd->len > sd->total_len)
			sd->len = sd->total_len;
513

514
		ret = pipe_buf_confirm(pipe, buf);
515
		if (unlikely(ret)) {
516 517 518 519
			if (ret == -ENODATA)
				ret = 0;
			return ret;
		}
520 521 522 523 524

		ret = actor(pipe, buf, sd);
		if (ret <= 0)
			return ret;

525 526 527 528 529 530 531 532 533
		buf->offset += ret;
		buf->len -= ret;

		sd->num_spliced += ret;
		sd->len -= ret;
		sd->pos += ret;
		sd->total_len -= ret;

		if (!buf->len) {
534
			pipe_buf_release(pipe, buf);
535
			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
536
			pipe->nrbufs--;
A
Al Viro 已提交
537
			if (pipe->files)
538 539
				sd->need_wakeup = true;
		}
540

541 542 543
		if (!sd->total_len)
			return 0;
	}
544

545 546
	return 1;
}
547

548 549 550 551 552 553 554 555 556 557
/**
 * splice_from_pipe_next - wait for some data to splice from
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wait for some data and return a positive
 *    value (one) if pipe buffers are available.  It will return zero
 *    or -errno if no more data needs to be spliced.
 */
A
Al Viro 已提交
558
static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
559
{
560 561 562 563 564 565 566
	/*
	 * Check for signal early to make process killable when there are
	 * always buffers available
	 */
	if (signal_pending(current))
		return -ERESTARTSYS;

567 568 569
	while (!pipe->nrbufs) {
		if (!pipe->writers)
			return 0;
J
Jens Axboe 已提交
570

571 572
		if (!pipe->waiting_writers && sd->num_spliced)
			return 0;
I
Ingo Molnar 已提交
573

574 575
		if (sd->flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;
576

577 578
		if (signal_pending(current))
			return -ERESTARTSYS;
579

580 581 582
		if (sd->need_wakeup) {
			wakeup_pipe_writers(pipe);
			sd->need_wakeup = false;
583 584
		}

585 586
		pipe_wait(pipe);
	}
587

588 589
	return 1;
}
590

591 592
/**
 * splice_from_pipe_begin - start splicing from pipe
593
 * @sd:		information about the splice operation
594 595 596 597 598 599
 *
 * Description:
 *    This function should be called before a loop containing
 *    splice_from_pipe_next() and splice_from_pipe_feed() to
 *    initialize the necessary fields of @sd.
 */
A
Al Viro 已提交
600
static void splice_from_pipe_begin(struct splice_desc *sd)
601 602 603 604
{
	sd->num_spliced = 0;
	sd->need_wakeup = false;
}
605

606 607 608 609 610 611 612 613 614 615
/**
 * splice_from_pipe_end - finish splicing from pipe
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wake up pipe writers if necessary.  It should
 *    be called after a loop containing splice_from_pipe_next() and
 *    splice_from_pipe_feed().
 */
A
Al Viro 已提交
616
static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
617 618 619 620
{
	if (sd->need_wakeup)
		wakeup_pipe_writers(pipe);
}
621

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
/**
 * __splice_from_pipe - splice data from a pipe to given actor
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
 *    This function does little more than loop over the pipe and call
 *    @actor to do the actual moving of a single struct pipe_buffer to
 *    the desired destination. See pipe_to_file, pipe_to_sendpage, or
 *    pipe_to_user.
 *
 */
ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
			   splice_actor *actor)
{
	int ret;
639

640 641
	splice_from_pipe_begin(sd);
	do {
J
Jan Kara 已提交
642
		cond_resched();
643 644 645 646 647 648 649
		ret = splice_from_pipe_next(pipe, sd);
		if (ret > 0)
			ret = splice_from_pipe_feed(pipe, sd, actor);
	} while (ret > 0);
	splice_from_pipe_end(pipe, sd);

	return sd->num_spliced ? sd->num_spliced : ret;
650
}
M
Mark Fasheh 已提交
651
EXPORT_SYMBOL(__splice_from_pipe);
652

653 654 655 656 657 658 659 660 661 662
/**
 * splice_from_pipe - splice data from a pipe to a file
 * @pipe:	pipe to splice from
 * @out:	file to splice to
 * @ppos:	position in @out
 * @len:	how many bytes to splice
 * @flags:	splice modifier flags
 * @actor:	handler that splices the data
 *
 * Description:
663
 *    See __splice_from_pipe. This function locks the pipe inode,
664 665 666
 *    otherwise it's identical to __splice_from_pipe().
 *
 */
667 668 669 670 671
ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
			 loff_t *ppos, size_t len, unsigned int flags,
			 splice_actor *actor)
{
	ssize_t ret;
J
Jens Axboe 已提交
672 673 674 675
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
676
		.u.file = out,
J
Jens Axboe 已提交
677
	};
678

679
	pipe_lock(pipe);
J
Jens Axboe 已提交
680
	ret = __splice_from_pipe(pipe, &sd, actor);
681
	pipe_unlock(pipe);
682 683 684 685

	return ret;
}

A
Al Viro 已提交
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
/**
 * iter_file_splice_write - splice data from a pipe to a file
 * @pipe:	pipe info
 * @out:	file to write to
 * @ppos:	position in @out
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    Will either move or copy pages (determined by @flags options) from
 *    the given pipe inode to the given file.
 *    This one is ->write_iter-based.
 *
 */
ssize_t
iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
			  loff_t *ppos, size_t len, unsigned int flags)
{
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
	};
	int nbufs = pipe->buffers;
	struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
	ssize_t ret;

	if (unlikely(!array))
		return -ENOMEM;

	pipe_lock(pipe);

	splice_from_pipe_begin(&sd);
	while (sd.total_len) {
		struct iov_iter from;
		size_t left;
		int n, idx;

		ret = splice_from_pipe_next(pipe, &sd);
		if (ret <= 0)
			break;

		if (unlikely(nbufs < pipe->buffers)) {
			kfree(array);
			nbufs = pipe->buffers;
			array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
			if (!array) {
				ret = -ENOMEM;
				break;
			}
		}

		/* build the vector */
		left = sd.total_len;
		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {
			struct pipe_buffer *buf = pipe->bufs + idx;
			size_t this_len = buf->len;

			if (this_len > left)
				this_len = left;

			if (idx == pipe->buffers - 1)
				idx = -1;

753
			ret = pipe_buf_confirm(pipe, buf);
A
Al Viro 已提交
754 755 756 757 758 759 760 761 762 763 764 765
			if (unlikely(ret)) {
				if (ret == -ENODATA)
					ret = 0;
				goto done;
			}

			array[n].bv_page = buf->page;
			array[n].bv_len = this_len;
			array[n].bv_offset = buf->offset;
			left -= this_len;
		}

A
Al Viro 已提交
766 767
		iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n,
			      sd.total_len - left);
768
		ret = vfs_iter_write(out, &from, &sd.pos);
A
Al Viro 已提交
769 770 771 772 773
		if (ret <= 0)
			break;

		sd.num_spliced += ret;
		sd.total_len -= ret;
774
		*ppos = sd.pos;
A
Al Viro 已提交
775 776 777 778 779 780 781

		/* dismiss the fully eaten buffers, adjust the partial one */
		while (ret) {
			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
			if (ret >= buf->len) {
				ret -= buf->len;
				buf->len = 0;
782
				pipe_buf_release(pipe, buf);
A
Al Viro 已提交
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
				pipe->nrbufs--;
				if (pipe->files)
					sd.need_wakeup = true;
			} else {
				buf->offset += ret;
				buf->len -= ret;
				ret = 0;
			}
		}
	}
done:
	kfree(array);
	splice_from_pipe_end(pipe, &sd);

	pipe_unlock(pipe);

	if (sd.num_spliced)
		ret = sd.num_spliced;

	return ret;
}

EXPORT_SYMBOL(iter_file_splice_write);

808 809
static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			  struct splice_desc *sd)
810
{
811 812
	int ret;
	void *data;
813
	loff_t tmp = sd->pos;
814

A
Al Viro 已提交
815
	data = kmap(buf->page);
816
	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
A
Al Viro 已提交
817
	kunmap(buf->page);
818 819

	return ret;
820 821 822 823 824 825
}

static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
					 struct file *out, loff_t *ppos,
					 size_t len, unsigned int flags)
{
826
	ssize_t ret;
827

828 829 830
	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
	if (ret > 0)
		*ppos += ret;
831

832
	return ret;
833 834
}

835 836
/**
 * generic_splice_sendpage - splice data from a pipe to a socket
837
 * @pipe:	pipe to splice from
838
 * @out:	socket to write to
839
 * @ppos:	position in @out
840 841 842
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
843 844 845
 * Description:
 *    Will send @len bytes from the pipe to a network socket. No data copying
 *    is involved.
846 847
 *
 */
848
ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
849
				loff_t *ppos, size_t len, unsigned int flags)
850
{
851
	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
852 853
}

854
EXPORT_SYMBOL(generic_splice_sendpage);
J
Jeff Garzik 已提交
855

856 857 858
/*
 * Attempt to initiate a splice from pipe to file.
 */
859
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
860
			   loff_t *ppos, size_t len, unsigned int flags)
861
{
862 863
	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
				loff_t *, size_t, unsigned int);
864

A
Al Viro 已提交
865
	if (out->f_op->splice_write)
866 867
		splice_write = out->f_op->splice_write;
	else
868 869
		splice_write = default_file_splice_write;

870
	return splice_write(pipe, out, ppos, len, flags);
871 872
}

873 874 875
/*
 * Attempt to initiate a splice from a file to a pipe.
 */
876 877 878
static long do_splice_to(struct file *in, loff_t *ppos,
			 struct pipe_inode_info *pipe, size_t len,
			 unsigned int flags)
879
{
880 881
	ssize_t (*splice_read)(struct file *, loff_t *,
			       struct pipe_inode_info *, size_t, unsigned int);
882 883
	int ret;

884
	if (unlikely(!(in->f_mode & FMODE_READ)))
885 886
		return -EBADF;

887
	ret = rw_verify_area(READ, in, ppos, len);
888 889 890
	if (unlikely(ret < 0))
		return ret;

891 892 893
	if (unlikely(len > MAX_RW_COUNT))
		len = MAX_RW_COUNT;

A
Al Viro 已提交
894
	if (in->f_op->splice_read)
895 896
		splice_read = in->f_op->splice_read;
	else
897 898 899
		splice_read = default_file_splice_read;

	return splice_read(in, ppos, pipe, len, flags);
900 901
}

902 903 904 905 906 907 908 909 910
/**
 * splice_direct_to_actor - splices data directly between two non-pipes
 * @in:		file to splice from
 * @sd:		actor information on where to splice to
 * @actor:	handles the data splicing
 *
 * Description:
 *    This is a special case helper to splice directly between two
 *    points, without requiring an explicit pipe. Internally an allocated
R
Randy Dunlap 已提交
911
 *    pipe is cached in the process, and reused during the lifetime of
912 913
 *    that process.
 *
J
Jens Axboe 已提交
914 915 916
 */
ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
			       splice_direct_actor *actor)
917 918 919 920
{
	struct pipe_inode_info *pipe;
	long ret, bytes;
	umode_t i_mode;
J
Jens Axboe 已提交
921
	size_t len;
922
	int i, flags, more;
923 924 925 926 927 928

	/*
	 * We require the input being a regular file, as we don't want to
	 * randomly drop data for eg socket -> socket splicing. Use the
	 * piped splicing for that!
	 */
A
Al Viro 已提交
929
	i_mode = file_inode(in)->i_mode;
930 931 932 933 934 935 936 937
	if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
		return -EINVAL;

	/*
	 * neither in nor out is a pipe, setup an internal pipe attached to
	 * 'out' and transfer the wanted data from 'in' to 'out' through that
	 */
	pipe = current->splice_pipe;
938
	if (unlikely(!pipe)) {
A
Al Viro 已提交
939
		pipe = alloc_pipe_info();
940 941 942 943 944
		if (!pipe)
			return -ENOMEM;

		/*
		 * We don't have an immediate reader, but we'll read the stuff
945
		 * out of the pipe right after the splice_to_pipe(). So set
946 947 948 949 950 951 952 953
		 * PIPE_READERS appropriately.
		 */
		pipe->readers = 1;

		current->splice_pipe = pipe;
	}

	/*
I
Ingo Molnar 已提交
954
	 * Do the splice.
955 956 957
	 */
	ret = 0;
	bytes = 0;
J
Jens Axboe 已提交
958 959 960 961 962 963 964
	len = sd->total_len;
	flags = sd->flags;

	/*
	 * Don't block on output, we have to drain the direct pipe.
	 */
	sd->flags &= ~SPLICE_F_NONBLOCK;
965
	more = sd->flags & SPLICE_F_MORE;
966 967

	while (len) {
968
		size_t read_len;
969
		loff_t pos = sd->pos, prev_pos = pos;
970

971
		ret = do_splice_to(in, &pos, pipe, len, flags);
972
		if (unlikely(ret <= 0))
973 974 975
			goto out_release;

		read_len = ret;
J
Jens Axboe 已提交
976
		sd->total_len = read_len;
977

978 979 980 981 982 983 984 985 986
		/*
		 * If more data is pending, set SPLICE_F_MORE
		 * If this is the last data and SPLICE_F_MORE was not set
		 * initially, clears it.
		 */
		if (read_len < len)
			sd->flags |= SPLICE_F_MORE;
		else if (!more)
			sd->flags &= ~SPLICE_F_MORE;
987 988 989 990 991
		/*
		 * NOTE: nonblocking mode only applies to the input. We
		 * must not do the output in nonblocking mode as then we
		 * could get stuck data in the internal pipe:
		 */
J
Jens Axboe 已提交
992
		ret = actor(pipe, sd);
993 994
		if (unlikely(ret <= 0)) {
			sd->pos = prev_pos;
995
			goto out_release;
996
		}
997 998 999

		bytes += ret;
		len -= ret;
1000
		sd->pos = pos;
1001

1002 1003
		if (ret < read_len) {
			sd->pos = prev_pos + ret;
1004
			goto out_release;
1005
		}
1006 1007
	}

1008
done:
1009
	pipe->nrbufs = pipe->curbuf = 0;
1010
	file_accessed(in);
1011 1012 1013 1014 1015 1016 1017
	return bytes;

out_release:
	/*
	 * If we did an incomplete transfer we must release
	 * the pipe buffers in question:
	 */
1018
	for (i = 0; i < pipe->buffers; i++) {
1019 1020
		struct pipe_buffer *buf = pipe->bufs + i;

1021 1022
		if (buf->ops)
			pipe_buf_release(pipe, buf);
1023 1024
	}

1025 1026
	if (!bytes)
		bytes = ret;
J
Jens Axboe 已提交
1027

1028
	goto done;
J
Jens Axboe 已提交
1029 1030 1031 1032 1033 1034
}
EXPORT_SYMBOL(splice_direct_to_actor);

static int direct_splice_actor(struct pipe_inode_info *pipe,
			       struct splice_desc *sd)
{
1035
	struct file *file = sd->u.file;
J
Jens Axboe 已提交
1036

1037
	return do_splice_from(pipe, file, sd->opos, sd->total_len,
1038
			      sd->flags);
J
Jens Axboe 已提交
1039 1040
}

1041 1042 1043 1044 1045
/**
 * do_splice_direct - splices data directly between two files
 * @in:		file to splice from
 * @ppos:	input file offset
 * @out:	file to splice to
1046
 * @opos:	output file offset
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    For use by do_sendfile(). splice can easily emulate sendfile, but
 *    doing it in the application would incur an extra system call
 *    (splice in + splice out, as compared to just sendfile()). So this helper
 *    can splice directly through a process-private pipe.
 *
 */
J
Jens Axboe 已提交
1057
long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1058
		      loff_t *opos, size_t len, unsigned int flags)
J
Jens Axboe 已提交
1059 1060 1061 1062 1063 1064
{
	struct splice_desc sd = {
		.len		= len,
		.total_len	= len,
		.flags		= flags,
		.pos		= *ppos,
1065
		.u.file		= out,
1066
		.opos		= opos,
J
Jens Axboe 已提交
1067
	};
1068
	long ret;
J
Jens Axboe 已提交
1069

1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
	if (unlikely(!(out->f_mode & FMODE_WRITE)))
		return -EBADF;

	if (unlikely(out->f_flags & O_APPEND))
		return -EINVAL;

	ret = rw_verify_area(WRITE, out, opos, len);
	if (unlikely(ret < 0))
		return ret;

J
Jens Axboe 已提交
1080
	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1081
	if (ret > 0)
1082
		*ppos = sd.pos;
1083

J
Jens Axboe 已提交
1084
	return ret;
1085
}
1086
EXPORT_SYMBOL(do_splice_direct);
1087

1088 1089
static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
{
1090 1091 1092 1093 1094 1095 1096
	for (;;) {
		if (unlikely(!pipe->readers)) {
			send_sig(SIGPIPE, current, 0);
			return -EPIPE;
		}
		if (pipe->nrbufs != pipe->buffers)
			return 0;
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
		if (flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;
		if (signal_pending(current))
			return -ERESTARTSYS;
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
	}
}

1107 1108 1109
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags);
1110

1111 1112 1113
/*
 * Determine where to splice to/from.
 */
1114 1115 1116
static long do_splice(struct file *in, loff_t __user *off_in,
		      struct file *out, loff_t __user *off_out,
		      size_t len, unsigned int flags)
1117
{
1118 1119
	struct pipe_inode_info *ipipe;
	struct pipe_inode_info *opipe;
1120
	loff_t offset;
J
Jens Axboe 已提交
1121
	long ret;
1122

1123 1124
	ipipe = get_pipe_info(in);
	opipe = get_pipe_info(out);
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143

	if (ipipe && opipe) {
		if (off_in || off_out)
			return -ESPIPE;

		if (!(in->f_mode & FMODE_READ))
			return -EBADF;

		if (!(out->f_mode & FMODE_WRITE))
			return -EBADF;

		/* Splicing to self would be fun, but... */
		if (ipipe == opipe)
			return -EINVAL;

		return splice_pipe_to_pipe(ipipe, opipe, len, flags);
	}

	if (ipipe) {
1144 1145
		if (off_in)
			return -ESPIPE;
1146
		if (off_out) {
1147
			if (!(out->f_mode & FMODE_PWRITE))
1148
				return -EINVAL;
1149
			if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1150
				return -EFAULT;
1151 1152 1153
		} else {
			offset = out->f_pos;
		}
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
		if (unlikely(!(out->f_mode & FMODE_WRITE)))
			return -EBADF;

		if (unlikely(out->f_flags & O_APPEND))
			return -EINVAL;

		ret = rw_verify_area(WRITE, out, &offset, len);
		if (unlikely(ret < 0))
			return ret;

1165
		file_start_write(out);
1166
		ret = do_splice_from(ipipe, out, &offset, len, flags);
1167
		file_end_write(out);
J
Jens Axboe 已提交
1168

1169 1170 1171
		if (!off_out)
			out->f_pos = offset;
		else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1172 1173 1174
			ret = -EFAULT;

		return ret;
1175
	}
1176

1177
	if (opipe) {
1178 1179
		if (off_out)
			return -ESPIPE;
1180
		if (off_in) {
1181
			if (!(in->f_mode & FMODE_PREAD))
1182
				return -EINVAL;
1183
			if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1184
				return -EFAULT;
1185 1186 1187
		} else {
			offset = in->f_pos;
		}
1188

1189 1190 1191 1192 1193 1194 1195
		pipe_lock(opipe);
		ret = wait_for_space(opipe, flags);
		if (!ret)
			ret = do_splice_to(in, &offset, opipe, len, flags);
		pipe_unlock(opipe);
		if (ret > 0)
			wakeup_pipe_readers(opipe);
1196 1197 1198
		if (!off_in)
			in->f_pos = offset;
		else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1199 1200 1201
			ret = -EFAULT;

		return ret;
1202
	}
1203 1204 1205 1206

	return -EINVAL;
}

A
Al Viro 已提交
1207 1208 1209
static int iter_to_pipe(struct iov_iter *from,
			struct pipe_inode_info *pipe,
			unsigned flags)
1210
{
A
Al Viro 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
	struct pipe_buffer buf = {
		.ops = &user_page_pipe_buf_ops,
		.flags = flags
	};
	size_t total = 0;
	int ret = 0;
	bool failed = false;

	while (iov_iter_count(from) && !failed) {
		struct page *pages[16];
1221 1222
		ssize_t copied;
		size_t start;
A
Al Viro 已提交
1223
		int n;
1224

A
Al Viro 已提交
1225 1226 1227 1228 1229
		copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
		if (copied <= 0) {
			ret = copied;
			break;
		}
1230

A
Al Viro 已提交
1231
		for (n = 0; copied; n++, start = 0) {
1232
			int size = min_t(int, copied, PAGE_SIZE - start);
A
Al Viro 已提交
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
			if (!failed) {
				buf.page = pages[n];
				buf.offset = start;
				buf.len = size;
				ret = add_to_pipe(pipe, &buf);
				if (unlikely(ret < 0)) {
					failed = true;
				} else {
					iov_iter_advance(from, ret);
					total += ret;
				}
			} else {
				put_page(pages[n]);
			}
1247
			copied -= size;
1248 1249
		}
	}
A
Al Viro 已提交
1250
	return total ? total : ret;
1251 1252
}

1253 1254 1255
static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			struct splice_desc *sd)
{
1256 1257
	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
	return n == sd->len ? n : -EFAULT;
1258 1259 1260 1261 1262 1263
}

/*
 * For lack of a better implementation, implement vmsplice() to userspace
 * as a simple copy of the pipes pages to the user iov.
 */
1264
static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov,
1265 1266 1267 1268 1269
			     unsigned long nr_segs, unsigned int flags)
{
	struct pipe_inode_info *pipe;
	struct splice_desc sd;
	long ret;
1270 1271 1272
	struct iovec iovstack[UIO_FASTIOV];
	struct iovec *iov = iovstack;
	struct iov_iter iter;
1273

1274
	pipe = get_pipe_info(file);
1275 1276 1277
	if (!pipe)
		return -EBADF;

1278 1279 1280 1281
	ret = import_iovec(READ, uiov, nr_segs,
			   ARRAY_SIZE(iovstack), &iov, &iter);
	if (ret < 0)
		return ret;
1282

1283
	sd.total_len = iov_iter_count(&iter);
1284 1285 1286 1287
	sd.len = 0;
	sd.flags = flags;
	sd.u.data = &iter;
	sd.pos = 0;
1288

1289 1290 1291 1292 1293
	if (sd.total_len) {
		pipe_lock(pipe);
		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
		pipe_unlock(pipe);
	}
1294

1295
	kfree(iov);
1296 1297 1298
	return ret;
}

1299 1300 1301 1302 1303
/*
 * vmsplice splices a user address range into a pipe. It can be thought of
 * as splice-from-memory, where the regular splice is splice-from-file (or
 * to file). In both cases the output is a pipe, naturally.
 */
1304
static long vmsplice_to_pipe(struct file *file, const struct iovec __user *uiov,
1305
			     unsigned long nr_segs, unsigned int flags)
1306
{
1307
	struct pipe_inode_info *pipe;
1308 1309 1310
	struct iovec iovstack[UIO_FASTIOV];
	struct iovec *iov = iovstack;
	struct iov_iter from;
1311
	long ret;
A
Al Viro 已提交
1312 1313 1314 1315
	unsigned buf_flag = 0;

	if (flags & SPLICE_F_GIFT)
		buf_flag = PIPE_BUF_FLAG_GIFT;
1316

1317
	pipe = get_pipe_info(file);
1318
	if (!pipe)
1319 1320
		return -EBADF;

1321 1322 1323 1324 1325
	ret = import_iovec(WRITE, uiov, nr_segs,
			   ARRAY_SIZE(iovstack), &iov, &from);
	if (ret < 0)
		return ret;

1326 1327
	pipe_lock(pipe);
	ret = wait_for_space(pipe, flags);
A
Al Viro 已提交
1328 1329
	if (!ret)
		ret = iter_to_pipe(&from, pipe, buf_flag);
1330 1331 1332
	pipe_unlock(pipe);
	if (ret > 0)
		wakeup_pipe_readers(pipe);
1333
	kfree(iov);
1334
	return ret;
1335 1336
}

1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
/*
 * Note that vmsplice only really supports true splicing _from_ user memory
 * to a pipe, not the other way around. Splicing from user memory is a simple
 * operation that can be supported without any funky alignment restrictions
 * or nasty vm tricks. We simply map in the user memory and fill them into
 * a pipe. The reverse isn't quite as easy, though. There are two possible
 * solutions for that:
 *
 *	- memcpy() the data internally, at which point we might as well just
 *	  do a regular read() on the buffer anyway.
 *	- Lots of nasty vm tricks, that are neither fast nor flexible (it
 *	  has restriction limitations on both ends of the pipe).
 *
 * Currently we punt and implement it as a normal copy, see pipe_to_user().
 *
 */
1353 1354
SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
		unsigned long, nr_segs, unsigned int, flags)
1355
{
1356
	struct fd f;
1357 1358
	long error;

1359 1360 1361 1362 1363
	if (unlikely(nr_segs > UIO_MAXIOV))
		return -EINVAL;
	else if (unlikely(!nr_segs))
		return 0;

1364
	error = -EBADF;
1365 1366 1367 1368 1369 1370 1371 1372
	f = fdget(fd);
	if (f.file) {
		if (f.file->f_mode & FMODE_WRITE)
			error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
		else if (f.file->f_mode & FMODE_READ)
			error = vmsplice_to_user(f.file, iov, nr_segs, flags);

		fdput(f);
1373 1374 1375 1376 1377
	}

	return error;
}

1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
		    unsigned int, nr_segs, unsigned int, flags)
{
	unsigned i;
	struct iovec __user *iov;
	if (nr_segs > UIO_MAXIOV)
		return -EINVAL;
	iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec));
	for (i = 0; i < nr_segs; i++) {
		struct compat_iovec v;
		if (get_user(v.iov_base, &iov32[i].iov_base) ||
		    get_user(v.iov_len, &iov32[i].iov_len) ||
		    put_user(compat_ptr(v.iov_base), &iov[i].iov_base) ||
		    put_user(v.iov_len, &iov[i].iov_len))
			return -EFAULT;
	}
	return sys_vmsplice(fd, iov, nr_segs, flags);
}
#endif

1399 1400 1401
SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
		int, fd_out, loff_t __user *, off_out,
		size_t, len, unsigned int, flags)
1402
{
1403
	struct fd in, out;
1404 1405 1406 1407 1408 1409
	long error;

	if (unlikely(!len))
		return 0;

	error = -EBADF;
1410 1411 1412 1413 1414 1415 1416 1417
	in = fdget(fd_in);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			out = fdget(fd_out);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_splice(in.file, off_in,
							  out.file, off_out,
1418
							  len, flags);
1419
				fdput(out);
1420 1421
			}
		}
1422
		fdput(in);
1423 1424 1425
	}
	return error;
}
1426

1427 1428 1429 1430
/*
 * Make sure there's data to read. Wait for input if we can, otherwise
 * return an appropriate error.
 */
1431
static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
	if (pipe->nrbufs)
		return 0;

	ret = 0;
1443
	pipe_lock(pipe);
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460

	while (!pipe->nrbufs) {
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		if (!pipe->writers)
			break;
		if (!pipe->waiting_writers) {
			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}
		}
		pipe_wait(pipe);
	}

1461
	pipe_unlock(pipe);
1462 1463 1464 1465 1466 1467 1468
	return ret;
}

/*
 * Make sure there's writeable room. Wait for room if we can, otherwise
 * return an appropriate error.
 */
1469
static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1470 1471 1472 1473 1474 1475 1476
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
1477
	if (pipe->nrbufs < pipe->buffers)
1478 1479 1480
		return 0;

	ret = 0;
1481
	pipe_lock(pipe);
1482

1483
	while (pipe->nrbufs >= pipe->buffers) {
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
		if (!pipe->readers) {
			send_sig(SIGPIPE, current, 0);
			ret = -EPIPE;
			break;
		}
		if (flags & SPLICE_F_NONBLOCK) {
			ret = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
	}

1502
	pipe_unlock(pipe);
1503 1504 1505
	return ret;
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
/*
 * Splice contents of ipipe to opipe.
 */
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
	int ret = 0, nbuf;
	bool input_wakeup = false;


retry:
	ret = ipipe_prep(ipipe, flags);
	if (ret)
		return ret;

	ret = opipe_prep(opipe, flags);
	if (ret)
		return ret;

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
	 * grabbing by pipe info address. Otherwise two different processes
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
	pipe_double_lock(ipipe, opipe);

	do {
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

		if (!ipipe->nrbufs && !ipipe->writers)
			break;

		/*
		 * Cannot make any progress, because either the input
		 * pipe is empty or the output pipe is full.
		 */
1549
		if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
			/* Already processed some buffers, break */
			if (ret)
				break;

			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}

			/*
			 * We raced with another reader/writer and haven't
			 * managed to process any buffers.  A zero return
			 * value means EOF, so retry instead.
			 */
			pipe_unlock(ipipe);
			pipe_unlock(opipe);
			goto retry;
		}

		ibuf = ipipe->bufs + ipipe->curbuf;
1570
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1571 1572 1573 1574 1575 1576 1577 1578 1579
		obuf = opipe->bufs + nbuf;

		if (len >= ibuf->len) {
			/*
			 * Simply move the whole buffer from ipipe to opipe
			 */
			*obuf = *ibuf;
			ibuf->ops = NULL;
			opipe->nrbufs++;
1580
			ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1581 1582 1583 1584 1585 1586 1587
			ipipe->nrbufs--;
			input_wakeup = true;
		} else {
			/*
			 * Get a reference to this pipe buffer,
			 * so we can copy the contents over.
			 */
M
Miklos Szeredi 已提交
1588
			pipe_buf_get(ipipe, ibuf);
1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
			*obuf = *ibuf;

			/*
			 * Don't inherit the gift flag, we need to
			 * prevent multiple steals of this page.
			 */
			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;

			obuf->len = len;
			opipe->nrbufs++;
			ibuf->offset += obuf->len;
			ibuf->len -= obuf->len;
		}
		ret += obuf->len;
		len -= obuf->len;
	} while (len);

	pipe_unlock(ipipe);
	pipe_unlock(opipe);

	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
N
Namhyung Kim 已提交
1612 1613 1614
	if (ret > 0)
		wakeup_pipe_readers(opipe);

1615 1616 1617 1618 1619 1620
	if (input_wakeup)
		wakeup_pipe_writers(ipipe);

	return ret;
}

1621 1622 1623 1624 1625 1626 1627 1628
/*
 * Link contents of ipipe to opipe.
 */
static int link_pipe(struct pipe_inode_info *ipipe,
		     struct pipe_inode_info *opipe,
		     size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
1629
	int ret = 0, i = 0, nbuf;
1630 1631 1632

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
1633
	 * grabbing by pipe info address. Otherwise two different processes
1634 1635
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
1636
	pipe_double_lock(ipipe, opipe);
1637

1638
	do {
1639 1640 1641 1642 1643 1644 1645
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

1646 1647 1648 1649
		/*
		 * If we have iterated all input buffers or ran out of
		 * output room, break.
		 */
1650
		if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
1651
			break;
1652

1653 1654
		ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1655 1656

		/*
1657 1658
		 * Get a reference to this pipe buffer,
		 * so we can copy the contents over.
1659
		 */
M
Miklos Szeredi 已提交
1660
		pipe_buf_get(ipipe, ibuf);
1661 1662 1663 1664

		obuf = opipe->bufs + nbuf;
		*obuf = *ibuf;

1665
		/*
1666 1667
		 * Don't inherit the gift flag, we need to
		 * prevent multiple steals of this page.
1668
		 */
1669
		obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1670

1671 1672
		if (obuf->len > len)
			obuf->len = len;
1673

1674 1675 1676 1677 1678
		opipe->nrbufs++;
		ret += obuf->len;
		len -= obuf->len;
		i++;
	} while (len);
1679

1680 1681 1682 1683 1684 1685 1686
	/*
	 * return EAGAIN if we have the potential of some data in the
	 * future, otherwise just return 0
	 */
	if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
		ret = -EAGAIN;

1687 1688
	pipe_unlock(ipipe);
	pipe_unlock(opipe);
1689

1690 1691 1692
	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
N
Namhyung Kim 已提交
1693 1694
	if (ret > 0)
		wakeup_pipe_readers(opipe);
1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707

	return ret;
}

/*
 * This is a tee(1) implementation that works on pipes. It doesn't copy
 * any data, it simply references the 'in' pages on the 'out' pipe.
 * The 'flags' used are the SPLICE_F_* variants, currently the only
 * applicable one is SPLICE_F_NONBLOCK.
 */
static long do_tee(struct file *in, struct file *out, size_t len,
		   unsigned int flags)
{
1708 1709
	struct pipe_inode_info *ipipe = get_pipe_info(in);
	struct pipe_inode_info *opipe = get_pipe_info(out);
1710
	int ret = -EINVAL;
1711 1712

	/*
1713 1714
	 * Duplicate the contents of ipipe to opipe without actually
	 * copying the data.
1715
	 */
1716 1717 1718 1719 1720
	if (ipipe && opipe && ipipe != opipe) {
		/*
		 * Keep going, unless we encounter an error. The ipipe/opipe
		 * ordering doesn't really matter.
		 */
1721
		ret = ipipe_prep(ipipe, flags);
1722
		if (!ret) {
1723
			ret = opipe_prep(opipe, flags);
1724
			if (!ret)
1725 1726 1727
				ret = link_pipe(ipipe, opipe, len, flags);
		}
	}
1728

1729
	return ret;
1730 1731
}

1732
SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1733
{
1734 1735
	struct fd in;
	int error;
1736 1737 1738 1739 1740

	if (unlikely(!len))
		return 0;

	error = -EBADF;
1741 1742 1743 1744 1745 1746 1747 1748 1749
	in = fdget(fdin);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			struct fd out = fdget(fdout);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_tee(in.file, out.file,
							len, flags);
				fdput(out);
1750 1751
			}
		}
1752
 		fdput(in);
1753 1754 1755 1756
 	}

	return error;
}