splice.c 49.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * "splice": joining two ropes together by interweaving their strands.
 *
 * This is the "extended pipe" functionality, where a pipe is used as
 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
 * buffer that you can use to transfer data from one end to the other.
 *
 * The traditional unix read/write is extended with a "splice()" operation
 * that transfers data buffers to or from a pipe buffer.
 *
 * Named by Larry McVoy, original implementation from Linus, extended by
12 13
 * Jens to support splicing to files, network, direct splicing, etc and
 * fixing lots of bugs.
14
 *
15
 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
16 17
 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
18 19 20 21 22
 *
 */
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>
23
#include <linux/splice.h>
24
#include <linux/memcontrol.h>
25
#include <linux/mm_inline.h>
26
#include <linux/swap.h>
27
#include <linux/writeback.h>
28
#include <linux/export.h>
29
#include <linux/syscalls.h>
30
#include <linux/uio.h>
31
#include <linux/security.h>
32
#include <linux/gfp.h>
33
#include <linux/socket.h>
34
#include <linux/compat.h>
35
#include <linux/aio.h>
36
#include "internal.h"
37

38 39 40 41 42 43
/*
 * Attempt to steal a page from a pipe buffer. This should perhaps go into
 * a vm helper function, it's already simplified quite a bit by the
 * addition of remove_mapping(). If success is returned, the caller may
 * attempt to reuse this page for another destination.
 */
44
static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
45 46 47
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
48
	struct address_space *mapping;
49

50 51
	lock_page(page);

52 53 54
	mapping = page_mapping(page);
	if (mapping) {
		WARN_ON(!PageUptodate(page));
55

56 57 58 59 60 61 62 63 64
		/*
		 * At least for ext2 with nobh option, we need to wait on
		 * writeback completing on this page, since we'll remove it
		 * from the pagecache.  Otherwise truncate wont wait on the
		 * page, allowing the disk blocks to be reused by someone else
		 * before we actually wrote our data to them. fs corruption
		 * ensues.
		 */
		wait_on_page_writeback(page);
65

66 67
		if (page_has_private(page) &&
		    !try_to_release_page(page, GFP_KERNEL))
68
			goto out_unlock;
69

70 71 72 73 74 75 76 77
		/*
		 * If we succeeded in removing the mapping, set LRU flag
		 * and return good.
		 */
		if (remove_mapping(mapping, page)) {
			buf->flags |= PIPE_BUF_FLAG_LRU;
			return 0;
		}
78
	}
79

80 81 82 83
	/*
	 * Raced with truncate or failed to remove page from current
	 * address space, unlock and return failure.
	 */
84
out_unlock:
85 86
	unlock_page(page);
	return 1;
87 88
}

89
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
90 91 92
					struct pipe_buffer *buf)
{
	page_cache_release(buf->page);
J
Jens Axboe 已提交
93
	buf->flags &= ~PIPE_BUF_FLAG_LRU;
94 95
}

96 97 98 99
/*
 * Check whether the contents of buf is OK to access. Since the content
 * is a page cache page, IO may be in flight.
 */
100 101
static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
				       struct pipe_buffer *buf)
102 103
{
	struct page *page = buf->page;
104
	int err;
105 106

	if (!PageUptodate(page)) {
107 108 109 110
		lock_page(page);

		/*
		 * Page got truncated/unhashed. This will cause a 0-byte
111
		 * splice, if this is the first page.
112 113 114 115 116
		 */
		if (!page->mapping) {
			err = -ENODATA;
			goto error;
		}
117

118
		/*
119
		 * Uh oh, read-error from disk.
120 121 122 123 124 125 126
		 */
		if (!PageUptodate(page)) {
			err = -EIO;
			goto error;
		}

		/*
127
		 * Page is ok afterall, we are done.
128
		 */
129 130 131
		unlock_page(page);
	}

132
	return 0;
133 134
error:
	unlock_page(page);
135
	return err;
136 137
}

138
const struct pipe_buf_operations page_cache_pipe_buf_ops = {
139
	.can_merge = 0,
140
	.confirm = page_cache_pipe_buf_confirm,
141
	.release = page_cache_pipe_buf_release,
142
	.steal = page_cache_pipe_buf_steal,
143
	.get = generic_pipe_buf_get,
144 145
};

146 147 148
static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
149 150 151
	if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
		return 1;

J
Jens Axboe 已提交
152
	buf->flags |= PIPE_BUF_FLAG_LRU;
153
	return generic_pipe_buf_steal(pipe, buf);
154 155
}

156
static const struct pipe_buf_operations user_page_pipe_buf_ops = {
157
	.can_merge = 0,
158
	.confirm = generic_pipe_buf_confirm,
159 160
	.release = page_cache_pipe_buf_release,
	.steal = user_page_pipe_buf_steal,
161
	.get = generic_pipe_buf_get,
162 163
};

164 165 166 167 168 169 170 171
static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
}

172 173 174 175 176 177
/**
 * splice_to_pipe - fill passed data into a pipe
 * @pipe:	pipe to fill
 * @spd:	data to fill
 *
 * Description:
178
 *    @spd contains a map of pages and len/offset tuples, along with
179 180 181
 *    the struct pipe_buf_operations associated with these pages. This
 *    function will link that data to the pipe.
 *
182
 */
183 184
ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
		       struct splice_pipe_desc *spd)
185
{
186
	unsigned int spd_pages = spd->nr_pages;
187
	int ret, do_wakeup, page_nr;
188 189 190

	ret = 0;
	do_wakeup = 0;
191
	page_nr = 0;
192

193
	pipe_lock(pipe);
194 195

	for (;;) {
196
		if (!pipe->readers) {
197 198 199 200 201 202
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

203 204
		if (pipe->nrbufs < pipe->buffers) {
			int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
205
			struct pipe_buffer *buf = pipe->bufs + newbuf;
206

207 208 209
			buf->page = spd->pages[page_nr];
			buf->offset = spd->partial[page_nr].offset;
			buf->len = spd->partial[page_nr].len;
210
			buf->private = spd->partial[page_nr].private;
211
			buf->ops = spd->ops;
212 213 214
			if (spd->flags & SPLICE_F_GIFT)
				buf->flags |= PIPE_BUF_FLAG_GIFT;

215
			pipe->nrbufs++;
216 217 218
			page_nr++;
			ret += buf->len;

A
Al Viro 已提交
219
			if (pipe->files)
220
				do_wakeup = 1;
221

222
			if (!--spd->nr_pages)
223
				break;
224
			if (pipe->nrbufs < pipe->buffers)
225 226 227 228 229
				continue;

			break;
		}

230
		if (spd->flags & SPLICE_F_NONBLOCK) {
231 232 233 234 235
			if (!ret)
				ret = -EAGAIN;
			break;
		}

236 237 238 239 240 241 242
		if (signal_pending(current)) {
			if (!ret)
				ret = -ERESTARTSYS;
			break;
		}

		if (do_wakeup) {
243
			smp_mb();
244 245 246
			if (waitqueue_active(&pipe->wait))
				wake_up_interruptible_sync(&pipe->wait);
			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
247 248 249
			do_wakeup = 0;
		}

250 251 252
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
253 254
	}

255
	pipe_unlock(pipe);
256

257 258
	if (do_wakeup)
		wakeup_pipe_readers(pipe);
259

260
	while (page_nr < spd_pages)
261
		spd->spd_release(spd, page_nr++);
262 263 264 265

	return ret;
}

266
void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
267 268 269 270
{
	page_cache_release(spd->pages[i]);
}

271 272 273 274
/*
 * Check if we need to grow the arrays holding pages and partial page
 * descriptions.
 */
275
int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
276
{
277 278 279 280
	unsigned int buffers = ACCESS_ONCE(pipe->buffers);

	spd->nr_pages_max = buffers;
	if (buffers <= PIPE_DEF_BUFFERS)
281 282
		return 0;

283 284
	spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
	spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
285 286 287 288 289 290 291 292 293

	if (spd->pages && spd->partial)
		return 0;

	kfree(spd->pages);
	kfree(spd->partial);
	return -ENOMEM;
}

294
void splice_shrink_spd(struct splice_pipe_desc *spd)
295
{
296
	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
297 298 299 300 301 302
		return;

	kfree(spd->pages);
	kfree(spd->partial);
}

303
static int
304 305 306
__generic_file_splice_read(struct file *in, loff_t *ppos,
			   struct pipe_inode_info *pipe, size_t len,
			   unsigned int flags)
307 308
{
	struct address_space *mapping = in->f_mapping;
309
	unsigned int loff, nr_pages, req_pages;
310 311
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
312
	struct page *page;
313 314
	pgoff_t index, end_index;
	loff_t isize;
315
	int error, page_nr;
316 317 318
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
319
		.nr_pages_max = PIPE_DEF_BUFFERS,
320 321
		.flags = flags,
		.ops = &page_cache_pipe_buf_ops,
322
		.spd_release = spd_release_page,
323
	};
324

325 326 327
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

328
	index = *ppos >> PAGE_CACHE_SHIFT;
329
	loff = *ppos & ~PAGE_CACHE_MASK;
330
	req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
331
	nr_pages = min(req_pages, spd.nr_pages_max);
332

333 334 335
	/*
	 * Lookup the (hopefully) full range of pages we need.
	 */
336
	spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
337
	index += spd.nr_pages;
338

339 340
	/*
	 * If find_get_pages_contig() returned fewer pages than we needed,
341
	 * readahead/allocate the rest and fill in the holes.
342
	 */
343
	if (spd.nr_pages < nr_pages)
344 345
		page_cache_sync_readahead(mapping, &in->f_ra, in,
				index, req_pages - spd.nr_pages);
346

347
	error = 0;
348
	while (spd.nr_pages < nr_pages) {
349
		/*
350 351
		 * Page could be there, find_get_pages_contig() breaks on
		 * the first hole.
352
		 */
353 354 355
		page = find_get_page(mapping, index);
		if (!page) {
			/*
356
			 * page didn't exist, allocate one.
357 358 359 360 361 362
			 */
			page = page_cache_alloc_cold(mapping);
			if (!page)
				break;

			error = add_to_page_cache_lru(page, mapping, index,
363
						GFP_KERNEL);
364 365
			if (unlikely(error)) {
				page_cache_release(page);
366 367
				if (error == -EEXIST)
					continue;
368 369
				break;
			}
370 371 372 373 374
			/*
			 * add_to_page_cache() locks the page, unlock it
			 * to avoid convoluting the logic below even more.
			 */
			unlock_page(page);
375 376
		}

377
		spd.pages[spd.nr_pages++] = page;
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
		index++;
	}

	/*
	 * Now loop over the map and see if we need to start IO on any
	 * pages, fill in the partial map, etc.
	 */
	index = *ppos >> PAGE_CACHE_SHIFT;
	nr_pages = spd.nr_pages;
	spd.nr_pages = 0;
	for (page_nr = 0; page_nr < nr_pages; page_nr++) {
		unsigned int this_len;

		if (!len)
			break;

		/*
		 * this_len is the max we'll use from this page
		 */
		this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
398
		page = spd.pages[page_nr];
399

400
		if (PageReadahead(page))
401
			page_cache_async_readahead(mapping, &in->f_ra, in,
402
					page, index, req_pages - page_nr);
403

404 405 406 407
		/*
		 * If the page isn't uptodate, we may need to start io on it
		 */
		if (!PageUptodate(page)) {
408
			lock_page(page);
409 410

			/*
411 412 413 414
			 * Page was truncated, or invalidated by the
			 * filesystem.  Redo the find/create, but this time the
			 * page is kept locked, so there's no chance of another
			 * race with truncate/invalidate.
415 416 417
			 */
			if (!page->mapping) {
				unlock_page(page);
418 419 420 421 422 423 424
				page = find_or_create_page(mapping, index,
						mapping_gfp_mask(mapping));

				if (!page) {
					error = -ENOMEM;
					break;
				}
425 426
				page_cache_release(spd.pages[page_nr]);
				spd.pages[page_nr] = page;
427 428 429 430 431 432 433 434
			}
			/*
			 * page was already under io and is now done, great
			 */
			if (PageUptodate(page)) {
				unlock_page(page);
				goto fill_it;
			}
435

436 437 438 439
			/*
			 * need to read in the page
			 */
			error = mapping->a_ops->readpage(in, page);
440
			if (unlikely(error)) {
441 442 443 444 445 446
				/*
				 * We really should re-lookup the page here,
				 * but it complicates things a lot. Instead
				 * lets just do what we already stored, and
				 * we'll get it the next time we are called.
				 */
447
				if (error == AOP_TRUNCATED_PAGE)
448 449
					error = 0;

450 451
				break;
			}
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
		}
fill_it:
		/*
		 * i_size must be checked after PageUptodate.
		 */
		isize = i_size_read(mapping->host);
		end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
		if (unlikely(!isize || index > end_index))
			break;

		/*
		 * if this is the last page, see if we need to shrink
		 * the length and stop
		 */
		if (end_index == index) {
			unsigned int plen;
468 469

			/*
470
			 * max good bytes in this page
471
			 */
472 473
			plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
			if (plen <= loff)
474 475 476
				break;

			/*
477
			 * force quit after adding this page
478
			 */
479 480
			this_len = min(this_len, plen - loff);
			len = this_len;
481
		}
482

483 484
		spd.partial[page_nr].offset = loff;
		spd.partial[page_nr].len = this_len;
485
		len -= this_len;
486
		loff = 0;
487 488
		spd.nr_pages++;
		index++;
489 490
	}

491
	/*
492
	 * Release any pages at the end, if we quit early. 'page_nr' is how far
493 494 495
	 * we got, 'nr_pages' is how many pages are in the map.
	 */
	while (page_nr < nr_pages)
496
		page_cache_release(spd.pages[page_nr++]);
497
	in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
498

499
	if (spd.nr_pages)
500
		error = splice_to_pipe(pipe, &spd);
501

502
	splice_shrink_spd(&spd);
503
	return error;
504 505
}

506 507 508
/**
 * generic_file_splice_read - splice data from file to a pipe
 * @in:		file to splice from
509
 * @ppos:	position in @in
510 511 512 513
 * @pipe:	pipe to splice to
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
514 515 516 517 518
 * Description:
 *    Will read pages from given file and fill them into a pipe. Can be
 *    used as long as the address_space operations for the source implements
 *    a readpage() hook.
 *
519
 */
520 521 522
ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
523
{
524
	loff_t isize, left;
525
	int ret;
526 527 528 529 530 531 532 533

	isize = i_size_read(in->f_mapping->host);
	if (unlikely(*ppos >= isize))
		return 0;

	left = isize - *ppos;
	if (unlikely(left < len))
		len = left;
534

535
	ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
536
	if (ret > 0) {
537
		*ppos += ret;
538 539
		file_accessed(in);
	}
540 541 542

	return ret;
}
543 544
EXPORT_SYMBOL(generic_file_splice_read);

545 546 547 548 549 550 551 552
static const struct pipe_buf_operations default_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};

553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	return 1;
}

/* Pipe buffer operations for a socket and similar. */
const struct pipe_buf_operations nosteal_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_nosteal,
	.get = generic_pipe_buf_get,
};
EXPORT_SYMBOL(nosteal_pipe_buf_ops);

569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
			    unsigned long vlen, loff_t offset)
{
	mm_segment_t old_fs;
	loff_t pos = offset;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
	res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
	set_fs(old_fs);

	return res;
}

585
ssize_t kernel_write(struct file *file, const char *buf, size_t count,
586
			    loff_t pos)
587 588 589 590 591 592 593
{
	mm_segment_t old_fs;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
594
	res = vfs_write(file, (__force const char __user *)buf, count, &pos);
595 596 597 598
	set_fs(old_fs);

	return res;
}
599
EXPORT_SYMBOL(kernel_write);
600

601 602 603 604 605 606 607
ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
{
	unsigned int nr_pages;
	unsigned int nr_freed;
	size_t offset;
608 609 610
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
	struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
611 612 613 614 615 616 617
	ssize_t res;
	size_t this_len;
	int error;
	int i;
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
618
		.nr_pages_max = PIPE_DEF_BUFFERS,
619 620 621 622 623
		.flags = flags,
		.ops = &default_pipe_buf_ops,
		.spd_release = spd_release_page,
	};

624 625 626 627 628
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

	res = -ENOMEM;
	vec = __vec;
629 630
	if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
		vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
631 632 633 634
		if (!vec)
			goto shrink_ret;
	}

635 636 637
	offset = *ppos & ~PAGE_CACHE_MASK;
	nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;

638
	for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
639 640
		struct page *page;

641
		page = alloc_page(GFP_USER);
642 643 644 645 646
		error = -ENOMEM;
		if (!page)
			goto err;

		this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
647
		vec[i].iov_base = (void __user *) page_address(page);
648
		vec[i].iov_len = this_len;
649
		spd.pages[i] = page;
650 651 652 653 654 655
		spd.nr_pages++;
		len -= this_len;
		offset = 0;
	}

	res = kernel_readv(in, vec, spd.nr_pages, *ppos);
656 657
	if (res < 0) {
		error = res;
658
		goto err;
659
	}
660 661 662 663 664 665 666 667

	error = 0;
	if (!res)
		goto err;

	nr_freed = 0;
	for (i = 0; i < spd.nr_pages; i++) {
		this_len = min_t(size_t, vec[i].iov_len, res);
668 669
		spd.partial[i].offset = 0;
		spd.partial[i].len = this_len;
670
		if (!this_len) {
671 672
			__free_page(spd.pages[i]);
			spd.pages[i] = NULL;
673 674 675 676 677 678 679 680 681 682
			nr_freed++;
		}
		res -= this_len;
	}
	spd.nr_pages -= nr_freed;

	res = splice_to_pipe(pipe, &spd);
	if (res > 0)
		*ppos += res;

683 684 685
shrink_ret:
	if (vec != __vec)
		kfree(vec);
686
	splice_shrink_spd(&spd);
687 688 689
	return res;

err:
690
	for (i = 0; i < spd.nr_pages; i++)
691
		__free_page(spd.pages[i]);
692

693 694
	res = error;
	goto shrink_ret;
695 696 697
}
EXPORT_SYMBOL(default_file_splice_read);

698
/*
699
 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
700
 * using sendpage(). Return the number of bytes sent.
701
 */
702
static int pipe_to_sendpage(struct pipe_inode_info *pipe,
703 704
			    struct pipe_buffer *buf, struct splice_desc *sd)
{
705
	struct file *file = sd->u.file;
706
	loff_t pos = sd->pos;
707
	int more;
708

A
Al Viro 已提交
709
	if (!likely(file->f_op->sendpage))
710 711
		return -EINVAL;

712
	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
713 714

	if (sd->len < sd->total_len && pipe->nrbufs > 1)
715
		more |= MSG_SENDPAGE_NOTLAST;
716

717 718
	return file->f_op->sendpage(file, buf->page, buf->offset,
				    sd->len, &pos, more);
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
}

/*
 * This is a little more tricky than the file -> pipe splicing. There are
 * basically three cases:
 *
 *	- Destination page already exists in the address space and there
 *	  are users of it. For that case we have no other option that
 *	  copying the data. Tough luck.
 *	- Destination page already exists in the address space, but there
 *	  are no users of it. Make sure it's uptodate, then drop it. Fall
 *	  through to last case.
 *	- Destination page does not exist, we can add the pipe page to
 *	  the page cache and avoid the copy.
 *
734 735 736 737 738 739
 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
 * sd->flags), we attempt to migrate pages from the pipe to the output
 * file address space page cache. This is possible if no one else has
 * the pipe page referenced outside of the pipe and page cache. If
 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
 * a new page in the output file page cache and fill/dirty that.
740
 */
741 742
int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
		 struct splice_desc *sd)
743
{
744
	struct file *file = sd->u.file;
745
	struct address_space *mapping = file->f_mapping;
746
	unsigned int offset, this_len;
747
	struct page *page;
748
	void *fsdata;
749
	int ret;
750 751 752

	offset = sd->pos & ~PAGE_CACHE_MASK;

753 754 755 756
	this_len = sd->len;
	if (this_len + offset > PAGE_CACHE_SIZE)
		this_len = PAGE_CACHE_SIZE - offset;

757 758 759 760
	ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
				AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
	if (unlikely(ret))
		goto out;
761

762
	if (buf->page != page) {
763
		char *src = kmap_atomic(buf->page);
764
		char *dst = kmap_atomic(page);
765

766
		memcpy(dst + offset, src + buf->offset, this_len);
767
		flush_dcache_page(page);
768
		kunmap_atomic(dst);
769
		kunmap_atomic(src);
770
	}
771 772
	ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
				page, fsdata);
773 774 775
out:
	return ret;
}
776
EXPORT_SYMBOL(pipe_to_file);
777

778 779 780 781 782 783 784 785
static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
}

786
/**
787
 * splice_from_pipe_feed - feed available data from a pipe to a file
788 789 790 791 792
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
793 794 795 796 797 798 799
 *    This function loops over the pipe and calls @actor to do the
 *    actual moving of a single struct pipe_buffer to the desired
 *    destination.  It returns when there's no more buffers left in
 *    the pipe or if the requested number of bytes (@sd->total_len)
 *    have been copied.  It returns a positive number (one) if the
 *    pipe needs to be filled with more data, zero if the required
 *    number of bytes have been copied and -errno on error.
800
 *
801 802 803 804
 *    This, together with splice_from_pipe_{begin,end,next}, may be
 *    used to implement the functionality of __splice_from_pipe() when
 *    locking is required around copying the pipe buffers to the
 *    destination.
805
 */
806 807
int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
			  splice_actor *actor)
808
{
809
	int ret;
810

811 812 813
	while (pipe->nrbufs) {
		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
		const struct pipe_buf_operations *ops = buf->ops;
814

815 816 817
		sd->len = buf->len;
		if (sd->len > sd->total_len)
			sd->len = sd->total_len;
818

819 820
		ret = buf->ops->confirm(pipe, buf);
		if (unlikely(ret)) {
821 822 823 824
			if (ret == -ENODATA)
				ret = 0;
			return ret;
		}
825 826 827 828 829

		ret = actor(pipe, buf, sd);
		if (ret <= 0)
			return ret;

830 831 832 833 834 835 836 837 838 839 840
		buf->offset += ret;
		buf->len -= ret;

		sd->num_spliced += ret;
		sd->len -= ret;
		sd->pos += ret;
		sd->total_len -= ret;

		if (!buf->len) {
			buf->ops = NULL;
			ops->release(pipe, buf);
841
			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
842
			pipe->nrbufs--;
A
Al Viro 已提交
843
			if (pipe->files)
844 845
				sd->need_wakeup = true;
		}
846

847 848 849
		if (!sd->total_len)
			return 0;
	}
850

851 852 853
	return 1;
}
EXPORT_SYMBOL(splice_from_pipe_feed);
854

855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
/**
 * splice_from_pipe_next - wait for some data to splice from
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wait for some data and return a positive
 *    value (one) if pipe buffers are available.  It will return zero
 *    or -errno if no more data needs to be spliced.
 */
int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
{
	while (!pipe->nrbufs) {
		if (!pipe->writers)
			return 0;
870

871 872
		if (!pipe->waiting_writers && sd->num_spliced)
			return 0;
873

874 875
		if (sd->flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;
876

877 878
		if (signal_pending(current))
			return -ERESTARTSYS;
879

880 881 882
		if (sd->need_wakeup) {
			wakeup_pipe_writers(pipe);
			sd->need_wakeup = false;
883 884
		}

885 886
		pipe_wait(pipe);
	}
887

888 889 890
	return 1;
}
EXPORT_SYMBOL(splice_from_pipe_next);
891

892 893
/**
 * splice_from_pipe_begin - start splicing from pipe
894
 * @sd:		information about the splice operation
895 896 897 898 899 900 901 902 903 904 905 906
 *
 * Description:
 *    This function should be called before a loop containing
 *    splice_from_pipe_next() and splice_from_pipe_feed() to
 *    initialize the necessary fields of @sd.
 */
void splice_from_pipe_begin(struct splice_desc *sd)
{
	sd->num_spliced = 0;
	sd->need_wakeup = false;
}
EXPORT_SYMBOL(splice_from_pipe_begin);
907

908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
/**
 * splice_from_pipe_end - finish splicing from pipe
 * @pipe:	pipe to splice from
 * @sd:		information about the splice operation
 *
 * Description:
 *    This function will wake up pipe writers if necessary.  It should
 *    be called after a loop containing splice_from_pipe_next() and
 *    splice_from_pipe_feed().
 */
void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
{
	if (sd->need_wakeup)
		wakeup_pipe_writers(pipe);
}
EXPORT_SYMBOL(splice_from_pipe_end);
924

925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
/**
 * __splice_from_pipe - splice data from a pipe to given actor
 * @pipe:	pipe to splice from
 * @sd:		information to @actor
 * @actor:	handler that splices the data
 *
 * Description:
 *    This function does little more than loop over the pipe and call
 *    @actor to do the actual moving of a single struct pipe_buffer to
 *    the desired destination. See pipe_to_file, pipe_to_sendpage, or
 *    pipe_to_user.
 *
 */
ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
			   splice_actor *actor)
{
	int ret;
942

943 944 945 946 947 948 949 950 951
	splice_from_pipe_begin(sd);
	do {
		ret = splice_from_pipe_next(pipe, sd);
		if (ret > 0)
			ret = splice_from_pipe_feed(pipe, sd, actor);
	} while (ret > 0);
	splice_from_pipe_end(pipe, sd);

	return sd->num_spliced ? sd->num_spliced : ret;
952
}
M
Mark Fasheh 已提交
953
EXPORT_SYMBOL(__splice_from_pipe);
954

955 956 957 958 959 960 961 962 963 964
/**
 * splice_from_pipe - splice data from a pipe to a file
 * @pipe:	pipe to splice from
 * @out:	file to splice to
 * @ppos:	position in @out
 * @len:	how many bytes to splice
 * @flags:	splice modifier flags
 * @actor:	handler that splices the data
 *
 * Description:
965
 *    See __splice_from_pipe. This function locks the pipe inode,
966 967 968
 *    otherwise it's identical to __splice_from_pipe().
 *
 */
969 970 971 972 973
ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
			 loff_t *ppos, size_t len, unsigned int flags,
			 splice_actor *actor)
{
	ssize_t ret;
974 975 976 977
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
978
		.u.file = out,
979
	};
980

981
	pipe_lock(pipe);
982
	ret = __splice_from_pipe(pipe, &sd, actor);
983
	pipe_unlock(pipe);
984 985 986 987

	return ret;
}

988 989
/**
 * generic_file_splice_write - splice data from a pipe to a file
990
 * @pipe:	pipe info
991
 * @out:	file to write to
992
 * @ppos:	position in @out
993 994 995
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
996 997 998
 * Description:
 *    Will either move or copy pages (determined by @flags options) from
 *    the given pipe inode to the given file.
999 1000
 *
 */
1001 1002
ssize_t
generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
1003
			  loff_t *ppos, size_t len, unsigned int flags)
1004
{
1005
	struct address_space *mapping = out->f_mapping;
1006
	struct inode *inode = mapping->host;
1007 1008 1009 1010 1011 1012
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
	};
1013 1014
	ssize_t ret;

1015
	pipe_lock(pipe);
1016 1017 1018 1019 1020 1021 1022 1023 1024

	splice_from_pipe_begin(&sd);
	do {
		ret = splice_from_pipe_next(pipe, &sd);
		if (ret <= 0)
			break;

		mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
		ret = file_remove_suid(out);
1025
		if (!ret) {
1026 1027 1028 1029
			ret = file_update_time(out);
			if (!ret)
				ret = splice_from_pipe_feed(pipe, &sd,
							    pipe_to_file);
1030
		}
1031 1032 1033 1034
		mutex_unlock(&inode->i_mutex);
	} while (ret > 0);
	splice_from_pipe_end(pipe, &sd);

1035
	pipe_unlock(pipe);
1036 1037 1038 1039

	if (sd.num_spliced)
		ret = sd.num_spliced;

J
Jens Axboe 已提交
1040
	if (ret > 0) {
1041
		int err;
1042

1043 1044 1045 1046 1047
		err = generic_write_sync(out, *ppos, ret);
		if (err)
			ret = err;
		else
			*ppos += ret;
1048
		balance_dirty_pages_ratelimited(mapping);
1049 1050 1051
	}

	return ret;
1052 1053
}

1054 1055
EXPORT_SYMBOL(generic_file_splice_write);

1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
/**
 * iter_file_splice_write - splice data from a pipe to a file
 * @pipe:	pipe info
 * @out:	file to write to
 * @ppos:	position in @out
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    Will either move or copy pages (determined by @flags options) from
 *    the given pipe inode to the given file.
 *    This one is ->write_iter-based.
 *
 */
ssize_t
iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
			  loff_t *ppos, size_t len, unsigned int flags)
{
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
	};
	int nbufs = pipe->buffers;
	struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
	ssize_t ret;

	if (unlikely(!array))
		return -ENOMEM;

	pipe_lock(pipe);

	splice_from_pipe_begin(&sd);
	while (sd.total_len) {
		struct iov_iter from;
		struct kiocb kiocb;
		size_t left;
		int n, idx;

		ret = splice_from_pipe_next(pipe, &sd);
		if (ret <= 0)
			break;

		if (unlikely(nbufs < pipe->buffers)) {
			kfree(array);
			nbufs = pipe->buffers;
			array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
			if (!array) {
				ret = -ENOMEM;
				break;
			}
		}

		/* build the vector */
		left = sd.total_len;
		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {
			struct pipe_buffer *buf = pipe->bufs + idx;
			size_t this_len = buf->len;

			if (this_len > left)
				this_len = left;

			if (idx == pipe->buffers - 1)
				idx = -1;

			ret = buf->ops->confirm(pipe, buf);
			if (unlikely(ret)) {
				if (ret == -ENODATA)
					ret = 0;
				goto done;
			}

			array[n].bv_page = buf->page;
			array[n].bv_len = this_len;
			array[n].bv_offset = buf->offset;
			left -= this_len;
		}

		/* ... iov_iter */
		from.type = ITER_BVEC | WRITE;
		from.bvec = array;
		from.nr_segs = n;
		from.count = sd.total_len - left;
		from.iov_offset = 0;

		/* ... and iocb */
		init_sync_kiocb(&kiocb, out);
		kiocb.ki_pos = sd.pos;
		kiocb.ki_nbytes = sd.total_len - left;

		/* now, send it */
		ret = out->f_op->write_iter(&kiocb, &from);
		if (-EIOCBQUEUED == ret)
			ret = wait_on_sync_kiocb(&kiocb);

		if (ret <= 0)
			break;

		sd.num_spliced += ret;
		sd.total_len -= ret;
		*ppos = sd.pos = kiocb.ki_pos;

		/* dismiss the fully eaten buffers, adjust the partial one */
		while (ret) {
			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
			if (ret >= buf->len) {
				const struct pipe_buf_operations *ops = buf->ops;
				ret -= buf->len;
				buf->len = 0;
				buf->ops = NULL;
				ops->release(pipe, buf);
				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
				pipe->nrbufs--;
				if (pipe->files)
					sd.need_wakeup = true;
			} else {
				buf->offset += ret;
				buf->len -= ret;
				ret = 0;
			}
		}
	}
done:
	kfree(array);
	splice_from_pipe_end(pipe, &sd);

	pipe_unlock(pipe);

	if (sd.num_spliced)
		ret = sd.num_spliced;

	return ret;
}

EXPORT_SYMBOL(iter_file_splice_write);

1195 1196
static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			  struct splice_desc *sd)
1197
{
1198 1199
	int ret;
	void *data;
1200
	loff_t tmp = sd->pos;
1201

1202
	data = kmap(buf->page);
1203
	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
1204
	kunmap(buf->page);
1205 1206

	return ret;
1207 1208 1209 1210 1211 1212
}

static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
					 struct file *out, loff_t *ppos,
					 size_t len, unsigned int flags)
{
1213
	ssize_t ret;
1214

1215 1216 1217
	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
	if (ret > 0)
		*ppos += ret;
1218

1219
	return ret;
1220 1221
}

1222 1223
/**
 * generic_splice_sendpage - splice data from a pipe to a socket
1224
 * @pipe:	pipe to splice from
1225
 * @out:	socket to write to
1226
 * @ppos:	position in @out
1227 1228 1229
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
1230 1231 1232
 * Description:
 *    Will send @len bytes from the pipe to a network socket. No data copying
 *    is involved.
1233 1234
 *
 */
1235
ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1236
				loff_t *ppos, size_t len, unsigned int flags)
1237
{
1238
	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1239 1240
}

1241
EXPORT_SYMBOL(generic_splice_sendpage);
J
Jeff Garzik 已提交
1242

1243 1244 1245
/*
 * Attempt to initiate a splice from pipe to file.
 */
1246
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1247
			   loff_t *ppos, size_t len, unsigned int flags)
1248
{
1249 1250
	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
				loff_t *, size_t, unsigned int);
1251

A
Al Viro 已提交
1252
	if (out->f_op->splice_write)
1253 1254
		splice_write = out->f_op->splice_write;
	else
1255 1256
		splice_write = default_file_splice_write;

1257
	return splice_write(pipe, out, ppos, len, flags);
1258 1259
}

1260 1261 1262
/*
 * Attempt to initiate a splice from a file to a pipe.
 */
1263 1264 1265
static long do_splice_to(struct file *in, loff_t *ppos,
			 struct pipe_inode_info *pipe, size_t len,
			 unsigned int flags)
1266
{
1267 1268
	ssize_t (*splice_read)(struct file *, loff_t *,
			       struct pipe_inode_info *, size_t, unsigned int);
1269 1270
	int ret;

1271
	if (unlikely(!(in->f_mode & FMODE_READ)))
1272 1273
		return -EBADF;

1274
	ret = rw_verify_area(READ, in, ppos, len);
1275 1276 1277
	if (unlikely(ret < 0))
		return ret;

A
Al Viro 已提交
1278
	if (in->f_op->splice_read)
1279 1280
		splice_read = in->f_op->splice_read;
	else
1281 1282 1283
		splice_read = default_file_splice_read;

	return splice_read(in, ppos, pipe, len, flags);
1284 1285
}

1286 1287 1288 1289 1290 1291 1292 1293 1294
/**
 * splice_direct_to_actor - splices data directly between two non-pipes
 * @in:		file to splice from
 * @sd:		actor information on where to splice to
 * @actor:	handles the data splicing
 *
 * Description:
 *    This is a special case helper to splice directly between two
 *    points, without requiring an explicit pipe. Internally an allocated
1295
 *    pipe is cached in the process, and reused during the lifetime of
1296 1297
 *    that process.
 *
1298 1299 1300
 */
ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
			       splice_direct_actor *actor)
1301 1302 1303 1304
{
	struct pipe_inode_info *pipe;
	long ret, bytes;
	umode_t i_mode;
1305 1306
	size_t len;
	int i, flags;
1307 1308 1309 1310 1311 1312

	/*
	 * We require the input being a regular file, as we don't want to
	 * randomly drop data for eg socket -> socket splicing. Use the
	 * piped splicing for that!
	 */
A
Al Viro 已提交
1313
	i_mode = file_inode(in)->i_mode;
1314 1315 1316 1317 1318 1319 1320 1321
	if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
		return -EINVAL;

	/*
	 * neither in nor out is a pipe, setup an internal pipe attached to
	 * 'out' and transfer the wanted data from 'in' to 'out' through that
	 */
	pipe = current->splice_pipe;
1322
	if (unlikely(!pipe)) {
1323
		pipe = alloc_pipe_info();
1324 1325 1326 1327 1328
		if (!pipe)
			return -ENOMEM;

		/*
		 * We don't have an immediate reader, but we'll read the stuff
1329
		 * out of the pipe right after the splice_to_pipe(). So set
1330 1331 1332 1333 1334 1335 1336 1337
		 * PIPE_READERS appropriately.
		 */
		pipe->readers = 1;

		current->splice_pipe = pipe;
	}

	/*
1338
	 * Do the splice.
1339 1340 1341
	 */
	ret = 0;
	bytes = 0;
1342 1343 1344 1345 1346 1347 1348
	len = sd->total_len;
	flags = sd->flags;

	/*
	 * Don't block on output, we have to drain the direct pipe.
	 */
	sd->flags &= ~SPLICE_F_NONBLOCK;
1349 1350

	while (len) {
1351
		size_t read_len;
1352
		loff_t pos = sd->pos, prev_pos = pos;
1353

1354
		ret = do_splice_to(in, &pos, pipe, len, flags);
1355
		if (unlikely(ret <= 0))
1356 1357 1358
			goto out_release;

		read_len = ret;
1359
		sd->total_len = read_len;
1360 1361 1362 1363 1364 1365

		/*
		 * NOTE: nonblocking mode only applies to the input. We
		 * must not do the output in nonblocking mode as then we
		 * could get stuck data in the internal pipe:
		 */
1366
		ret = actor(pipe, sd);
1367 1368
		if (unlikely(ret <= 0)) {
			sd->pos = prev_pos;
1369
			goto out_release;
1370
		}
1371 1372 1373

		bytes += ret;
		len -= ret;
1374
		sd->pos = pos;
1375

1376 1377
		if (ret < read_len) {
			sd->pos = prev_pos + ret;
1378
			goto out_release;
1379
		}
1380 1381
	}

1382
done:
1383
	pipe->nrbufs = pipe->curbuf = 0;
1384
	file_accessed(in);
1385 1386 1387 1388 1389 1390 1391
	return bytes;

out_release:
	/*
	 * If we did an incomplete transfer we must release
	 * the pipe buffers in question:
	 */
1392
	for (i = 0; i < pipe->buffers; i++) {
1393 1394 1395 1396 1397 1398 1399 1400
		struct pipe_buffer *buf = pipe->bufs + i;

		if (buf->ops) {
			buf->ops->release(pipe, buf);
			buf->ops = NULL;
		}
	}

1401 1402
	if (!bytes)
		bytes = ret;
1403

1404
	goto done;
1405 1406 1407 1408 1409 1410
}
EXPORT_SYMBOL(splice_direct_to_actor);

static int direct_splice_actor(struct pipe_inode_info *pipe,
			       struct splice_desc *sd)
{
1411
	struct file *file = sd->u.file;
1412

1413
	return do_splice_from(pipe, file, sd->opos, sd->total_len,
1414
			      sd->flags);
1415 1416
}

1417 1418 1419 1420 1421
/**
 * do_splice_direct - splices data directly between two files
 * @in:		file to splice from
 * @ppos:	input file offset
 * @out:	file to splice to
1422
 * @opos:	output file offset
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
 * @len:	number of bytes to splice
 * @flags:	splice modifier flags
 *
 * Description:
 *    For use by do_sendfile(). splice can easily emulate sendfile, but
 *    doing it in the application would incur an extra system call
 *    (splice in + splice out, as compared to just sendfile()). So this helper
 *    can splice directly through a process-private pipe.
 *
 */
1433
long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1434
		      loff_t *opos, size_t len, unsigned int flags)
1435 1436 1437 1438 1439 1440
{
	struct splice_desc sd = {
		.len		= len,
		.total_len	= len,
		.flags		= flags,
		.pos		= *ppos,
1441
		.u.file		= out,
1442
		.opos		= opos,
1443
	};
1444
	long ret;
1445

1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
	if (unlikely(!(out->f_mode & FMODE_WRITE)))
		return -EBADF;

	if (unlikely(out->f_flags & O_APPEND))
		return -EINVAL;

	ret = rw_verify_area(WRITE, out, opos, len);
	if (unlikely(ret < 0))
		return ret;

1456
	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1457
	if (ret > 0)
1458
		*ppos = sd.pos;
1459

1460
	return ret;
1461 1462
}

1463 1464 1465
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags);
1466

1467 1468 1469
/*
 * Determine where to splice to/from.
 */
1470 1471 1472
static long do_splice(struct file *in, loff_t __user *off_in,
		      struct file *out, loff_t __user *off_out,
		      size_t len, unsigned int flags)
1473
{
1474 1475
	struct pipe_inode_info *ipipe;
	struct pipe_inode_info *opipe;
1476
	loff_t offset;
J
Jens Axboe 已提交
1477
	long ret;
1478

1479 1480
	ipipe = get_pipe_info(in);
	opipe = get_pipe_info(out);
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499

	if (ipipe && opipe) {
		if (off_in || off_out)
			return -ESPIPE;

		if (!(in->f_mode & FMODE_READ))
			return -EBADF;

		if (!(out->f_mode & FMODE_WRITE))
			return -EBADF;

		/* Splicing to self would be fun, but... */
		if (ipipe == opipe)
			return -EINVAL;

		return splice_pipe_to_pipe(ipipe, opipe, len, flags);
	}

	if (ipipe) {
1500 1501
		if (off_in)
			return -ESPIPE;
1502
		if (off_out) {
1503
			if (!(out->f_mode & FMODE_PWRITE))
1504
				return -EINVAL;
1505
			if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1506
				return -EFAULT;
1507 1508 1509
		} else {
			offset = out->f_pos;
		}
1510

1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
		if (unlikely(!(out->f_mode & FMODE_WRITE)))
			return -EBADF;

		if (unlikely(out->f_flags & O_APPEND))
			return -EINVAL;

		ret = rw_verify_area(WRITE, out, &offset, len);
		if (unlikely(ret < 0))
			return ret;

1521
		file_start_write(out);
1522
		ret = do_splice_from(ipipe, out, &offset, len, flags);
1523
		file_end_write(out);
J
Jens Axboe 已提交
1524

1525 1526 1527
		if (!off_out)
			out->f_pos = offset;
		else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1528 1529 1530
			ret = -EFAULT;

		return ret;
1531
	}
1532

1533
	if (opipe) {
1534 1535
		if (off_out)
			return -ESPIPE;
1536
		if (off_in) {
1537
			if (!(in->f_mode & FMODE_PREAD))
1538
				return -EINVAL;
1539
			if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1540
				return -EFAULT;
1541 1542 1543
		} else {
			offset = in->f_pos;
		}
1544

1545
		ret = do_splice_to(in, &offset, opipe, len, flags);
J
Jens Axboe 已提交
1546

1547 1548 1549
		if (!off_in)
			in->f_pos = offset;
		else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
J
Jens Axboe 已提交
1550 1551 1552
			ret = -EFAULT;

		return ret;
1553
	}
1554 1555 1556 1557

	return -EINVAL;
}

1558 1559 1560 1561 1562 1563 1564 1565 1566
/*
 * Map an iov into an array of pages and offset/length tupples. With the
 * partial_page structure, we can map several non-contiguous ranges into
 * our ones pages[] map instead of splitting that operation into pieces.
 * Could easily be exported as a generic helper for other users, in which
 * case one would probably want to add a 'max_nr_pages' parameter as well.
 */
static int get_iovec_page_array(const struct iovec __user *iov,
				unsigned int nr_vecs, struct page **pages,
1567
				struct partial_page *partial, bool aligned,
1568
				unsigned int pipe_buffers)
1569 1570 1571 1572 1573
{
	int buffers = 0, error = 0;

	while (nr_vecs) {
		unsigned long off, npages;
1574
		struct iovec entry;
1575 1576 1577 1578
		void __user *base;
		size_t len;
		int i;

1579
		error = -EFAULT;
1580
		if (copy_from_user(&entry, iov, sizeof(entry)))
1581 1582
			break;

1583 1584 1585
		base = entry.iov_base;
		len = entry.iov_len;

1586 1587 1588
		/*
		 * Sanity check this iovec. 0 read succeeds.
		 */
1589
		error = 0;
1590 1591 1592
		if (unlikely(!len))
			break;
		error = -EFAULT;
1593
		if (!access_ok(VERIFY_READ, base, len))
1594 1595 1596 1597 1598 1599 1600
			break;

		/*
		 * Get this base offset and number of pages, then map
		 * in the user pages.
		 */
		off = (unsigned long) base & ~PAGE_MASK;
1601 1602 1603 1604 1605 1606 1607 1608 1609

		/*
		 * If asked for alignment, the offset must be zero and the
		 * length a multiple of the PAGE_SIZE.
		 */
		error = -EINVAL;
		if (aligned && (off || len & ~PAGE_MASK))
			break;

1610
		npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1611 1612
		if (npages > pipe_buffers - buffers)
			npages = pipe_buffers - buffers;
1613

1614 1615
		error = get_user_pages_fast((unsigned long)base, npages,
					0, &pages[buffers]);
1616 1617 1618 1619 1620 1621 1622 1623

		if (unlikely(error <= 0))
			break;

		/*
		 * Fill this contiguous range into the partial page map.
		 */
		for (i = 0; i < error; i++) {
1624
			const int plen = min_t(size_t, len, PAGE_SIZE - off);
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646

			partial[buffers].offset = off;
			partial[buffers].len = plen;

			off = 0;
			len -= plen;
			buffers++;
		}

		/*
		 * We didn't complete this iov, stop here since it probably
		 * means we have to move some of this into a pipe to
		 * be able to continue.
		 */
		if (len)
			break;

		/*
		 * Don't continue if we mapped fewer pages than we asked for,
		 * or if we mapped the max number of pages that we have
		 * room for.
		 */
1647
		if (error < npages || buffers == pipe_buffers)
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659
			break;

		nr_vecs--;
		iov++;
	}

	if (buffers)
		return buffers;

	return error;
}

1660 1661 1662
static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			struct splice_desc *sd)
{
1663 1664
	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
	return n == sd->len ? n : -EFAULT;
1665 1666 1667 1668 1669 1670
}

/*
 * For lack of a better implementation, implement vmsplice() to userspace
 * as a simple copy of the pipes pages to the user iov.
 */
1671
static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov,
1672 1673 1674 1675 1676
			     unsigned long nr_segs, unsigned int flags)
{
	struct pipe_inode_info *pipe;
	struct splice_desc sd;
	long ret;
1677 1678 1679 1680
	struct iovec iovstack[UIO_FASTIOV];
	struct iovec *iov = iovstack;
	struct iov_iter iter;
	ssize_t count = 0;
1681

1682
	pipe = get_pipe_info(file);
1683 1684 1685
	if (!pipe)
		return -EBADF;

1686 1687 1688 1689
	ret = rw_copy_check_uvector(READ, uiov, nr_segs,
				    ARRAY_SIZE(iovstack), iovstack, &iov);
	if (ret <= 0)
		return ret;
1690

1691
	iov_iter_init(&iter, READ, iov, nr_segs, count);
1692

1693 1694 1695 1696 1697
	sd.len = 0;
	sd.total_len = count;
	sd.flags = flags;
	sd.u.data = &iter;
	sd.pos = 0;
1698

1699 1700
	pipe_lock(pipe);
	ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1701
	pipe_unlock(pipe);
1702

1703 1704
	if (iov != iovstack)
		kfree(iov);
1705 1706 1707 1708

	return ret;
}

1709 1710 1711 1712 1713
/*
 * vmsplice splices a user address range into a pipe. It can be thought of
 * as splice-from-memory, where the regular splice is splice-from-file (or
 * to file). In both cases the output is a pipe, naturally.
 */
1714 1715
static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
			     unsigned long nr_segs, unsigned int flags)
1716
{
1717
	struct pipe_inode_info *pipe;
1718 1719
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
1720 1721 1722
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
1723
		.nr_pages_max = PIPE_DEF_BUFFERS,
1724 1725
		.flags = flags,
		.ops = &user_page_pipe_buf_ops,
1726
		.spd_release = spd_release_page,
1727
	};
1728
	long ret;
1729

1730
	pipe = get_pipe_info(file);
1731
	if (!pipe)
1732 1733
		return -EBADF;

1734 1735 1736 1737
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

	spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1738
					    spd.partial, false,
1739
					    spd.nr_pages_max);
1740
	if (spd.nr_pages <= 0)
1741 1742 1743
		ret = spd.nr_pages;
	else
		ret = splice_to_pipe(pipe, &spd);
1744

1745
	splice_shrink_spd(&spd);
1746
	return ret;
1747 1748
}

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
/*
 * Note that vmsplice only really supports true splicing _from_ user memory
 * to a pipe, not the other way around. Splicing from user memory is a simple
 * operation that can be supported without any funky alignment restrictions
 * or nasty vm tricks. We simply map in the user memory and fill them into
 * a pipe. The reverse isn't quite as easy, though. There are two possible
 * solutions for that:
 *
 *	- memcpy() the data internally, at which point we might as well just
 *	  do a regular read() on the buffer anyway.
 *	- Lots of nasty vm tricks, that are neither fast nor flexible (it
 *	  has restriction limitations on both ends of the pipe).
 *
 * Currently we punt and implement it as a normal copy, see pipe_to_user().
 *
 */
1765 1766
SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
		unsigned long, nr_segs, unsigned int, flags)
1767
{
1768
	struct fd f;
1769 1770
	long error;

1771 1772 1773 1774 1775
	if (unlikely(nr_segs > UIO_MAXIOV))
		return -EINVAL;
	else if (unlikely(!nr_segs))
		return 0;

1776
	error = -EBADF;
1777 1778 1779 1780 1781 1782 1783 1784
	f = fdget(fd);
	if (f.file) {
		if (f.file->f_mode & FMODE_WRITE)
			error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
		else if (f.file->f_mode & FMODE_READ)
			error = vmsplice_to_user(f.file, iov, nr_segs, flags);

		fdput(f);
1785 1786 1787 1788 1789
	}

	return error;
}

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
		    unsigned int, nr_segs, unsigned int, flags)
{
	unsigned i;
	struct iovec __user *iov;
	if (nr_segs > UIO_MAXIOV)
		return -EINVAL;
	iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec));
	for (i = 0; i < nr_segs; i++) {
		struct compat_iovec v;
		if (get_user(v.iov_base, &iov32[i].iov_base) ||
		    get_user(v.iov_len, &iov32[i].iov_len) ||
		    put_user(compat_ptr(v.iov_base), &iov[i].iov_base) ||
		    put_user(v.iov_len, &iov[i].iov_len))
			return -EFAULT;
	}
	return sys_vmsplice(fd, iov, nr_segs, flags);
}
#endif

1811 1812 1813
SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
		int, fd_out, loff_t __user *, off_out,
		size_t, len, unsigned int, flags)
1814
{
1815
	struct fd in, out;
1816 1817 1818 1819 1820 1821
	long error;

	if (unlikely(!len))
		return 0;

	error = -EBADF;
1822 1823 1824 1825 1826 1827 1828 1829
	in = fdget(fd_in);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			out = fdget(fd_out);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_splice(in.file, off_in,
							  out.file, off_out,
1830
							  len, flags);
1831
				fdput(out);
1832 1833
			}
		}
1834
		fdput(in);
1835 1836 1837
	}
	return error;
}
1838

1839 1840 1841 1842
/*
 * Make sure there's data to read. Wait for input if we can, otherwise
 * return an appropriate error.
 */
1843
static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
	if (pipe->nrbufs)
		return 0;

	ret = 0;
1855
	pipe_lock(pipe);
1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872

	while (!pipe->nrbufs) {
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		if (!pipe->writers)
			break;
		if (!pipe->waiting_writers) {
			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}
		}
		pipe_wait(pipe);
	}

1873
	pipe_unlock(pipe);
1874 1875 1876 1877 1878 1879 1880
	return ret;
}

/*
 * Make sure there's writeable room. Wait for room if we can, otherwise
 * return an appropriate error.
 */
1881
static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1882 1883 1884 1885 1886 1887 1888
{
	int ret;

	/*
	 * Check ->nrbufs without the inode lock first. This function
	 * is speculative anyways, so missing one is ok.
	 */
1889
	if (pipe->nrbufs < pipe->buffers)
1890 1891 1892
		return 0;

	ret = 0;
1893
	pipe_lock(pipe);
1894

1895
	while (pipe->nrbufs >= pipe->buffers) {
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913
		if (!pipe->readers) {
			send_sig(SIGPIPE, current, 0);
			ret = -EPIPE;
			break;
		}
		if (flags & SPLICE_F_NONBLOCK) {
			ret = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
	}

1914
	pipe_unlock(pipe);
1915 1916 1917
	return ret;
}

1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
/*
 * Splice contents of ipipe to opipe.
 */
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
			       struct pipe_inode_info *opipe,
			       size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
	int ret = 0, nbuf;
	bool input_wakeup = false;


retry:
	ret = ipipe_prep(ipipe, flags);
	if (ret)
		return ret;

	ret = opipe_prep(opipe, flags);
	if (ret)
		return ret;

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
	 * grabbing by pipe info address. Otherwise two different processes
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
	pipe_double_lock(ipipe, opipe);

	do {
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

		if (!ipipe->nrbufs && !ipipe->writers)
			break;

		/*
		 * Cannot make any progress, because either the input
		 * pipe is empty or the output pipe is full.
		 */
1961
		if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
			/* Already processed some buffers, break */
			if (ret)
				break;

			if (flags & SPLICE_F_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}

			/*
			 * We raced with another reader/writer and haven't
			 * managed to process any buffers.  A zero return
			 * value means EOF, so retry instead.
			 */
			pipe_unlock(ipipe);
			pipe_unlock(opipe);
			goto retry;
		}

		ibuf = ipipe->bufs + ipipe->curbuf;
1982
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1983 1984 1985 1986 1987 1988 1989 1990 1991
		obuf = opipe->bufs + nbuf;

		if (len >= ibuf->len) {
			/*
			 * Simply move the whole buffer from ipipe to opipe
			 */
			*obuf = *ibuf;
			ibuf->ops = NULL;
			opipe->nrbufs++;
1992
			ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
			ipipe->nrbufs--;
			input_wakeup = true;
		} else {
			/*
			 * Get a reference to this pipe buffer,
			 * so we can copy the contents over.
			 */
			ibuf->ops->get(ipipe, ibuf);
			*obuf = *ibuf;

			/*
			 * Don't inherit the gift flag, we need to
			 * prevent multiple steals of this page.
			 */
			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;

			obuf->len = len;
			opipe->nrbufs++;
			ibuf->offset += obuf->len;
			ibuf->len -= obuf->len;
		}
		ret += obuf->len;
		len -= obuf->len;
	} while (len);

	pipe_unlock(ipipe);
	pipe_unlock(opipe);

	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
2024 2025 2026
	if (ret > 0)
		wakeup_pipe_readers(opipe);

2027 2028 2029 2030 2031 2032
	if (input_wakeup)
		wakeup_pipe_writers(ipipe);

	return ret;
}

2033 2034 2035 2036 2037 2038 2039 2040
/*
 * Link contents of ipipe to opipe.
 */
static int link_pipe(struct pipe_inode_info *ipipe,
		     struct pipe_inode_info *opipe,
		     size_t len, unsigned int flags)
{
	struct pipe_buffer *ibuf, *obuf;
2041
	int ret = 0, i = 0, nbuf;
2042 2043 2044

	/*
	 * Potential ABBA deadlock, work around it by ordering lock
2045
	 * grabbing by pipe info address. Otherwise two different processes
2046 2047
	 * could deadlock (one doing tee from A -> B, the other from B -> A).
	 */
2048
	pipe_double_lock(ipipe, opipe);
2049

2050
	do {
2051 2052 2053 2054 2055 2056 2057
		if (!opipe->readers) {
			send_sig(SIGPIPE, current, 0);
			if (!ret)
				ret = -EPIPE;
			break;
		}

2058 2059 2060 2061
		/*
		 * If we have iterated all input buffers or ran out of
		 * output room, break.
		 */
2062
		if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
2063
			break;
2064

2065 2066
		ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
		nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
2067 2068

		/*
2069 2070
		 * Get a reference to this pipe buffer,
		 * so we can copy the contents over.
2071
		 */
2072 2073 2074 2075 2076
		ibuf->ops->get(ipipe, ibuf);

		obuf = opipe->bufs + nbuf;
		*obuf = *ibuf;

2077
		/*
2078 2079
		 * Don't inherit the gift flag, we need to
		 * prevent multiple steals of this page.
2080
		 */
2081
		obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2082

2083 2084
		if (obuf->len > len)
			obuf->len = len;
2085

2086 2087 2088 2089 2090
		opipe->nrbufs++;
		ret += obuf->len;
		len -= obuf->len;
		i++;
	} while (len);
2091

2092 2093 2094 2095 2096 2097 2098
	/*
	 * return EAGAIN if we have the potential of some data in the
	 * future, otherwise just return 0
	 */
	if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
		ret = -EAGAIN;

2099 2100
	pipe_unlock(ipipe);
	pipe_unlock(opipe);
2101

2102 2103 2104
	/*
	 * If we put data in the output pipe, wakeup any potential readers.
	 */
2105 2106
	if (ret > 0)
		wakeup_pipe_readers(opipe);
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119

	return ret;
}

/*
 * This is a tee(1) implementation that works on pipes. It doesn't copy
 * any data, it simply references the 'in' pages on the 'out' pipe.
 * The 'flags' used are the SPLICE_F_* variants, currently the only
 * applicable one is SPLICE_F_NONBLOCK.
 */
static long do_tee(struct file *in, struct file *out, size_t len,
		   unsigned int flags)
{
2120 2121
	struct pipe_inode_info *ipipe = get_pipe_info(in);
	struct pipe_inode_info *opipe = get_pipe_info(out);
2122
	int ret = -EINVAL;
2123 2124

	/*
2125 2126
	 * Duplicate the contents of ipipe to opipe without actually
	 * copying the data.
2127
	 */
2128 2129 2130 2131 2132
	if (ipipe && opipe && ipipe != opipe) {
		/*
		 * Keep going, unless we encounter an error. The ipipe/opipe
		 * ordering doesn't really matter.
		 */
2133
		ret = ipipe_prep(ipipe, flags);
2134
		if (!ret) {
2135
			ret = opipe_prep(opipe, flags);
2136
			if (!ret)
2137 2138 2139
				ret = link_pipe(ipipe, opipe, len, flags);
		}
	}
2140

2141
	return ret;
2142 2143
}

2144
SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2145
{
2146 2147
	struct fd in;
	int error;
2148 2149 2150 2151 2152

	if (unlikely(!len))
		return 0;

	error = -EBADF;
2153 2154 2155 2156 2157 2158 2159 2160 2161
	in = fdget(fdin);
	if (in.file) {
		if (in.file->f_mode & FMODE_READ) {
			struct fd out = fdget(fdout);
			if (out.file) {
				if (out.file->f_mode & FMODE_WRITE)
					error = do_tee(in.file, out.file,
							len, flags);
				fdput(out);
2162 2163
			}
		}
2164
 		fdput(in);
2165 2166 2167 2168
 	}

	return error;
}
新手
引导
客服 返回
顶部