io_uring.c 199.6 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
47
#include <net/compat.h>
J
Jens Axboe 已提交
48 49
#include <linux/refcount.h>
#include <linux/uio.h>
50
#include <linux/bits.h>
J
Jens Axboe 已提交
51 52 53 54 55 56 57 58 59

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
60
#include <linux/kthread.h>
J
Jens Axboe 已提交
61
#include <linux/blkdev.h>
62
#include <linux/bvec.h>
J
Jens Axboe 已提交
63 64 65
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
66
#include <net/scm.h>
J
Jens Axboe 已提交
67 68 69 70
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
71 72
#include <linux/sizes.h>
#include <linux/hugetlb.h>
73
#include <linux/highmem.h>
74 75
#include <linux/namei.h>
#include <linux/fsnotify.h>
J
Jens Axboe 已提交
76
#include <linux/fadvise.h>
77
#include <linux/eventpoll.h>
78
#include <linux/fs_struct.h>
P
Pavel Begunkov 已提交
79
#include <linux/splice.h>
80
#include <linux/task_work.h>
81
#include <linux/pagemap.h>
J
Jens Axboe 已提交
82

83 84 85
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
86 87 88
#include <uapi/linux/io_uring.h>

#include "internal.h"
89
#include "io-wq.h"
J
Jens Axboe 已提交
90

91
#define IORING_MAX_ENTRIES	32768
92
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
93 94 95 96 97 98 99 100

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
101 102 103 104 105 106

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
107
/*
108 109
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
110 111 112 113
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
114
struct io_rings {
S
Stefan Bühler 已提交
115 116 117 118
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
119 120 121
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
122
	 */
123
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
124
	/*
125
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
126 127
	 * ring_entries - 1)
	 */
128 129 130
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
131 132 133 134 135 136 137 138 139 140 141 142
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
143
	u32			sq_dropped;
S
Stefan Bühler 已提交
144
	/*
145
	 * Runtime SQ flags
S
Stefan Bühler 已提交
146 147 148 149 150 151 152
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
153
	u32			sq_flags;
154 155 156 157 158 159 160
	/*
	 * Runtime CQ flags
	 *
	 * Written by the application, shouldn't be modified by the
	 * kernel.
	 */
	u32                     cq_flags;
S
Stefan Bühler 已提交
161 162 163
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
L
LimingWu 已提交
164
	 * there are not more requests pending than there is space in
S
Stefan Bühler 已提交
165 166 167 168 169 170 171 172 173
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
174
	u32			cq_overflow;
S
Stefan Bühler 已提交
175 176 177 178 179 180 181
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
182
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
183 184
};

185 186 187 188 189 190 191
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

192 193
struct fixed_file_table {
	struct file		**files;
194 195
};

196 197 198 199 200
struct fixed_file_ref_node {
	struct percpu_ref		refs;
	struct list_head		node;
	struct list_head		file_list;
	struct fixed_file_data		*file_data;
201
	struct llist_node		llist;
202 203
};

204 205 206 207
struct fixed_file_data {
	struct fixed_file_table		*table;
	struct io_ring_ctx		*ctx;

208
	struct percpu_ref		*cur_refs;
209 210
	struct percpu_ref		refs;
	struct completion		done;
211 212
	struct list_head		ref_list;
	spinlock_t			lock;
213 214
};

215 216 217 218 219 220 221
struct io_buffer {
	struct list_head list;
	__u64 addr;
	__s32 len;
	__u16 bid;
};

J
Jens Axboe 已提交
222 223 224 225 226 227 228
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
229
		unsigned int		compat: 1;
230
		unsigned int		limit_mem: 1;
231 232 233
		unsigned int		cq_overflow_flushed: 1;
		unsigned int		drain_next: 1;
		unsigned int		eventfd_async: 1;
J
Jens Axboe 已提交
234

235 236 237 238 239 240 241 242 243 244 245 246
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
247 248 249
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
250
		unsigned		sq_thread_idle;
251
		unsigned		cached_sq_dropped;
252
		atomic_t		cached_cq_overflow;
253
		unsigned long		sq_check_overflow;
254 255

		struct list_head	defer_list;
J
Jens Axboe 已提交
256
		struct list_head	timeout_list;
257
		struct list_head	cq_overflow_list;
258 259

		wait_queue_head_t	inflight_wait;
260
		struct io_uring_sqe	*sq_sqes;
J
Jens Axboe 已提交
261 262
	} ____cacheline_aligned_in_smp;

263 264
	struct io_rings	*rings;

J
Jens Axboe 已提交
265
	/* IO offload */
266
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
267
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
268
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
269
	wait_queue_head_t	sqo_wait;
270

J
Jens Axboe 已提交
271 272 273 274 275
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
276
	struct fixed_file_data	*file_data;
J
Jens Axboe 已提交
277
	unsigned		nr_user_files;
P
Pavel Begunkov 已提交
278 279
	int 			ring_fd;
	struct file 		*ring_file;
J
Jens Axboe 已提交
280

281 282 283 284
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
285 286
	struct user_struct	*user;

287
	const struct cred	*creds;
288

289 290
	struct completion	ref_comp;
	struct completion	sq_thread_comp;
291

292 293 294
	/* if all else fails... */
	struct io_kiocb		*fallback_req;

295 296 297 298
#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif

299 300
	struct idr		io_buffer_idr;

301 302
	struct idr		personality_idr;

303 304 305 306 307
	struct {
		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		unsigned		cq_mask;
		atomic_t		cq_timeouts;
308
		unsigned long		cq_check_overflow;
309 310 311 312
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
		struct eventfd_ctx	*cq_ev_fd;
	} ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
313 314 315 316 317 318 319 320

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
321

J
Jens Axboe 已提交
322 323 324 325 326 327 328
		/*
		 * ->poll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct list_head	poll_list;
329 330
		struct hlist_head	*cancel_hash;
		unsigned		cancel_hash_bits;
331
		bool			poll_multi_file;
332

333 334
		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
335
	} ____cacheline_aligned_in_smp;
336

337 338 339
	struct delayed_work		file_put_work;
	struct llist_head		file_put_llist;

340
	struct work_struct		exit_work;
J
Jens Axboe 已提交
341 342
};

J
Jens Axboe 已提交
343 344 345 346
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
347 348
struct io_poll_iocb {
	struct file			*file;
349 350 351 352
	union {
		struct wait_queue_head	*head;
		u64			addr;
	};
353
	__poll_t			events;
J
Jens Axboe 已提交
354
	bool				done;
355
	bool				canceled;
356
	struct wait_queue_entry		wait;
357 358
};

359 360 361 362 363 364
struct io_close {
	struct file			*file;
	struct file			*put_file;
	int				fd;
};

365 366 367 368 369 370 371
struct io_timeout_data {
	struct io_kiocb			*req;
	struct hrtimer			timer;
	struct timespec64		ts;
	enum hrtimer_mode		mode;
};

372 373 374 375 376
struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
377
	unsigned long			nofile;
378 379 380 381 382 383 384
};

struct io_sync {
	struct file			*file;
	loff_t				len;
	loff_t				off;
	int				flags;
385
	int				mode;
386 387
};

388 389 390 391 392
struct io_cancel {
	struct file			*file;
	u64				addr;
};

393 394 395 396
struct io_timeout {
	struct file			*file;
	u64				addr;
	int				flags;
397 398
	u32				off;
	u32				target_seq;
399 400
};

401 402 403 404 405 406 407
struct io_rw {
	/* NOTE: kiocb has the file as the first member, so don't do it here */
	struct kiocb			kiocb;
	u64				addr;
	u64				len;
};

408 409 410 411 412 413
struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
};

414 415
struct io_sr_msg {
	struct file			*file;
416 417 418 419
	union {
		struct user_msghdr __user *msg;
		void __user		*buf;
	};
420
	int				msg_flags;
421
	int				bgid;
422
	size_t				len;
423
	struct io_buffer		*kbuf;
424 425
};

426 427 428 429
struct io_open {
	struct file			*file;
	int				dfd;
	struct filename			*filename;
430
	struct open_how			how;
431
	unsigned long			nofile;
432 433
};

434 435 436 437 438 439 440
struct io_files_update {
	struct file			*file;
	u64				arg;
	u32				nr_args;
	u32				offset;
};

J
Jens Axboe 已提交
441 442 443 444 445 446 447
struct io_fadvise {
	struct file			*file;
	u64				offset;
	u32				len;
	u32				advice;
};

J
Jens Axboe 已提交
448 449 450 451 452 453 454
struct io_madvise {
	struct file			*file;
	u64				addr;
	u32				len;
	u32				advice;
};

455 456 457 458 459 460
struct io_epoll {
	struct file			*file;
	int				epfd;
	int				op;
	int				fd;
	struct epoll_event		event;
461 462
};

P
Pavel Begunkov 已提交
463 464 465 466 467 468 469 470 471
struct io_splice {
	struct file			*file_out;
	struct file			*file_in;
	loff_t				off_out;
	loff_t				off_in;
	u64				len;
	unsigned int			flags;
};

472 473 474 475 476 477 478 479 480
struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__s32				len;
	__u32				bgid;
	__u16				nbufs;
	__u16				bid;
};

481 482 483 484 485
struct io_statx {
	struct file			*file;
	int				dfd;
	unsigned int			mask;
	unsigned int			flags;
B
Bijan Mottahedeh 已提交
486
	const char __user		*filename;
487 488 489
	struct statx __user		*buffer;
};

490 491 492 493
struct io_async_connect {
	struct sockaddr_storage		address;
};

494 495 496 497 498
struct io_async_msghdr {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	struct sockaddr __user		*uaddr;
	struct msghdr			msg;
499
	struct sockaddr_storage		addr;
500 501
};

502 503 504 505 506
struct io_async_rw {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	ssize_t				nr_segs;
	ssize_t				size;
507 508
	struct wait_page_queue		wpq;
	struct callback_head		task_work;
509 510
};

511
struct io_async_ctx {
512 513
	union {
		struct io_async_rw	rw;
514
		struct io_async_msghdr	msg;
515
		struct io_async_connect	connect;
516
		struct io_timeout_data	timeout;
517
	};
518 519
};

520 521 522 523 524 525
enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
526
	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
527

528
	REQ_F_LINK_HEAD_BIT,
529 530 531 532 533 534 535 536 537 538
	REQ_F_LINK_NEXT_BIT,
	REQ_F_FAIL_LINK_BIT,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_TIMEOUT_BIT,
	REQ_F_ISREG_BIT,
	REQ_F_TIMEOUT_NOSEQ_BIT,
	REQ_F_COMP_LOCKED_BIT,
P
Pavel Begunkov 已提交
539
	REQ_F_NEED_CLEANUP_BIT,
540
	REQ_F_OVERFLOW_BIT,
541
	REQ_F_POLLED_BIT,
542
	REQ_F_BUFFER_SELECTED_BIT,
543
	REQ_F_NO_FILE_TABLE_BIT,
544
	REQ_F_QUEUE_TIMEOUT_BIT,
545
	REQ_F_WORK_INITIALIZED_BIT,
P
Pavel Begunkov 已提交
546
	REQ_F_TASK_PINNED_BIT,
547 548 549

	/* not a real bit, just to check we're not overflowing the space */
	__REQ_F_LAST_BIT,
550 551 552 553 554 555 556 557 558 559 560 561 562
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),
563 564
	/* IOSQE_BUFFER_SELECT */
	REQ_F_BUFFER_SELECT	= BIT(REQ_F_BUFFER_SELECT_BIT),
565

566 567
	/* head of a link */
	REQ_F_LINK_HEAD		= BIT(REQ_F_LINK_HEAD_BIT),
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	/* already grabbed next link */
	REQ_F_LINK_NEXT		= BIT(REQ_F_LINK_NEXT_BIT),
	/* fail rest of links */
	REQ_F_FAIL_LINK		= BIT(REQ_F_FAIL_LINK_BIT),
	/* on inflight list */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* has linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* timeout request */
	REQ_F_TIMEOUT		= BIT(REQ_F_TIMEOUT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* no timeout sequence */
	REQ_F_TIMEOUT_NOSEQ	= BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
	/* completion under lock */
	REQ_F_COMP_LOCKED	= BIT(REQ_F_COMP_LOCKED_BIT),
P
Pavel Begunkov 已提交
588 589
	/* needs cleanup */
	REQ_F_NEED_CLEANUP	= BIT(REQ_F_NEED_CLEANUP_BIT),
590 591
	/* in overflow list */
	REQ_F_OVERFLOW		= BIT(REQ_F_OVERFLOW_BIT),
592 593
	/* already went through poll handler */
	REQ_F_POLLED		= BIT(REQ_F_POLLED_BIT),
594 595
	/* buffer already selected */
	REQ_F_BUFFER_SELECTED	= BIT(REQ_F_BUFFER_SELECTED_BIT),
596 597
	/* doesn't need file table for this request */
	REQ_F_NO_FILE_TABLE	= BIT(REQ_F_NO_FILE_TABLE_BIT),
598 599
	/* needs to queue linked timeout */
	REQ_F_QUEUE_TIMEOUT	= BIT(REQ_F_QUEUE_TIMEOUT_BIT),
600 601
	/* io_wq_work is initialized */
	REQ_F_WORK_INITIALIZED	= BIT(REQ_F_WORK_INITIALIZED_BIT),
P
Pavel Begunkov 已提交
602 603
	/* req->task is refcounted */
	REQ_F_TASK_PINNED	= BIT(REQ_F_TASK_PINNED_BIT),
604 605 606 607 608
};

struct async_poll {
	struct io_poll_iocb	poll;
	struct io_wq_work	work;
609 610
};

J
Jens Axboe 已提交
611 612 613 614 615 616
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
617
struct io_kiocb {
618
	union {
J
Jens Axboe 已提交
619
		struct file		*file;
620
		struct io_rw		rw;
621
		struct io_poll_iocb	poll;
622 623
		struct io_accept	accept;
		struct io_sync		sync;
624
		struct io_cancel	cancel;
625
		struct io_timeout	timeout;
626
		struct io_connect	connect;
627
		struct io_sr_msg	sr_msg;
628
		struct io_open		open;
629
		struct io_close		close;
630
		struct io_files_update	files_update;
J
Jens Axboe 已提交
631
		struct io_fadvise	fadvise;
J
Jens Axboe 已提交
632
		struct io_madvise	madvise;
633
		struct io_epoll		epoll;
P
Pavel Begunkov 已提交
634
		struct io_splice	splice;
635
		struct io_provide_buf	pbuf;
636
		struct io_statx		statx;
637
	};
J
Jens Axboe 已提交
638

639
	struct io_async_ctx		*io;
640
	int				cflags;
641
	u8				opcode;
642 643
	/* polled IO has completed */
	u8				iopoll_completed;
J
Jens Axboe 已提交
644

645 646
	u16				buf_index;

J
Jens Axboe 已提交
647
	struct io_ring_ctx	*ctx;
648
	struct list_head	list;
J
Jens Axboe 已提交
649
	unsigned int		flags;
J
Jens Axboe 已提交
650
	refcount_t		refs;
651 652
	struct task_struct	*task;
	unsigned long		fsize;
J
Jens Axboe 已提交
653
	u64			user_data;
J
Jens Axboe 已提交
654
	u32			result;
655
	u32			sequence;
J
Jens Axboe 已提交
656

657 658
	struct list_head	link_list;

659 660
	struct list_head	inflight_entry;

661 662
	struct percpu_ref	*fixed_file_refs;

663 664 665
	union {
		/*
		 * Only commands that never go async can use the below fields,
666 667 668
		 * obviously. Right now only IORING_OP_POLL_ADD uses them, and
		 * async armed poll handlers for regular commands. The latter
		 * restore the work, if needed.
669 670 671
		 */
		struct {
			struct callback_head	task_work;
672 673
			struct hlist_node	hash_node;
			struct async_poll	*apoll;
674 675 676
		};
		struct io_wq_work	work;
	};
J
Jens Axboe 已提交
677 678
};

J
Jens Axboe 已提交
679
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
680

681 682 683
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
684 685 686 687
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
688
	unsigned int		free_reqs;
J
Jens Axboe 已提交
689

690 691 692 693 694 695 696 697 698 699
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		used_refs;
	unsigned int		ios_left;
};

700 701 702 703 704 705 706
struct io_op_def {
	/* needs req->io allocated for deferral/async */
	unsigned		async_ctx : 1;
	/* needs current->mm setup, does mm access */
	unsigned		needs_mm : 1;
	/* needs req->file assigned */
	unsigned		needs_file : 1;
707 708
	/* don't fail if file grab fails */
	unsigned		needs_file_no_error : 1;
709 710 711 712
	/* hash wq insertion if file is a regular file */
	unsigned		hash_reg_file : 1;
	/* unbound wq insertion if file is a non-regular file */
	unsigned		unbound_nonreg_file : 1;
713 714
	/* opcode is not supported by this kernel */
	unsigned		not_supported : 1;
715 716
	/* needs file table */
	unsigned		file_table : 1;
717 718
	/* needs ->fs */
	unsigned		needs_fs : 1;
719 720 721
	/* set if opcode supports polled "wait" */
	unsigned		pollin : 1;
	unsigned		pollout : 1;
722 723
	/* op supports buffer selection */
	unsigned		buffer_select : 1;
724 725 726
};

static const struct io_op_def io_op_defs[] = {
727 728
	[IORING_OP_NOP] = {},
	[IORING_OP_READV] = {
729 730 731 732
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
733
		.pollin			= 1,
734
		.buffer_select		= 1,
735
	},
736
	[IORING_OP_WRITEV] = {
737 738 739 740 741
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
742
		.pollout		= 1,
743
	},
744
	[IORING_OP_FSYNC] = {
745 746
		.needs_file		= 1,
	},
747
	[IORING_OP_READ_FIXED] = {
748 749
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
750
		.pollin			= 1,
751
	},
752
	[IORING_OP_WRITE_FIXED] = {
753 754 755
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
756
		.pollout		= 1,
757
	},
758
	[IORING_OP_POLL_ADD] = {
759 760 761
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
762 763
	[IORING_OP_POLL_REMOVE] = {},
	[IORING_OP_SYNC_FILE_RANGE] = {
764 765
		.needs_file		= 1,
	},
766
	[IORING_OP_SENDMSG] = {
767 768 769 770
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
771
		.needs_fs		= 1,
772
		.pollout		= 1,
773
	},
774
	[IORING_OP_RECVMSG] = {
775 776 777 778
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
779
		.needs_fs		= 1,
780
		.pollin			= 1,
781
		.buffer_select		= 1,
782
	},
783
	[IORING_OP_TIMEOUT] = {
784 785 786
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
787 788
	[IORING_OP_TIMEOUT_REMOVE] = {},
	[IORING_OP_ACCEPT] = {
789 790 791
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
792
		.file_table		= 1,
793
		.pollin			= 1,
794
	},
795 796
	[IORING_OP_ASYNC_CANCEL] = {},
	[IORING_OP_LINK_TIMEOUT] = {
797 798 799
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
800
	[IORING_OP_CONNECT] = {
801 802 803 804
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
805
		.pollout		= 1,
806
	},
807
	[IORING_OP_FALLOCATE] = {
808 809
		.needs_file		= 1,
	},
810
	[IORING_OP_OPENAT] = {
811
		.file_table		= 1,
812
		.needs_fs		= 1,
813
	},
814
	[IORING_OP_CLOSE] = {
815 816
		.needs_file		= 1,
		.needs_file_no_error	= 1,
817
		.file_table		= 1,
818
	},
819
	[IORING_OP_FILES_UPDATE] = {
820
		.needs_mm		= 1,
821
		.file_table		= 1,
822
	},
823
	[IORING_OP_STATX] = {
824
		.needs_mm		= 1,
825
		.needs_fs		= 1,
826
		.file_table		= 1,
827
	},
828
	[IORING_OP_READ] = {
829 830 831
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
832
		.pollin			= 1,
833
		.buffer_select		= 1,
834
	},
835
	[IORING_OP_WRITE] = {
836 837 838
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
839
		.pollout		= 1,
840
	},
841
	[IORING_OP_FADVISE] = {
J
Jens Axboe 已提交
842 843
		.needs_file		= 1,
	},
844
	[IORING_OP_MADVISE] = {
J
Jens Axboe 已提交
845 846
		.needs_mm		= 1,
	},
847
	[IORING_OP_SEND] = {
848 849 850
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
851
		.pollout		= 1,
852
	},
853
	[IORING_OP_RECV] = {
854 855 856
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
857
		.pollin			= 1,
858
		.buffer_select		= 1,
859
	},
860
	[IORING_OP_OPENAT2] = {
861
		.file_table		= 1,
862
		.needs_fs		= 1,
863
	},
864 865 866 867
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.file_table		= 1,
	},
P
Pavel Begunkov 已提交
868 869 870 871
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
872 873
	},
	[IORING_OP_PROVIDE_BUFFERS] = {},
874
	[IORING_OP_REMOVE_BUFFERS] = {},
P
Pavel Begunkov 已提交
875 876 877 878 879
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
880 881
};

882 883 884 885 886
enum io_mem_account {
	ACCT_LOCKED,
	ACCT_PINNED,
};

887
static void io_wq_submit_work(struct io_wq_work **workptr);
888
static void io_cqring_fill_event(struct io_kiocb *req, long res);
889
static void io_put_req(struct io_kiocb *req);
890
static void __io_double_put_req(struct io_kiocb *req);
891 892
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
static void io_queue_linked_timeout(struct io_kiocb *req);
893 894 895
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *ip,
				 unsigned nr_args);
896
static int io_grab_files(struct io_kiocb *req);
P
Pavel Begunkov 已提交
897
static void io_cleanup_req(struct io_kiocb *req);
898 899 900 901
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
		       int fd, struct file **out_file, bool fixed);
static void __io_queue_sqe(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe);
902

903 904 905 906 907 908 909
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock);
static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
			     struct iovec *iovec, struct iovec *fast_iov,
			     struct iov_iter *iter);

J
Jens Axboe 已提交
910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

P
Pavel Begunkov 已提交
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
static void io_get_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		return;
	get_task_struct(req->task);
	req->flags |= REQ_F_TASK_PINNED;
}

/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
static void __io_put_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		put_task_struct(req->task);
}

942 943
static void io_file_put_work(struct work_struct *work);

944 945 946 947 948 949 950 951 952 953 954 955 956
/*
 * Note: must call io_req_init_async() for the first time you
 * touch any members of io_wq_work.
 */
static inline void io_req_init_async(struct io_kiocb *req)
{
	if (req->flags & REQ_F_WORK_INITIALIZED)
		return;

	memset(&req->work, 0, sizeof(req->work));
	req->flags |= REQ_F_WORK_INITIALIZED;
}

957 958 959 960 961
static inline bool io_async_submit(struct io_ring_ctx *ctx)
{
	return ctx->flags & IORING_SETUP_SQPOLL;
}

J
Jens Axboe 已提交
962 963 964 965
static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

966
	complete(&ctx->ref_comp);
J
Jens Axboe 已提交
967 968 969 970 971
}

static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;
972
	int hash_bits;
J
Jens Axboe 已提交
973 974 975 976 977

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

978 979 980 981
	ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
	if (!ctx->fallback_req)
		goto err;

982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
	/*
	 * Use 5 bits less than the max cq entries, that should give us around
	 * 32 entries per hash list if totally full and uniformly spread.
	 */
	hash_bits = ilog2(p->cq_entries);
	hash_bits -= 5;
	if (hash_bits <= 0)
		hash_bits = 1;
	ctx->cancel_hash_bits = hash_bits;
	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
					GFP_KERNEL);
	if (!ctx->cancel_hash)
		goto err;
	__hash_init(ctx->cancel_hash, 1U << hash_bits);

997
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
998 999
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
		goto err;
J
Jens Axboe 已提交
1000 1001

	ctx->flags = p->flags;
1002
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
1003
	init_waitqueue_head(&ctx->cq_wait);
1004
	INIT_LIST_HEAD(&ctx->cq_overflow_list);
1005 1006
	init_completion(&ctx->ref_comp);
	init_completion(&ctx->sq_thread_comp);
1007
	idr_init(&ctx->io_buffer_idr);
1008
	idr_init(&ctx->personality_idr);
J
Jens Axboe 已提交
1009 1010 1011
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
J
Jens Axboe 已提交
1012
	INIT_LIST_HEAD(&ctx->poll_list);
1013
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
1014
	INIT_LIST_HEAD(&ctx->timeout_list);
1015 1016 1017
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
1018 1019
	INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work);
	init_llist_head(&ctx->file_put_llist);
J
Jens Axboe 已提交
1020
	return ctx;
1021
err:
1022 1023
	if (ctx->fallback_req)
		kmem_cache_free(req_cachep, ctx->fallback_req);
1024
	kfree(ctx->cancel_hash);
1025 1026
	kfree(ctx);
	return NULL;
J
Jens Axboe 已提交
1027 1028
}

B
Bob Liu 已提交
1029
static inline bool __req_need_defer(struct io_kiocb *req)
1030
{
1031 1032
	struct io_ring_ctx *ctx = req->ctx;

1033 1034
	return req->sequence != ctx->cached_cq_tail
				+ atomic_read(&ctx->cached_cq_overflow);
1035 1036
}

B
Bob Liu 已提交
1037
static inline bool req_need_defer(struct io_kiocb *req)
1038
{
1039
	if (unlikely(req->flags & REQ_F_IO_DRAIN))
B
Bob Liu 已提交
1040
		return __req_need_defer(req);
1041

B
Bob Liu 已提交
1042
	return false;
1043 1044 1045
}

static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
1046
{
1047
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1048

1049 1050
	/* order cqe stores with ring update */
	smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
1051

1052 1053 1054
	if (wq_has_sleeper(&ctx->cq_wait)) {
		wake_up_interruptible(&ctx->cq_wait);
		kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
J
Jens Axboe 已提交
1055 1056 1057
	}
}

1058 1059 1060 1061 1062 1063
static inline void io_req_work_grab_env(struct io_kiocb *req,
					const struct io_op_def *def)
{
	if (!req->work.mm && def->needs_mm) {
		mmgrab(current->mm);
		req->work.mm = current->mm;
J
Jens Axboe 已提交
1064
	}
1065 1066
	if (!req->work.creds)
		req->work.creds = get_current_cred();
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
	if (!req->work.fs && def->needs_fs) {
		spin_lock(&current->fs->lock);
		if (!current->fs->in_exec) {
			req->work.fs = current->fs;
			req->work.fs->users++;
		} else {
			req->work.flags |= IO_WQ_WORK_CANCEL;
		}
		spin_unlock(&current->fs->lock);
	}
J
Jens Axboe 已提交
1077 1078
}

1079
static inline void io_req_work_drop_env(struct io_kiocb *req)
1080
{
1081 1082 1083
	if (!(req->flags & REQ_F_WORK_INITIALIZED))
		return;

1084 1085 1086 1087 1088 1089 1090 1091
	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
	if (req->work.fs) {
		struct fs_struct *fs = req->work.fs;

		spin_lock(&req->work.fs->lock);
		if (--fs->users)
			fs = NULL;
		spin_unlock(&req->work.fs->lock);
		if (fs)
			free_fs_struct(fs);
	}
1102 1103
}

1104
static inline void io_prep_async_work(struct io_kiocb *req,
1105
				      struct io_kiocb **link)
1106
{
1107
	const struct io_op_def *def = &io_op_defs[req->opcode];
1108

1109 1110
	if (req->flags & REQ_F_ISREG) {
		if (def->hash_reg_file)
1111
			io_wq_hash_work(&req->work, file_inode(req->file));
1112 1113
	} else {
		if (def->unbound_nonreg_file)
1114
			req->work.flags |= IO_WQ_WORK_UNBOUND;
1115
	}
1116

P
Pavel Begunkov 已提交
1117
	io_req_init_async(req);
1118
	io_req_work_grab_env(req, def);
1119

1120
	*link = io_prep_linked_timeout(req);
1121 1122
}

1123
static inline void io_queue_async_work(struct io_kiocb *req)
1124
{
1125
	struct io_ring_ctx *ctx = req->ctx;
1126 1127
	struct io_kiocb *link;

1128
	io_prep_async_work(req, &link);
1129

1130 1131 1132
	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
					&req->work, req->flags);
	io_wq_enqueue(ctx->io_wq, &req->work);
1133 1134 1135

	if (link)
		io_queue_linked_timeout(link);
1136 1137
}

J
Jens Axboe 已提交
1138 1139 1140 1141
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

1142
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
J
Jens Axboe 已提交
1143 1144
	if (ret != -1) {
		atomic_inc(&req->ctx->cq_timeouts);
1145
		list_del_init(&req->list);
1146
		req->flags |= REQ_F_COMP_LOCKED;
1147
		io_cqring_fill_event(req, 0);
1148
		io_put_req(req);
J
Jens Axboe 已提交
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

1162
static void __io_queue_deferred(struct io_ring_ctx *ctx)
1163
{
1164 1165 1166
	do {
		struct io_kiocb *req = list_first_entry(&ctx->defer_list,
							struct io_kiocb, list);
1167

1168 1169 1170 1171 1172 1173 1174
		if (req_need_defer(req))
			break;
		list_del_init(&req->list);
		io_queue_async_work(req);
	} while (!list_empty(&ctx->defer_list));
}

1175
static void io_flush_timeouts(struct io_ring_ctx *ctx)
1176
{
1177 1178 1179
	while (!list_empty(&ctx->timeout_list)) {
		struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
							struct io_kiocb, list);
1180

1181 1182
		if (req->flags & REQ_F_TIMEOUT_NOSEQ)
			break;
1183 1184
		if (req->timeout.target_seq != ctx->cached_cq_tail
					- atomic_read(&ctx->cq_timeouts))
1185
			break;
1186

1187
		list_del_init(&req->list);
J
Jens Axboe 已提交
1188
		io_kill_timeout(req);
1189 1190
	}
}
J
Jens Axboe 已提交
1191

1192 1193 1194
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	io_flush_timeouts(ctx);
1195 1196
	__io_commit_cqring(ctx);

1197 1198
	if (unlikely(!list_empty(&ctx->defer_list)))
		__io_queue_deferred(ctx);
1199 1200
}

J
Jens Axboe 已提交
1201 1202
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
1203
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1204 1205 1206
	unsigned tail;

	tail = ctx->cached_cq_tail;
1207 1208 1209 1210 1211
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
1212
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
1213 1214 1215
		return NULL;

	ctx->cached_cq_tail++;
1216
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
1217 1218
}

1219 1220
static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
{
1221 1222
	if (!ctx->cq_ev_fd)
		return false;
1223 1224
	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
		return false;
1225 1226
	if (!ctx->eventfd_async)
		return true;
1227
	return io_wq_current_is_worker();
1228 1229
}

1230
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1231 1232 1233 1234 1235
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
1236
	if (io_should_trigger_evfd(ctx))
1237 1238 1239
		eventfd_signal(ctx->cq_ev_fd, 1);
}

1240 1241
/* Returns true if there are no backlogged entries after the flush */
static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1242 1243 1244 1245 1246 1247 1248 1249 1250
{
	struct io_rings *rings = ctx->rings;
	struct io_uring_cqe *cqe;
	struct io_kiocb *req;
	unsigned long flags;
	LIST_HEAD(list);

	if (!force) {
		if (list_empty_careful(&ctx->cq_overflow_list))
1251
			return true;
1252 1253
		if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
		    rings->cq_ring_entries))
1254
			return false;
1255 1256 1257 1258 1259 1260
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/* if force is set, the ring is going away. always drop after that */
	if (force)
1261
		ctx->cq_overflow_flushed = 1;
1262

1263
	cqe = NULL;
1264 1265 1266 1267 1268 1269 1270 1271
	while (!list_empty(&ctx->cq_overflow_list)) {
		cqe = io_get_cqring(ctx);
		if (!cqe && !force)
			break;

		req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
						list);
		list_move(&req->list, &list);
1272
		req->flags &= ~REQ_F_OVERFLOW;
1273 1274 1275
		if (cqe) {
			WRITE_ONCE(cqe->user_data, req->user_data);
			WRITE_ONCE(cqe->res, req->result);
1276
			WRITE_ONCE(cqe->flags, req->cflags);
1277 1278 1279 1280 1281 1282 1283
		} else {
			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
		}
	}

	io_commit_cqring(ctx);
1284 1285 1286 1287
	if (cqe) {
		clear_bit(0, &ctx->sq_check_overflow);
		clear_bit(0, &ctx->cq_check_overflow);
	}
1288 1289 1290 1291 1292 1293
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	while (!list_empty(&list)) {
		req = list_first_entry(&list, struct io_kiocb, list);
		list_del(&req->list);
1294
		io_put_req(req);
1295
	}
1296 1297

	return cqe != NULL;
1298 1299
}

1300
static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1301
{
1302
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1303 1304
	struct io_uring_cqe *cqe;

1305
	trace_io_uring_complete(ctx, req->user_data, res);
J
Jens Axboe 已提交
1306

J
Jens Axboe 已提交
1307 1308 1309 1310 1311 1312
	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
1313
	if (likely(cqe)) {
1314
		WRITE_ONCE(cqe->user_data, req->user_data);
J
Jens Axboe 已提交
1315
		WRITE_ONCE(cqe->res, res);
1316
		WRITE_ONCE(cqe->flags, cflags);
1317
	} else if (ctx->cq_overflow_flushed) {
1318 1319
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
1320
	} else {
1321 1322 1323 1324
		if (list_empty(&ctx->cq_overflow_list)) {
			set_bit(0, &ctx->sq_check_overflow);
			set_bit(0, &ctx->cq_check_overflow);
		}
1325
		req->flags |= REQ_F_OVERFLOW;
1326 1327
		refcount_inc(&req->refs);
		req->result = res;
1328
		req->cflags = cflags;
1329
		list_add_tail(&req->list, &ctx->cq_overflow_list);
J
Jens Axboe 已提交
1330 1331 1332
	}
}

1333 1334 1335 1336 1337 1338
static void io_cqring_fill_event(struct io_kiocb *req, long res)
{
	__io_cqring_fill_event(req, res, 0);
}

static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1339
{
1340
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1341 1342 1343
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
1344
	__io_cqring_fill_event(req, res, cflags);
J
Jens Axboe 已提交
1345 1346 1347
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
1348
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1349 1350
}

1351 1352 1353 1354 1355
static void io_cqring_add_event(struct io_kiocb *req, long res)
{
	__io_cqring_add_event(req, res, 0);
}

1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
static inline bool io_is_fallback_req(struct io_kiocb *req)
{
	return req == (struct io_kiocb *)
			((unsigned long) req->ctx->fallback_req & ~1UL);
}

static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	req = ctx->fallback_req;
1367
	if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req))
1368 1369 1370 1371 1372
		return req;

	return NULL;
}

1373 1374
static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
				     struct io_submit_state *state)
J
Jens Axboe 已提交
1375
{
1376
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
1377 1378
	struct io_kiocb *req;

1379
	if (!state->free_reqs) {
J
Jens Axboe 已提交
1380 1381 1382 1383
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
1384 1385 1386 1387 1388 1389 1390 1391 1392
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
1393
				goto fallback;
1394 1395
			ret = 1;
		}
J
Jens Axboe 已提交
1396
		state->free_reqs = ret - 1;
1397
		req = state->reqs[ret - 1];
J
Jens Axboe 已提交
1398 1399
	} else {
		state->free_reqs--;
1400
		req = state->reqs[state->free_reqs];
J
Jens Axboe 已提交
1401 1402
	}

J
Jens Axboe 已提交
1403
	return req;
1404
fallback:
1405
	return io_get_fallback_req(ctx);
J
Jens Axboe 已提交
1406 1407
}

1408 1409 1410 1411
static inline void io_put_file(struct io_kiocb *req, struct file *file,
			  bool fixed)
{
	if (fixed)
1412
		percpu_ref_put(req->fixed_file_refs);
1413 1414 1415 1416
	else
		fput(file);
}

1417
static void __io_req_aux_free(struct io_kiocb *req)
J
Jens Axboe 已提交
1418
{
1419 1420 1421
	if (req->flags & REQ_F_NEED_CLEANUP)
		io_cleanup_req(req);

1422
	kfree(req->io);
1423 1424
	if (req->file)
		io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
P
Pavel Begunkov 已提交
1425
	__io_put_req_task(req);
1426
	io_req_work_drop_env(req);
J
Jens Axboe 已提交
1427 1428
}

J
Jens Axboe 已提交
1429
static void __io_free_req(struct io_kiocb *req)
J
Jens Axboe 已提交
1430
{
1431
	__io_req_aux_free(req);
1432 1433

	if (req->flags & REQ_F_INFLIGHT) {
1434
		struct io_ring_ctx *ctx = req->ctx;
1435 1436 1437 1438 1439 1440 1441 1442
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
	}
1443 1444

	percpu_ref_put(&req->ctx->refs);
1445 1446 1447
	if (likely(!io_is_fallback_req(req)))
		kmem_cache_free(req_cachep, req);
	else
1448
		clear_bit_unlock(0, (unsigned long *) &req->ctx->fallback_req);
1449 1450
}

1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478
struct req_batch {
	void *reqs[IO_IOPOLL_BATCH];
	int to_free;
	int need_iter;
};

static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
{
	if (!rb->to_free)
		return;
	if (rb->need_iter) {
		int i, inflight = 0;
		unsigned long flags;

		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

			if (req->flags & REQ_F_INFLIGHT)
				inflight++;
			__io_req_aux_free(req);
		}
		if (!inflight)
			goto do_free;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

1479
			if (req->flags & REQ_F_INFLIGHT) {
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
				list_del(&req->inflight_entry);
				if (!--inflight)
					break;
			}
		}
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);

		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
	}
do_free:
	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
	percpu_ref_put_many(&ctx->refs, rb->to_free);
	rb->to_free = rb->need_iter = 0;
1494 1495
}

1496
static bool io_link_cancel_timeout(struct io_kiocb *req)
1497
{
1498
	struct io_ring_ctx *ctx = req->ctx;
1499 1500
	int ret;

1501
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1502
	if (ret != -1) {
1503
		io_cqring_fill_event(req, -ECANCELED);
1504
		io_commit_cqring(ctx);
1505
		req->flags &= ~REQ_F_LINK_HEAD;
1506
		io_put_req(req);
1507 1508 1509 1510
		return true;
	}

	return false;
1511 1512
}

1513
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
J
Jens Axboe 已提交
1514
{
1515 1516
	struct io_ring_ctx *ctx = req->ctx;
	bool wake_ev = false;
J
Jens Axboe 已提交
1517

1518 1519 1520 1521
	/* Already got next link */
	if (req->flags & REQ_F_LINK_NEXT)
		return;

J
Jens Axboe 已提交
1522 1523 1524 1525 1526
	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
1527 1528 1529
	while (!list_empty(&req->link_list)) {
		struct io_kiocb *nxt = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
1530

1531 1532 1533
		if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
			     (nxt->flags & REQ_F_TIMEOUT))) {
			list_del_init(&nxt->link_list);
1534 1535 1536 1537
			wake_ev |= io_link_cancel_timeout(nxt);
			req->flags &= ~REQ_F_LINK_TIMEOUT;
			continue;
		}
J
Jens Axboe 已提交
1538

1539 1540
		list_del_init(&req->link_list);
		if (!list_empty(&nxt->link_list))
1541
			nxt->flags |= REQ_F_LINK_HEAD;
1542
		*nxtptr = nxt;
1543
		break;
J
Jens Axboe 已提交
1544
	}
1545

1546
	req->flags |= REQ_F_LINK_NEXT;
1547 1548
	if (wake_ev)
		io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1549 1550 1551
}

/*
1552
 * Called if REQ_F_LINK_HEAD is set, and we fail the head request
J
Jens Axboe 已提交
1553 1554 1555
 */
static void io_fail_links(struct io_kiocb *req)
{
1556 1557 1558 1559
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
J
Jens Axboe 已提交
1560 1561

	while (!list_empty(&req->link_list)) {
1562 1563
		struct io_kiocb *link = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
J
Jens Axboe 已提交
1564

1565
		list_del_init(&link->link_list);
1566
		trace_io_uring_fail_link(req, link);
1567 1568

		if ((req->flags & REQ_F_LINK_TIMEOUT) &&
1569
		    link->opcode == IORING_OP_LINK_TIMEOUT) {
1570
			io_link_cancel_timeout(link);
1571
		} else {
1572
			io_cqring_fill_event(link, -ECANCELED);
1573
			__io_double_put_req(link);
1574
		}
1575
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
1576
	}
1577 1578 1579 1580

	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1581 1582
}

1583
static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
J
Jens Axboe 已提交
1584
{
1585
	if (likely(!(req->flags & REQ_F_LINK_HEAD)))
1586 1587
		return;

J
Jens Axboe 已提交
1588 1589 1590 1591 1592 1593
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
1594 1595
	if (req->flags & REQ_F_FAIL_LINK) {
		io_fail_links(req);
1596 1597
	} else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
			REQ_F_LINK_TIMEOUT) {
1598 1599 1600 1601 1602 1603
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;

		/*
		 * If this is a timeout link, we could be racing with the
		 * timeout timer. Grab the completion lock for this case to
1604
		 * protect against that.
1605 1606 1607 1608 1609 1610
		 */
		spin_lock_irqsave(&ctx->completion_lock, flags);
		io_req_link_next(req, nxt);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
		io_req_link_next(req, nxt);
J
Jens Axboe 已提交
1611
	}
1612
}
J
Jens Axboe 已提交
1613

1614 1615
static void io_free_req(struct io_kiocb *req)
{
1616 1617 1618
	struct io_kiocb *nxt = NULL;

	io_req_find_next(req, &nxt);
1619
	__io_free_req(req);
1620 1621 1622

	if (nxt)
		io_queue_async_work(nxt);
1623 1624
}

1625 1626 1627
static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
{
	struct io_kiocb *link;
1628 1629 1630 1631
	const struct io_op_def *def = &io_op_defs[nxt->opcode];

	if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
		io_wq_hash_work(&nxt->work, file_inode(nxt->file));
1632 1633 1634

	*workptr = &nxt->work;
	link = io_prep_linked_timeout(nxt);
1635
	if (link)
1636
		nxt->flags |= REQ_F_QUEUE_TIMEOUT;
1637 1638
}

1639 1640 1641 1642
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
1643
__attribute__((nonnull))
1644
static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
1645
{
1646 1647
	if (refcount_dec_and_test(&req->refs)) {
		io_req_find_next(req, nxtptr);
1648
		__io_free_req(req);
1649
	}
J
Jens Axboe 已提交
1650 1651
}

1652 1653 1654 1655
static void io_put_req(struct io_kiocb *req)
{
	if (refcount_dec_and_test(&req->refs))
		io_free_req(req);
J
Jens Axboe 已提交
1656 1657
}

1658 1659
static void io_steal_work(struct io_kiocb *req,
			  struct io_wq_work **workptr)
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
{
	/*
	 * It's in an io-wq worker, so there always should be at least
	 * one reference, which will be dropped in io_put_work() just
	 * after the current handler returns.
	 *
	 * It also means, that if the counter dropped to 1, then there is
	 * no asynchronous users left, so it's safe to steal the next work.
	 */
	if (refcount_read(&req->refs) == 1) {
		struct io_kiocb *nxt = NULL;

		io_req_find_next(req, &nxt);
		if (nxt)
			io_wq_assign_next(workptr, nxt);
	}
}

1678 1679 1680 1681 1682
/*
 * Must only be used if we don't need to care about links, usually from
 * within the completion handling itself.
 */
static void __io_double_put_req(struct io_kiocb *req)
1683 1684 1685 1686 1687 1688
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		__io_free_req(req);
}

1689 1690 1691 1692 1693 1694 1695
static void io_double_put_req(struct io_kiocb *req)
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		io_free_req(req);
}

1696
static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
1697
{
1698 1699
	struct io_rings *rings = ctx->rings;

1700 1701 1702 1703 1704 1705 1706 1707
	if (test_bit(0, &ctx->cq_check_overflow)) {
		/*
		 * noflush == true is from the waitqueue handler, just ensure
		 * we wake up the task, and the next invocation will flush the
		 * entries. We cannot safely to it from here.
		 */
		if (noflush && !list_empty(&ctx->cq_overflow_list))
			return -1U;
1708

1709 1710
		io_cqring_overflow_flush(ctx, false);
	}
1711

1712 1713
	/* See comment at the top of this file */
	smp_rmb();
1714
	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
1715 1716
}

1717 1718 1719 1720 1721 1722 1723 1724
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

1725
static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
1726
{
1727
	if ((req->flags & REQ_F_LINK_HEAD) || io_is_fallback_req(req))
1728
		return false;
1729

1730
	if (req->file || req->io)
1731 1732 1733 1734 1735 1736
		rb->need_iter++;

	rb->reqs[rb->to_free++] = req;
	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		io_free_req_many(req->ctx, rb);
	return true;
1737 1738
}

1739 1740
static int io_put_kbuf(struct io_kiocb *req)
{
1741
	struct io_buffer *kbuf;
1742 1743
	int cflags;

1744
	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
1745 1746 1747 1748 1749 1750 1751
	cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	cflags |= IORING_CQE_F_BUFFER;
	req->rw.addr = 0;
	kfree(kbuf);
	return cflags;
}

1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
static void io_iopoll_queue(struct list_head *again)
{
	struct io_kiocb *req;

	do {
		req = list_first_entry(again, struct io_kiocb, list);
		list_del(&req->list);
		refcount_inc(&req->refs);
		io_queue_async_work(req);
	} while (!list_empty(again));
}

J
Jens Axboe 已提交
1764 1765 1766 1767 1768 1769
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
1770
	struct req_batch rb;
J
Jens Axboe 已提交
1771
	struct io_kiocb *req;
1772 1773 1774 1775
	LIST_HEAD(again);

	/* order with ->result store in io_complete_rw_iopoll() */
	smp_rmb();
J
Jens Axboe 已提交
1776

1777
	rb.to_free = rb.need_iter = 0;
J
Jens Axboe 已提交
1778
	while (!list_empty(done)) {
1779 1780
		int cflags = 0;

J
Jens Axboe 已提交
1781
		req = list_first_entry(done, struct io_kiocb, list);
1782 1783 1784 1785 1786
		if (READ_ONCE(req->result) == -EAGAIN) {
			req->iopoll_completed = 0;
			list_move_tail(&req->list, &again);
			continue;
		}
J
Jens Axboe 已提交
1787 1788
		list_del(&req->list);

1789 1790 1791 1792
		if (req->flags & REQ_F_BUFFER_SELECTED)
			cflags = io_put_kbuf(req);

		__io_cqring_fill_event(req, req->result, cflags);
J
Jens Axboe 已提交
1793 1794
		(*nr_events)++;

1795 1796 1797
		if (refcount_dec_and_test(&req->refs) &&
		    !io_req_multi_free(&rb, req))
			io_free_req(req);
J
Jens Axboe 已提交
1798 1799
	}

J
Jens Axboe 已提交
1800
	io_commit_cqring(ctx);
1801 1802
	if (ctx->flags & IORING_SETUP_SQPOLL)
		io_cqring_ev_posted(ctx);
1803
	io_free_req_many(ctx, &rb);
1804

1805 1806
	if (!list_empty(&again))
		io_iopoll_queue(&again);
1807 1808
}

J
Jens Axboe 已提交
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824
static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
1825
		struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
1826 1827

		/*
1828 1829 1830
		 * Move completed and retryable entries to our local lists.
		 * If we find a request that requires polling, break out
		 * and complete those lists first, if we have entries there.
J
Jens Axboe 已提交
1831
		 */
1832
		if (READ_ONCE(req->iopoll_completed)) {
J
Jens Axboe 已提交
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
			list_move_tail(&req->list, &done);
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
1855
 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
J
Jens Axboe 已提交
1856 1857 1858 1859 1860 1861
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
1862
	while (!list_empty(&ctx->poll_list) && !need_resched()) {
J
Jens Axboe 已提交
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
	while (!list_empty(&ctx->poll_list)) {
		unsigned int nr_events = 0;

		io_iopoll_getevents(ctx, &nr_events, 1);
1889 1890 1891 1892 1893 1894

		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
		 */
		cond_resched();
J
Jens Axboe 已提交
1895 1896 1897 1898
	}
	mutex_unlock(&ctx->uring_lock);
}

1899 1900
static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			   long min)
J
Jens Axboe 已提交
1901
{
1902
	int iters = 0, ret = 0;
1903

1904 1905 1906 1907 1908 1909
	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
J
Jens Axboe 已提交
1910 1911 1912
	do {
		int tmin = 0;

1913 1914 1915 1916 1917
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
1918
		if (io_cqring_events(ctx, false))
1919 1920
			break;

1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
			mutex_lock(&ctx->uring_lock);
		}

J
Jens Axboe 已提交
1936 1937 1938 1939 1940 1941 1942 1943 1944
		if (*nr_events < min)
			tmin = min - *nr_events;

		ret = io_iopoll_getevents(ctx, nr_events, tmin);
		if (ret <= 0)
			break;
		ret = 0;
	} while (min && !*nr_events && !need_resched());

1945
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
1946 1947 1948
	return ret;
}

1949
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
1950
{
1951 1952 1953 1954 1955 1956
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
1957

1958
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
1959
	}
1960
	file_end_write(req->file);
J
Jens Axboe 已提交
1961 1962
}

J
Jens Axboe 已提交
1963 1964 1965 1966 1967 1968
static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}

1969
static void io_complete_rw_common(struct kiocb *kiocb, long res)
J
Jens Axboe 已提交
1970
{
1971
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1972
	int cflags = 0;
J
Jens Axboe 已提交
1973

1974 1975
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1976

J
Jens Axboe 已提交
1977 1978
	if (res != req->result)
		req_set_fail_links(req);
1979 1980 1981
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_kbuf(req);
	__io_cqring_add_event(req, res, cflags);
1982 1983
}

1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
{
	struct mm_struct *mm = current->mm;

	if (mm) {
		kthread_unuse_mm(mm);
		mmput(mm);
	}
}

static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	if (io_op_defs[req->opcode].needs_mm && !current->mm) {
		if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
			return -EFAULT;
		kthread_use_mm(ctx->sqo_mm);
	}

	return 0;
}

#ifdef CONFIG_BLOCK
static bool io_resubmit_prep(struct io_kiocb *req, int error)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	ssize_t ret = -ECANCELED;
	struct iov_iter iter;
	int rw;

	if (error) {
		ret = error;
		goto end_req;
	}

	switch (req->opcode) {
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
		rw = READ;
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		rw = WRITE;
		break;
	default:
		printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
				req->opcode);
		goto end_req;
	}

	ret = io_import_iovec(rw, req, &iovec, &iter, false);
	if (ret < 0)
		goto end_req;
	ret = io_setup_async_rw(req, ret, iovec, inline_vecs, &iter);
	if (!ret)
		return true;
	kfree(iovec);
end_req:
	io_cqring_add_event(req, ret);
	req_set_fail_links(req);
	io_put_req(req);
	return false;
}

static void io_rw_resubmit(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct io_ring_ctx *ctx = req->ctx;
	int err;

	__set_current_state(TASK_RUNNING);

	err = io_sq_thread_acquire_mm(ctx, req);

	if (io_resubmit_prep(req, err)) {
		refcount_inc(&req->refs);
		io_queue_async_work(req);
	}
}
#endif

static bool io_rw_reissue(struct io_kiocb *req, long res)
{
#ifdef CONFIG_BLOCK
	struct task_struct *tsk;
	int ret;

	if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker())
		return false;

	tsk = req->task;
	init_task_work(&req->task_work, io_rw_resubmit);
	ret = task_work_add(tsk, &req->task_work, true);
	if (!ret)
		return true;
#endif
	return false;
}

2085 2086
static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
2087
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2088

2089 2090 2091 2092
	if (!io_rw_reissue(req, res)) {
		io_complete_rw_common(kiocb, res);
		io_put_req(req);
	}
J
Jens Axboe 已提交
2093 2094
}

J
Jens Axboe 已提交
2095 2096
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
2097
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
2098

2099 2100
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
2101

2102
	if (res != -EAGAIN && res != req->result)
J
Jens Axboe 已提交
2103
		req_set_fail_links(req);
2104 2105 2106 2107 2108

	WRITE_ONCE(req->result, res);
	/* order with io_poll_complete() checking ->result */
	if (res != -EAGAIN) {
		smp_wmb();
2109
		WRITE_ONCE(req->iopoll_completed, 1);
2110
	}
J
Jens Axboe 已提交
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->poll_list)) {
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

		list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
						list);
2135
		if (list_req->file != req->file)
J
Jens Axboe 已提交
2136 2137 2138 2139 2140 2141 2142
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
2143
	if (READ_ONCE(req->iopoll_completed))
J
Jens Axboe 已提交
2144 2145 2146
		list_add(&req->list, &ctx->poll_list);
	else
		list_add_tail(&req->list, &ctx->poll_list);
2147 2148 2149 2150

	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
	    wq_has_sleeper(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
J
Jens Axboe 已提交
2151 2152
}

P
Pavel Begunkov 已提交
2153
static void __io_state_file_put(struct io_submit_state *state)
2154
{
P
Pavel Begunkov 已提交
2155
	int diff = state->has_refs - state->used_refs;
2156

P
Pavel Begunkov 已提交
2157 2158 2159 2160 2161 2162 2163 2164 2165
	if (diff)
		fput_many(state->file, diff);
	state->file = NULL;
}

static inline void io_state_file_put(struct io_submit_state *state)
{
	if (state->file)
		__io_state_file_put(state);
2166 2167 2168 2169 2170 2171 2172
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
2173
static struct file *__io_file_get(struct io_submit_state *state, int fd)
2174 2175 2176 2177 2178 2179 2180 2181 2182 2183
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
			state->used_refs++;
			state->ios_left--;
			return state->file;
		}
P
Pavel Begunkov 已提交
2184
		__io_state_file_put(state);
2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->has_refs = state->ios_left;
	state->used_refs = 1;
	state->ios_left--;
	return state->file;
}

2197 2198 2199 2200 2201 2202 2203 2204 2205
static bool io_bdev_nowait(struct block_device *bdev)
{
#ifdef CONFIG_BLOCK
	return !bdev || queue_is_mq(bdev_get_queue(bdev));
#else
	return true;
#endif
}

J
Jens Axboe 已提交
2206 2207 2208 2209 2210
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
2211
static bool io_file_supports_async(struct file *file, int rw)
J
Jens Axboe 已提交
2212 2213 2214
{
	umode_t mode = file_inode(file)->i_mode;

2215 2216 2217 2218 2219 2220
	if (S_ISBLK(mode)) {
		if (io_bdev_nowait(file->f_inode->i_bdev))
			return true;
		return false;
	}
	if (S_ISCHR(mode) || S_ISSOCK(mode))
J
Jens Axboe 已提交
2221
		return true;
2222 2223 2224 2225 2226 2227
	if (S_ISREG(mode)) {
		if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
		    file->f_op != &io_uring_fops)
			return true;
		return false;
	}
J
Jens Axboe 已提交
2228

2229 2230 2231 2232
	/* any ->read/write should understand O_NONBLOCK */
	if (file->f_flags & O_NONBLOCK)
		return true;

2233 2234 2235 2236 2237 2238 2239
	if (!(file->f_mode & FMODE_NOWAIT))
		return false;

	if (rw == READ)
		return file->f_op->read_iter != NULL;

	return file->f_op->write_iter != NULL;
J
Jens Axboe 已提交
2240 2241
}

2242 2243
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		      bool force_nonblock)
J
Jens Axboe 已提交
2244
{
J
Jens Axboe 已提交
2245
	struct io_ring_ctx *ctx = req->ctx;
2246
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2247 2248
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
2249

2250 2251 2252
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

J
Jens Axboe 已提交
2253
	kiocb->ki_pos = READ_ONCE(sqe->off);
2254 2255 2256 2257
	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
		req->flags |= REQ_F_CUR_POS;
		kiocb->ki_pos = req->file->f_pos;
	}
J
Jens Axboe 已提交
2258
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
2259 2260 2261 2262
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
		return ret;
J
Jens Axboe 已提交
2263 2264 2265 2266 2267

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
2268
			return ret;
J
Jens Axboe 已提交
2269 2270 2271 2272 2273

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

2274
	/* don't allow async punt if RWF_NOWAIT was requested */
2275
	if (kiocb->ki_flags & IOCB_NOWAIT)
2276 2277
		req->flags |= REQ_F_NOWAIT;

2278 2279 2280
	if (kiocb->ki_flags & IOCB_DIRECT)
		io_get_req_task(req);

2281
	if (force_nonblock)
J
Jens Axboe 已提交
2282
		kiocb->ki_flags |= IOCB_NOWAIT;
2283

J
Jens Axboe 已提交
2284 2285 2286
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
2287
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
2288

J
Jens Axboe 已提交
2289 2290
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
2291
		req->result = 0;
2292
		req->iopoll_completed = 0;
J
Jens Axboe 已提交
2293
	} else {
J
Jens Axboe 已提交
2294 2295
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
2296 2297
		kiocb->ki_complete = io_complete_rw;
	}
2298

2299 2300
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
2301
	req->buf_index = READ_ONCE(sqe->buf_index);
J
Jens Axboe 已提交
2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

2326
static void kiocb_done(struct kiocb *kiocb, ssize_t ret)
2327
{
2328 2329 2330 2331
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

	if (req->flags & REQ_F_CUR_POS)
		req->file->f_pos = kiocb->ki_pos;
P
Pavel Begunkov 已提交
2332
	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
2333
		io_complete_rw(kiocb, ret, 0);
2334 2335 2336 2337
	else
		io_rw_done(kiocb, ret);
}

2338
static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
2339
			       struct iov_iter *iter)
2340
{
2341 2342
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
2343
	struct io_mapped_ubuf *imu;
2344
	u16 index, buf_index;
2345 2346 2347 2348 2349 2350 2351
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

2352
	buf_index = req->buf_index;
2353 2354 2355 2356 2357
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
2358
	buf_addr = req->rw.addr;
2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
2404
			iter->count -= bvec->bv_len + offset;
2405 2406 2407 2408
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

2409
	return len;
2410 2411
}

2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463
static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
{
	if (needs_lock)
		mutex_unlock(&ctx->uring_lock);
}

static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
{
	/*
	 * "Normal" inline submissions always hold the uring_lock, since we
	 * grab it from the system call. Same is true for the SQPOLL offload.
	 * The only exception is when we've detached the request and issue it
	 * from an async worker thread, grab the lock for that case.
	 */
	if (needs_lock)
		mutex_lock(&ctx->uring_lock);
}

static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
					  int bgid, struct io_buffer *kbuf,
					  bool needs_lock)
{
	struct io_buffer *head;

	if (req->flags & REQ_F_BUFFER_SELECTED)
		return kbuf;

	io_ring_submit_lock(req->ctx, needs_lock);

	lockdep_assert_held(&req->ctx->uring_lock);

	head = idr_find(&req->ctx->io_buffer_idr, bgid);
	if (head) {
		if (!list_empty(&head->list)) {
			kbuf = list_last_entry(&head->list, struct io_buffer,
							list);
			list_del(&kbuf->list);
		} else {
			kbuf = head;
			idr_remove(&req->ctx->io_buffer_idr, bgid);
		}
		if (*len > kbuf->len)
			*len = kbuf->len;
	} else {
		kbuf = ERR_PTR(-ENOBUFS);
	}

	io_ring_submit_unlock(req->ctx, needs_lock);

	return kbuf;
}

2464 2465 2466 2467
static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
					bool needs_lock)
{
	struct io_buffer *kbuf;
2468
	u16 bgid;
2469 2470

	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2471
	bgid = req->buf_index;
2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
	kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;
	req->rw.addr = (u64) (unsigned long) kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;
	return u64_to_user_ptr(kbuf->addr);
}

#ifdef CONFIG_COMPAT
static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
				bool needs_lock)
{
	struct compat_iovec __user *uiov;
	compat_ssize_t clen;
	void __user *buf;
	ssize_t len;

	uiov = u64_to_user_ptr(req->rw.addr);
	if (!access_ok(uiov, sizeof(*uiov)))
		return -EFAULT;
	if (__get_user(clen, &uiov->iov_len))
		return -EFAULT;
	if (clen < 0)
		return -EINVAL;

	len = clen;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = (compat_size_t) len;
	return 0;
}
#endif

static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				      bool needs_lock)
{
	struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
	void __user *buf;
	ssize_t len;

	if (copy_from_user(iov, uiov, sizeof(*uiov)))
		return -EFAULT;

	len = iov[0].iov_len;
	if (len < 0)
		return -EINVAL;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = len;
	return 0;
}

static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				    bool needs_lock)
{
2531 2532 2533 2534 2535 2536
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		struct io_buffer *kbuf;

		kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
		iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		iov[0].iov_len = kbuf->len;
2537
		return 0;
2538
	}
2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551
	if (!req->rw.len)
		return 0;
	else if (req->rw.len > 1)
		return -EINVAL;

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		return io_compat_import(req, iov, needs_lock);
#endif

	return __io_iov_buffer_select(req, iov, needs_lock);
}

2552
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2553 2554
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock)
J
Jens Axboe 已提交
2555
{
2556 2557
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
2558
	ssize_t ret;
2559 2560
	u8 opcode;

2561
	opcode = req->opcode;
2562
	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2563
		*iovec = NULL;
2564
		return io_import_fixed(req, rw, iter);
2565
	}
J
Jens Axboe 已提交
2566

2567
	/* buffer index only valid with fixed read/write, or buffer select  */
2568
	if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
2569 2570
		return -EINVAL;

2571
	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2572
		if (req->flags & REQ_F_BUFFER_SELECT) {
2573 2574
			buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
			if (IS_ERR(buf)) {
2575
				*iovec = NULL;
2576
				return PTR_ERR(buf);
2577
			}
2578
			req->rw.len = sqe_len;
2579 2580
		}

2581 2582
		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
2583
		return ret < 0 ? ret : sqe_len;
2584 2585
	}

2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
	if (req->io) {
		struct io_async_rw *iorw = &req->io->rw;

		*iovec = iorw->iov;
		iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
		if (iorw->iov == iorw->fast_iov)
			*iovec = NULL;
		return iorw->size;
	}

2596 2597
	if (req->flags & REQ_F_BUFFER_SELECT) {
		ret = io_iov_buffer_select(req, *iovec, needs_lock);
2598 2599 2600 2601
		if (!ret) {
			ret = (*iovec)->iov_len;
			iov_iter_init(iter, rw, *iovec, 1, ret);
		}
2602 2603 2604 2605
		*iovec = NULL;
		return ret;
	}

J
Jens Axboe 已提交
2606
#ifdef CONFIG_COMPAT
2607
	if (req->ctx->compat)
J
Jens Axboe 已提交
2608 2609 2610 2611 2612 2613 2614
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

2615
/*
2616 2617
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
2618
 */
2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
2635
		struct iovec iovec;
2636 2637
		ssize_t nr;

2638 2639 2640 2641 2642 2643 2644 2645 2646 2647
		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

2648 2649 2650 2651 2652 2653 2654 2655
		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, &kiocb->ki_pos);
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, &kiocb->ki_pos);
		}

2656 2657 2658
		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

2673
static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
2674 2675 2676 2677 2678 2679 2680 2681
			  struct iovec *iovec, struct iovec *fast_iov,
			  struct iov_iter *iter)
{
	req->io->rw.nr_segs = iter->nr_segs;
	req->io->rw.size = io_size;
	req->io->rw.iov = iovec;
	if (!req->io->rw.iov) {
		req->io->rw.iov = req->io->rw.fast_iov;
2682 2683 2684
		if (req->io->rw.iov != fast_iov)
			memcpy(req->io->rw.iov, fast_iov,
			       sizeof(struct iovec) * iter->nr_segs);
P
Pavel Begunkov 已提交
2685 2686
	} else {
		req->flags |= REQ_F_NEED_CLEANUP;
2687 2688 2689
	}
}

2690 2691 2692 2693 2694 2695
static inline int __io_alloc_async_ctx(struct io_kiocb *req)
{
	req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
	return req->io == NULL;
}

2696
static int io_alloc_async_ctx(struct io_kiocb *req)
2697
{
2698 2699
	if (!io_op_defs[req->opcode].async_ctx)
		return 0;
2700 2701

	return  __io_alloc_async_ctx(req);
2702 2703 2704 2705 2706 2707
}

static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
			     struct iovec *iovec, struct iovec *fast_iov,
			     struct iov_iter *iter)
{
2708
	if (!io_op_defs[req->opcode].async_ctx)
2709
		return 0;
2710
	if (!req->io) {
2711
		if (__io_alloc_async_ctx(req))
2712
			return -ENOMEM;
2713

2714 2715
		io_req_map_rw(req, io_size, iovec, fast_iov, iter);
	}
2716
	return 0;
2717 2718
}

2719 2720
static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
2721
{
2722 2723
	struct io_async_ctx *io;
	struct iov_iter iter;
2724 2725
	ssize_t ret;

2726 2727 2728
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2729

2730 2731
	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;
2732

2733 2734
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
2735 2736 2737 2738 2739
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
2740
	ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
2741 2742 2743 2744 2745 2746
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2747 2748
}

2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868
static void __io_async_buf_error(struct io_kiocb *req, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	io_cqring_fill_event(req, error);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	req_set_fail_links(req);
	io_double_put_req(req);
}

static void io_async_buf_cancel(struct callback_head *cb)
{
	struct io_async_rw *rw;
	struct io_kiocb *req;

	rw = container_of(cb, struct io_async_rw, task_work);
	req = rw->wpq.wait.private;
	__io_async_buf_error(req, -ECANCELED);
}

static void io_async_buf_retry(struct callback_head *cb)
{
	struct io_async_rw *rw;
	struct io_ring_ctx *ctx;
	struct io_kiocb *req;

	rw = container_of(cb, struct io_async_rw, task_work);
	req = rw->wpq.wait.private;
	ctx = req->ctx;

	__set_current_state(TASK_RUNNING);
	if (!io_sq_thread_acquire_mm(ctx, req)) {
		mutex_lock(&ctx->uring_lock);
		__io_queue_sqe(req, NULL);
		mutex_unlock(&ctx->uring_lock);
	} else {
		__io_async_buf_error(req, -EFAULT);
	}
}

static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
			     int sync, void *arg)
{
	struct wait_page_queue *wpq;
	struct io_kiocb *req = wait->private;
	struct io_async_rw *rw = &req->io->rw;
	struct wait_page_key *key = arg;
	struct task_struct *tsk;
	int ret;

	wpq = container_of(wait, struct wait_page_queue, wait);

	ret = wake_page_match(wpq, key);
	if (ret != 1)
		return ret;

	list_del_init(&wait->entry);

	init_task_work(&rw->task_work, io_async_buf_retry);
	/* submit ref gets dropped, acquire a new one */
	refcount_inc(&req->refs);
	tsk = req->task;
	ret = task_work_add(tsk, &rw->task_work, true);
	if (unlikely(ret)) {
		/* queue just for cancelation */
		init_task_work(&rw->task_work, io_async_buf_cancel);
		tsk = io_wq_get_task(req->ctx->io_wq);
		task_work_add(tsk, &rw->task_work, true);
	}
	wake_up_process(tsk);
	return 1;
}

static bool io_rw_should_retry(struct io_kiocb *req)
{
	struct kiocb *kiocb = &req->rw.kiocb;
	int ret;

	/* never retry for NOWAIT, we just complete with -EAGAIN */
	if (req->flags & REQ_F_NOWAIT)
		return false;

	/* already tried, or we're doing O_DIRECT */
	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_WAITQ))
		return false;
	/*
	 * just use poll if we can, and don't attempt if the fs doesn't
	 * support callback based unlocks
	 */
	if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
		return false;

	/*
	 * If request type doesn't require req->io to defer in general,
	 * we need to allocate it here
	 */
	if (!req->io && __io_alloc_async_ctx(req))
		return false;

	ret = kiocb_wait_page_queue_init(kiocb, &req->io->rw.wpq,
						io_async_buf_func, req);
	if (!ret) {
		io_get_req_task(req);
		return true;
	}

	return false;
}

static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (req->file->f_op->read_iter)
		return call_read_iter(req->file, &req->rw.kiocb, iter);
	return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
}

2869
static int io_read(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
2870 2871
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2872
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2873
	struct iov_iter iter;
2874
	size_t iov_count;
2875
	ssize_t io_size, ret;
J
Jens Axboe 已提交
2876

2877
	ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
2878 2879
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2880

2881 2882
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2883
		kiocb->ki_flags &= ~IOCB_NOWAIT;
2884

2885
	req->result = 0;
2886
	io_size = ret;
2887
	if (req->flags & REQ_F_LINK_HEAD)
2888 2889
		req->result = io_size;

2890
	/* If the file doesn't support async, just async punt */
2891
	if (force_nonblock && !io_file_supports_async(req->file, READ))
2892
		goto copy_iov;
J
Jens Axboe 已提交
2893

2894
	iov_count = iov_iter_count(&iter);
2895
	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2896
	if (!ret) {
2897
		unsigned long nr_segs = iter.nr_segs;
2898
		ssize_t ret2 = 0;
J
Jens Axboe 已提交
2899

2900
		ret2 = io_iter_do_read(req, &iter);
2901

2902
		/* Catch -EAGAIN return for forced non-blocking submission */
2903
		if (!force_nonblock || (ret2 != -EAGAIN && ret2 != -EIO)) {
2904
			kiocb_done(kiocb, ret2);
2905
		} else {
2906 2907
			iter.count = iov_count;
			iter.nr_segs = nr_segs;
2908
copy_iov:
2909
			ret = io_setup_async_rw(req, io_size, iovec,
2910 2911 2912
						inline_vecs, &iter);
			if (ret)
				goto out_free;
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
			/* if we can retry, do so with the callbacks armed */
			if (io_rw_should_retry(req)) {
				ret2 = io_iter_do_read(req, &iter);
				if (ret2 == -EIOCBQUEUED) {
					goto out_free;
				} else if (ret2 != -EAGAIN) {
					kiocb_done(kiocb, ret2);
					goto out_free;
				}
			}
			kiocb->ki_flags &= ~IOCB_WAITQ;
2924 2925
			return -EAGAIN;
		}
J
Jens Axboe 已提交
2926
	}
2927
out_free:
2928 2929
	if (!(req->flags & REQ_F_NEED_CLEANUP))
		kfree(iovec);
J
Jens Axboe 已提交
2930 2931 2932
	return ret;
}

2933 2934
static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 bool force_nonblock)
2935
{
2936 2937
	struct io_async_ctx *io;
	struct iov_iter iter;
2938 2939
	ssize_t ret;

2940 2941 2942
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2943

2944 2945
	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
		return -EBADF;
2946

2947 2948
	req->fsize = rlimit(RLIMIT_FSIZE);

2949 2950
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
2951 2952 2953 2954 2955
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
2956
	ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
2957 2958 2959 2960 2961 2962
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2963 2964
}

2965
static int io_write(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
2966 2967
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2968
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2969
	struct iov_iter iter;
2970
	size_t iov_count;
2971
	ssize_t ret, io_size;
J
Jens Axboe 已提交
2972

2973
	ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
2974 2975
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2976

2977 2978
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2979
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2980

2981
	req->result = 0;
2982
	io_size = ret;
2983
	if (req->flags & REQ_F_LINK_HEAD)
2984
		req->result = io_size;
J
Jens Axboe 已提交
2985

2986
	/* If the file doesn't support async, just async punt */
2987
	if (force_nonblock && !io_file_supports_async(req->file, WRITE))
2988
		goto copy_iov;
2989

2990 2991 2992
	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
2993
		goto copy_iov;
2994

2995
	iov_count = iov_iter_count(&iter);
2996
	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2997
	if (!ret) {
2998
		unsigned long nr_segs = iter.nr_segs;
2999 3000
		ssize_t ret2;

J
Jens Axboe 已提交
3001 3002 3003 3004 3005 3006 3007
		/*
		 * Open-code file_start_write here to grab freeze protection,
		 * which will be released by another thread in
		 * io_complete_rw().  Fool lockdep by telling it the lock got
		 * released so that it doesn't complain about the held lock when
		 * we return to userspace.
		 */
3008
		if (req->flags & REQ_F_ISREG) {
3009
			__sb_start_write(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
3010
						SB_FREEZE_WRITE, true);
3011
			__sb_writers_release(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
3012 3013 3014
						SB_FREEZE_WRITE);
		}
		kiocb->ki_flags |= IOCB_WRITE;
3015

3016 3017 3018
		if (!force_nonblock)
			current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;

3019 3020
		if (req->file->f_op->write_iter)
			ret2 = call_write_iter(req->file, kiocb, &iter);
3021
		else
3022
			ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
3023 3024 3025 3026

		if (!force_nonblock)
			current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;

3027
		/*
3028
		 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3029 3030 3031 3032
		 * retry them without IOCB_NOWAIT.
		 */
		if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
			ret2 = -EAGAIN;
3033
		if (!force_nonblock || ret2 != -EAGAIN) {
3034
			kiocb_done(kiocb, ret2);
3035
		} else {
3036 3037
			iter.count = iov_count;
			iter.nr_segs = nr_segs;
3038
copy_iov:
3039
			ret = io_setup_async_rw(req, io_size, iovec,
3040 3041 3042 3043 3044
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
J
Jens Axboe 已提交
3045
	}
3046
out_free:
3047 3048
	if (!(req->flags & REQ_F_NEED_CLEANUP))
		kfree(iovec);
J
Jens Axboe 已提交
3049 3050 3051
	return ret;
}

P
Pavel Begunkov 已提交
3052 3053
static int __io_splice_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
P
Pavel Begunkov 已提交
3054 3055 3056 3057 3058 3059 3060
{
	struct io_splice* sp = &req->splice;
	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
	int ret;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3061 3062
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
P
Pavel Begunkov 已提交
3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076

	sp->file_in = NULL;
	sp->len = READ_ONCE(sqe->len);
	sp->flags = READ_ONCE(sqe->splice_flags);

	if (unlikely(sp->flags & ~valid_flags))
		return -EINVAL;

	ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
			  (sp->flags & SPLICE_F_FD_IN_FIXED));
	if (ret)
		return ret;
	req->flags |= REQ_F_NEED_CLEANUP;

3077 3078 3079 3080 3081 3082
	if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
		/*
		 * Splice operation will be punted aync, and here need to
		 * modify io_wq_work.flags, so initialize io_wq_work firstly.
		 */
		io_req_init_async(req);
P
Pavel Begunkov 已提交
3083
		req->work.flags |= IO_WQ_WORK_UNBOUND;
3084
	}
P
Pavel Begunkov 已提交
3085 3086 3087 3088

	return 0;
}

P
Pavel Begunkov 已提交
3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
static int io_tee_prep(struct io_kiocb *req,
		       const struct io_uring_sqe *sqe)
{
	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
		return -EINVAL;
	return __io_splice_prep(req, sqe);
}

static int io_tee(struct io_kiocb *req, bool force_nonblock)
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	long ret = 0;

	if (force_nonblock)
		return -EAGAIN;
	if (sp->len)
		ret = do_tee(in, out, sp->len, flags);

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	io_cqring_add_event(req, ret);
	if (ret != sp->len)
		req_set_fail_links(req);
	io_put_req(req);
	return 0;
}

static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_splice* sp = &req->splice;

	sp->off_in = READ_ONCE(sqe->splice_off_in);
	sp->off_out = READ_ONCE(sqe->off);
	return __io_splice_prep(req, sqe);
}

3129
static int io_splice(struct io_kiocb *req, bool force_nonblock)
P
Pavel Begunkov 已提交
3130 3131 3132 3133 3134 3135
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	loff_t *poff_in, *poff_out;
3136
	long ret = 0;
P
Pavel Begunkov 已提交
3137

3138 3139
	if (force_nonblock)
		return -EAGAIN;
P
Pavel Begunkov 已提交
3140 3141 3142

	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
3143

3144
	if (sp->len)
3145
		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
P
Pavel Begunkov 已提交
3146 3147 3148 3149 3150 3151 3152

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	io_cqring_add_event(req, ret);
	if (ret != sp->len)
		req_set_fail_links(req);
3153
	io_put_req(req);
P
Pavel Begunkov 已提交
3154 3155 3156
	return 0;
}

J
Jens Axboe 已提交
3157 3158 3159
/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
3160
static int io_nop(struct io_kiocb *req)
J
Jens Axboe 已提交
3161 3162 3163
{
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
3164 3165 3166
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

3167
	io_cqring_add_event(req, 0);
3168
	io_put_req(req);
J
Jens Axboe 已提交
3169 3170 3171
	return 0;
}

3172
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
C
Christoph Hellwig 已提交
3173
{
J
Jens Axboe 已提交
3174
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
3175

J
Jens Axboe 已提交
3176 3177
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
3178

J
Jens Axboe 已提交
3179
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
3180
		return -EINVAL;
3181
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
3182 3183
		return -EINVAL;

3184 3185 3186 3187 3188 3189
	req->sync.flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
C
Christoph Hellwig 已提交
3190 3191 3192
	return 0;
}

3193
static int io_fsync(struct io_kiocb *req, bool force_nonblock)
3194 3195 3196 3197
{
	loff_t end = req->sync.off + req->sync.len;
	int ret;

3198 3199 3200 3201
	/* fsync always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

3202
	ret = vfs_fsync_range(req->file, req->sync.off,
3203 3204 3205 3206 3207
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3208
	io_put_req(req);
C
Christoph Hellwig 已提交
3209 3210 3211
	return 0;
}

3212 3213 3214 3215 3216
static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;
3217 3218
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3219 3220 3221 3222

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
3223
	req->fsize = rlimit(RLIMIT_FSIZE);
3224 3225 3226
	return 0;
}

3227
static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
3228
{
3229 3230
	int ret;

3231
	/* fallocate always requiring blocking context */
3232
	if (force_nonblock)
3233 3234
		return -EAGAIN;

3235 3236 3237 3238 3239 3240 3241 3242
	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
				req->sync.len);
	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
3243 3244 3245
	return 0;
}

3246
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3247
{
3248
	const char __user *fname;
3249
	int ret;
3250

3251
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3252
		return -EINVAL;
3253
	if (unlikely(sqe->ioprio || sqe->buf_index))
3254
		return -EINVAL;
3255
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
3256
		return -EBADF;
3257

3258 3259
	/* open.how should be already initialised */
	if (!(req->open.how.flags & O_PATH) && force_o_largefile())
3260
		req->open.how.flags |= O_LARGEFILE;
3261

3262 3263
	req->open.dfd = READ_ONCE(sqe->fd);
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3264
	req->open.filename = getname(fname);
3265 3266 3267 3268 3269
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}
3270
	req->open.nofile = rlimit(RLIMIT_NOFILE);
3271
	req->flags |= REQ_F_NEED_CLEANUP;
3272
	return 0;
3273 3274
}

3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286
static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	u64 flags, mode;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
	mode = READ_ONCE(sqe->len);
	flags = READ_ONCE(sqe->open_flags);
	req->open.how = build_open_how(flags, mode);
	return __io_openat_prep(req, sqe);
}

3287
static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
3288
{
3289 3290
	struct open_how __user *how;
	size_t len;
J
Jens Axboe 已提交
3291 3292
	int ret;

3293 3294
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3295 3296 3297 3298
	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	len = READ_ONCE(sqe->len);
	if (len < OPEN_HOW_SIZE_VER0)
		return -EINVAL;
3299

3300 3301 3302 3303
	ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
					len);
	if (ret)
		return ret;
3304

3305
	return __io_openat_prep(req, sqe);
3306 3307
}

3308
static int io_openat2(struct io_kiocb *req, bool force_nonblock)
3309 3310 3311 3312 3313
{
	struct open_flags op;
	struct file *file;
	int ret;

3314
	if (force_nonblock)
3315 3316
		return -EAGAIN;

3317
	ret = build_open_flags(&req->open.how, &op);
3318 3319 3320
	if (ret)
		goto err;

3321
	ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334
	if (ret < 0)
		goto err;

	file = do_filp_open(req->open.dfd, req->open.filename, &op);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
	} else {
		fsnotify_open(file);
		fd_install(ret, file);
	}
err:
	putname(req->open.filename);
3335
	req->flags &= ~REQ_F_NEED_CLEANUP;
3336 3337 3338
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3339
	io_put_req(req);
3340 3341 3342
	return 0;
}

3343
static int io_openat(struct io_kiocb *req, bool force_nonblock)
3344
{
3345
	return io_openat2(req, force_nonblock);
3346 3347
}

3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416
static int io_remove_buffers_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}

static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
			       int bgid, unsigned nbufs)
{
	unsigned i = 0;

	/* shouldn't happen */
	if (!nbufs)
		return 0;

	/* the head kbuf is the list itself */
	while (!list_empty(&buf->list)) {
		struct io_buffer *nxt;

		nxt = list_first_entry(&buf->list, struct io_buffer, list);
		list_del(&nxt->list);
		kfree(nxt);
		if (++i == nbufs)
			return i;
	}
	i++;
	kfree(buf);
	idr_remove(&ctx->io_buffer_idr, bgid);

	return i;
}

static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock)
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	ret = -ENOENT;
	head = idr_find(&ctx->io_buffer_idr, p->bgid);
	if (head)
		ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);

	io_ring_submit_lock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
	return 0;
}

3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432
static int io_provide_buffers_prep(struct io_kiocb *req,
				   const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

3433
	if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}

static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
{
	struct io_buffer *buf;
	u64 addr = pbuf->addr;
	int i, bid = pbuf->bid;

	for (i = 0; i < pbuf->nbufs; i++) {
		buf = kmalloc(sizeof(*buf), GFP_KERNEL);
		if (!buf)
			break;

		buf->addr = addr;
		buf->len = pbuf->len;
		buf->bid = bid;
		addr += pbuf->len;
		bid++;
		if (!*head) {
			INIT_LIST_HEAD(&buf->list);
			*head = buf;
		} else {
			list_add_tail(&buf->list, &(*head)->list);
		}
	}

	return i ? i : -ENOMEM;
}

static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head, *list;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	list = head = idr_find(&ctx->io_buffer_idr, p->bgid);

	ret = io_add_buffers(p, &head);
	if (ret < 0)
		goto out;

	if (!list) {
		ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
					GFP_KERNEL);
		if (ret < 0) {
3492
			__io_remove_buffers(ctx, head, p->bgid, -1U);
3493 3494 3495 3496 3497 3498 3499 3500 3501 3502
			goto out;
		}
	}
out:
	io_ring_submit_unlock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
	return 0;
3503 3504
}

3505 3506 3507 3508 3509 3510
static int io_epoll_ctl_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_EPOLL)
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3511 3512
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531

	req->epoll.epfd = READ_ONCE(sqe->fd);
	req->epoll.op = READ_ONCE(sqe->len);
	req->epoll.fd = READ_ONCE(sqe->off);

	if (ep_op_has_event(req->epoll.op)) {
		struct epoll_event __user *ev;

		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
		if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
			return -EFAULT;
	}

	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3532
static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)
3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544
{
#if defined(CONFIG_EPOLL)
	struct io_epoll *ie = &req->epoll;
	int ret;

	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3545
	io_put_req(req);
3546 3547 3548 3549 3550 3551
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3552 3553 3554 3555 3556
static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	if (sqe->ioprio || sqe->buf_index || sqe->off)
		return -EINVAL;
3557 3558
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3559 3560 3561 3562 3563 3564 3565 3566 3567 3568

	req->madvise.addr = READ_ONCE(sqe->addr);
	req->madvise.len = READ_ONCE(sqe->len);
	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3569
static int io_madvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3582
	io_put_req(req);
J
Jens Axboe 已提交
3583 3584 3585 3586 3587 3588
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3589 3590 3591 3592
static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr)
		return -EINVAL;
3593 3594
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3595 3596 3597 3598 3599 3600 3601

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}

3602
static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3603 3604 3605 3606
{
	struct io_fadvise *fa = &req->fadvise;
	int ret;

3607 3608 3609 3610 3611 3612 3613 3614 3615 3616
	if (force_nonblock) {
		switch (fa->advice) {
		case POSIX_FADV_NORMAL:
		case POSIX_FADV_RANDOM:
		case POSIX_FADV_SEQUENTIAL:
			break;
		default:
			return -EAGAIN;
		}
	}
J
Jens Axboe 已提交
3617 3618 3619 3620 3621

	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3622
	io_put_req(req);
J
Jens Axboe 已提交
3623 3624 3625
	return 0;
}

3626 3627
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
3628 3629
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3630 3631
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3632
	if (req->flags & REQ_F_FIXED_FILE)
3633
		return -EBADF;
3634

3635 3636
	req->statx.dfd = READ_ONCE(sqe->fd);
	req->statx.mask = READ_ONCE(sqe->len);
B
Bijan Mottahedeh 已提交
3637
	req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
3638 3639
	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	req->statx.flags = READ_ONCE(sqe->statx_flags);
3640 3641 3642 3643

	return 0;
}

3644
static int io_statx(struct io_kiocb *req, bool force_nonblock)
3645
{
3646
	struct io_statx *ctx = &req->statx;
3647 3648
	int ret;

3649 3650 3651 3652
	if (force_nonblock) {
		/* only need file table for an actual valid fd */
		if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD)
			req->flags |= REQ_F_NO_FILE_TABLE;
3653
		return -EAGAIN;
3654
	}
3655

B
Bijan Mottahedeh 已提交
3656 3657
	ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
		       ctx->buffer);
3658 3659 3660 3661

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3662
	io_put_req(req);
3663 3664 3665
	return 0;
}

3666 3667 3668 3669
static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	/*
	 * If we queue this for async, it must not be cancellable. That would
3670 3671
	 * leave the 'file' in an undeterminate state, and here need to modify
	 * io_wq_work.flags, so initialize io_wq_work firstly.
3672
	 */
3673
	io_req_init_async(req);
3674 3675
	req->work.flags |= IO_WQ_WORK_NO_CANCEL;

3676 3677
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
3678 3679 3680
	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
	    sqe->rw_flags || sqe->buf_index)
		return -EINVAL;
3681
	if (req->flags & REQ_F_FIXED_FILE)
3682
		return -EBADF;
3683 3684

	req->close.fd = READ_ONCE(sqe->fd);
3685 3686 3687
	if ((req->file && req->file->f_op == &io_uring_fops) ||
	    req->close.fd == req->ctx->ring_fd)
		return -EBADF;
3688

3689
	req->close.put_file = NULL;
3690 3691 3692
	return 0;
}

3693
static int io_close(struct io_kiocb *req, bool force_nonblock)
3694
{
3695
	struct io_close *close = &req->close;
3696 3697
	int ret;

3698 3699 3700 3701 3702 3703
	/* might be already done during nonblock submission */
	if (!close->put_file) {
		ret = __close_fd_get_file(close->fd, &close->put_file);
		if (ret < 0)
			return (ret == -ENOENT) ? -EBADF : ret;
	}
3704 3705

	/* if the file has a flush method, be safe and punt to async */
3706
	if (close->put_file->f_op->flush && force_nonblock) {
3707 3708
		/* was never set, but play safe */
		req->flags &= ~REQ_F_NOWAIT;
3709
		/* avoid grabbing files - we don't need the files */
3710
		req->flags |= REQ_F_NO_FILE_TABLE;
3711
		return -EAGAIN;
P
Pavel Begunkov 已提交
3712
	}
3713

3714 3715 3716 3717 3718 3719 3720 3721
	/* No ->flush() or already async, safely close from here */
	ret = filp_close(close->put_file, req->work.files);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	fput(close->put_file);
	close->put_file = NULL;
	io_put_req(req);
3722
	return 0;
3723 3724
}

3725
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

3737 3738 3739 3740 3741 3742
	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
	req->sync.flags = READ_ONCE(sqe->sync_range_flags);
	return 0;
}

3743
static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
3744 3745 3746
{
	int ret;

3747 3748 3749 3750
	/* sync_file_range always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

3751
	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
3752 3753 3754 3755
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
3756
	io_put_req(req);
3757 3758 3759
	return 0;
}

3760
#if defined(CONFIG_NET)
3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775
static int io_setup_async_msg(struct io_kiocb *req,
			      struct io_async_msghdr *kmsg)
{
	if (req->io)
		return -EAGAIN;
	if (io_alloc_async_ctx(req)) {
		if (kmsg->iov != kmsg->fast_iov)
			kfree(kmsg->iov);
		return -ENOMEM;
	}
	req->flags |= REQ_F_NEED_CLEANUP;
	memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
	return -EAGAIN;
}

3776
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3777
{
3778
	struct io_sr_msg *sr = &req->sr_msg;
3779
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
3780
	int ret;
3781

3782 3783 3784
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

3785 3786
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3787
	sr->len = READ_ONCE(sqe->len);
3788

3789 3790 3791 3792 3793
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

3794
	if (!io || req->opcode == IORING_OP_SEND)
3795
		return 0;
3796 3797 3798
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3799

3800
	io->msg.iov = io->msg.fast_iov;
P
Pavel Begunkov 已提交
3801
	ret = sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3802
					&io->msg.iov);
P
Pavel Begunkov 已提交
3803 3804 3805
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
3806 3807
}

3808
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3809
{
3810
	struct io_async_msghdr *kmsg = NULL;
J
Jens Axboe 已提交
3811 3812 3813 3814 3815
	struct socket *sock;
	int ret;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
3816
		struct io_async_ctx io;
J
Jens Axboe 已提交
3817 3818
		unsigned flags;

3819
		if (req->io) {
3820
			kmsg = &req->io->msg;
3821
			kmsg->msg.msg_name = &req->io->msg.addr;
3822 3823 3824 3825
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
3826
		} else {
3827 3828
			struct io_sr_msg *sr = &req->sr_msg;

3829
			kmsg = &io.msg;
3830
			kmsg->msg.msg_name = &io.msg.addr;
3831 3832 3833 3834

			io.msg.iov = io.msg.fast_iov;
			ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.iov);
3835
			if (ret)
3836
				return ret;
3837
		}
J
Jens Axboe 已提交
3838

3839 3840 3841 3842 3843 3844
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3845
		ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3846 3847
		if (force_nonblock && ret == -EAGAIN)
			return io_setup_async_msg(req, kmsg);
3848 3849
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
3850 3851
	}

3852
	if (kmsg && kmsg->iov != kmsg->fast_iov)
3853
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
3854
	req->flags &= ~REQ_F_NEED_CLEANUP;
3855
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3856 3857
	if (ret < 0)
		req_set_fail_links(req);
3858
	io_put_req(req);
3859
	return 0;
3860
}
J
Jens Axboe 已提交
3861

3862
static int io_send(struct io_kiocb *req, bool force_nonblock)
3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
{
	struct socket *sock;
	int ret;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

		ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
						&msg.msg_iter);
		if (ret)
			return ret;

		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3890 3891
		msg.msg_flags = flags;
		ret = sock_sendmsg(sock, &msg);
3892 3893 3894 3895 3896 3897 3898 3899 3900
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
3901
	io_put_req(req);
3902 3903 3904
	return 0;
}

3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct iovec __user *uiov;
	size_t iov_len;
	int ret;

	ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,
					&uiov, &iov_len);
	if (ret)
		return ret;

	if (req->flags & REQ_F_BUFFER_SELECT) {
		if (iov_len > 1)
			return -EINVAL;
		if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))
			return -EFAULT;
		sr->len = io->msg.iov[0].iov_len;
		iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,
				sr->len);
		io->msg.iov = NULL;
	} else {
		ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
					&io->msg.iov, &io->msg.msg.msg_iter);
		if (ret > 0)
			ret = 0;
	}

	return ret;
}

#ifdef CONFIG_COMPAT
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
					struct io_async_ctx *io)
{
	struct compat_msghdr __user *msg_compat;
	struct io_sr_msg *sr = &req->sr_msg;
	struct compat_iovec __user *uiov;
	compat_uptr_t ptr;
	compat_size_t len;
	int ret;

	msg_compat = (struct compat_msghdr __user *) sr->msg;
	ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,
					&ptr, &len);
	if (ret)
		return ret;

	uiov = compat_ptr(ptr);
	if (req->flags & REQ_F_BUFFER_SELECT) {
		compat_ssize_t clen;

		if (len > 1)
			return -EINVAL;
		if (!access_ok(uiov, sizeof(*uiov)))
			return -EFAULT;
		if (__get_user(clen, &uiov->iov_len))
			return -EFAULT;
		if (clen < 0)
			return -EINVAL;
		sr->len = io->msg.iov[0].iov_len;
		io->msg.iov = NULL;
	} else {
		ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
						&io->msg.iov,
						&io->msg.msg.msg_iter);
		if (ret < 0)
			return ret;
	}

	return 0;
}
#endif

static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
{
	io->msg.iov = io->msg.fast_iov;

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		return __io_compat_recvmsg_copy_hdr(req, io);
3986
#endif
3987 3988 3989 3990

	return __io_recvmsg_copy_hdr(req, io);
}

3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009
static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
					       int *cflags, bool needs_lock)
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct io_buffer *kbuf;

	if (!(req->flags & REQ_F_BUFFER_SELECT))
		return NULL;

	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;

	sr->kbuf = kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;

	*cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	*cflags |= IORING_CQE_F_BUFFER;
	return kbuf;
4010 4011
}

4012 4013
static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4014
{
4015
	struct io_sr_msg *sr = &req->sr_msg;
4016
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
4017
	int ret;
4018

4019 4020 4021
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

4022 4023
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4024
	sr->len = READ_ONCE(sqe->len);
4025
	sr->bgid = READ_ONCE(sqe->buf_group);
4026

4027 4028 4029 4030 4031
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

4032
	if (!io || req->opcode == IORING_OP_RECV)
4033
		return 0;
4034 4035 4036
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
4037

4038
	ret = io_recvmsg_copy_hdr(req, io);
P
Pavel Begunkov 已提交
4039 4040 4041
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
J
Jens Axboe 已提交
4042 4043
}

4044
static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
4045
{
4046
	struct io_async_msghdr *kmsg = NULL;
4047
	struct socket *sock;
4048
	int ret, cflags = 0;
4049 4050 4051

	sock = sock_from_file(req->file, &ret);
	if (sock) {
4052
		struct io_buffer *kbuf;
4053
		struct io_async_ctx io;
4054 4055 4056
		unsigned flags;

		if (req->io) {
4057
			kmsg = &req->io->msg;
4058
			kmsg->msg.msg_name = &req->io->msg.addr;
4059 4060 4061 4062
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
4063
		} else {
4064
			kmsg = &io.msg;
4065
			kmsg->msg.msg_name = &io.msg.addr;
4066

4067
			ret = io_recvmsg_copy_hdr(req, &io);
4068
			if (ret)
4069
				return ret;
4070 4071
		}

4072 4073 4074 4075 4076 4077 4078 4079 4080
		kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
		if (IS_ERR(kbuf)) {
			return PTR_ERR(kbuf);
		} else if (kbuf) {
			kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
			iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
					1, req->sr_msg.len);
		}

4081 4082 4083 4084 4085 4086 4087 4088
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

		ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
						kmsg->uaddr, flags);
4089 4090
		if (force_nonblock && ret == -EAGAIN)
			return io_setup_async_msg(req, kmsg);
4091 4092 4093 4094
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

4095
	if (kmsg && kmsg->iov != kmsg->fast_iov)
4096
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
4097
	req->flags &= ~REQ_F_NEED_CLEANUP;
4098
	__io_cqring_add_event(req, ret, cflags);
J
Jens Axboe 已提交
4099 4100
	if (ret < 0)
		req_set_fail_links(req);
4101
	io_put_req(req);
4102
	return 0;
J
Jens Axboe 已提交
4103
}
4104

4105
static int io_recv(struct io_kiocb *req, bool force_nonblock)
4106
{
4107
	struct io_buffer *kbuf = NULL;
4108
	struct socket *sock;
4109
	int ret, cflags = 0;
4110 4111 4112 4113

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
4114
		void __user *buf = sr->buf;
4115 4116 4117 4118
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

4119 4120 4121 4122 4123 4124 4125
		kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
		if (IS_ERR(kbuf))
			return PTR_ERR(kbuf);
		else if (kbuf)
			buf = u64_to_user_ptr(kbuf->addr);

		ret = import_single_range(READ, buf, sr->len, &iov,
4126
						&msg.msg_iter);
4127 4128
		if (ret) {
			kfree(kbuf);
4129
			return ret;
4130
		}
4131

4132
		req->flags |= REQ_F_NEED_CLEANUP;
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145
		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;
		msg.msg_iocb = NULL;
		msg.msg_flags = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

4146
		ret = sock_recvmsg(sock, &msg, flags);
4147 4148 4149 4150 4151 4152
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

4153 4154 4155
	kfree(kbuf);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	__io_cqring_add_event(req, ret, cflags);
4156 4157
	if (ret < 0)
		req_set_fail_links(req);
4158
	io_put_req(req);
4159 4160 4161
	return 0;
}

4162
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4163
{
4164 4165
	struct io_accept *accept = &req->accept;

4166 4167
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
4168
	if (sqe->ioprio || sqe->len || sqe->buf_index)
4169 4170
		return -EINVAL;

4171 4172
	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4173
	accept->flags = READ_ONCE(sqe->accept_flags);
4174
	accept->nofile = rlimit(RLIMIT_NOFILE);
4175 4176
	return 0;
}
4177

4178
static int io_accept(struct io_kiocb *req, bool force_nonblock)
4179 4180
{
	struct io_accept *accept = &req->accept;
4181
	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
4182 4183
	int ret;

4184 4185 4186
	if (req->file->f_flags & O_NONBLOCK)
		req->flags |= REQ_F_NOWAIT;

4187
	ret = __sys_accept4_file(req->file, file_flags, accept->addr,
4188 4189
					accept->addr_len, accept->flags,
					accept->nofile);
4190
	if (ret == -EAGAIN && force_nonblock)
4191
		return -EAGAIN;
4192 4193 4194
	if (ret < 0) {
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
4195
		req_set_fail_links(req);
4196
	}
4197
	io_cqring_add_event(req, ret);
4198
	io_put_req(req);
4199
	return 0;
4200 4201
}

4202
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4203
{
4204 4205
	struct io_connect *conn = &req->connect;
	struct io_async_ctx *io = req->io;
4206

4207 4208 4209 4210 4211
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

4212 4213 4214 4215 4216 4217 4218
	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	conn->addr_len =  READ_ONCE(sqe->addr2);

	if (!io)
		return 0;

	return move_addr_to_kernel(conn->addr, conn->addr_len,
4219
					&io->connect.address);
4220 4221
}

4222
static int io_connect(struct io_kiocb *req, bool force_nonblock)
4223
{
4224
	struct io_async_ctx __io, *io;
4225
	unsigned file_flags;
4226
	int ret;
4227

4228 4229 4230
	if (req->io) {
		io = req->io;
	} else {
4231 4232 4233
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
4234 4235 4236 4237 4238
		if (ret)
			goto out;
		io = &__io;
	}

4239 4240 4241 4242
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
4243
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
4244 4245 4246
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
4247 4248 4249
			ret = -ENOMEM;
			goto out;
		}
4250
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
4251
		return -EAGAIN;
4252
	}
4253 4254
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4255
out:
J
Jens Axboe 已提交
4256 4257
	if (ret < 0)
		req_set_fail_links(req);
4258
	io_cqring_add_event(req, ret);
4259
	io_put_req(req);
4260
	return 0;
4261 4262 4263 4264
}
#else /* !CONFIG_NET */
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
4265 4266 4267
	return -EOPNOTSUPP;
}

4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
{
	return -EOPNOTSUPP;
}

static int io_send(struct io_kiocb *req, bool force_nonblock)
{
	return -EOPNOTSUPP;
}

static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
{
	return -EOPNOTSUPP;
}

static int io_recv(struct io_kiocb *req, bool force_nonblock)
{
	return -EOPNOTSUPP;
}

static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

static int io_accept(struct io_kiocb *req, bool force_nonblock)
{
	return -EOPNOTSUPP;
}

static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

static int io_connect(struct io_kiocb *req, bool force_nonblock)
{
4311 4312
	return -EOPNOTSUPP;
}
4313
#endif /* CONFIG_NET */
4314

4315 4316 4317 4318 4319 4320 4321 4322 4323 4324
struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};

static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
			   __poll_t mask, task_work_func_t func)
{
	struct task_struct *tsk;
4325
	int ret;
4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

	trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);

	list_del_init(&poll->wait.entry);

	tsk = req->task;
	req->result = mask;
	init_task_work(&req->task_work, func);
	/*
4339 4340 4341 4342
	 * If this fails, then the task is exiting. When a task exits, the
	 * work gets canceled, so just cancel this request as well instead
	 * of executing it. We can't safely execute it anyway, as we may not
	 * have the needed state needed for it anyway.
4343
	 */
4344 4345
	ret = task_work_add(tsk, &req->task_work, true);
	if (unlikely(ret)) {
4346
		WRITE_ONCE(poll->canceled, true);
4347 4348 4349
		tsk = io_wq_get_task(req->ctx->io_wq);
		task_work_add(tsk, &req->task_work, true);
	}
4350 4351 4352 4353
	wake_up_process(tsk);
	return 1;
}

4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373
static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
	__acquires(&req->ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->result && !READ_ONCE(poll->canceled)) {
		struct poll_table_struct pt = { ._key = poll->events };

		req->result = vfs_poll(req->file, &pt) & poll->events;
	}

	spin_lock_irq(&ctx->completion_lock);
	if (!req->result && !READ_ONCE(poll->canceled)) {
		add_wait_queue(poll->head, &poll->wait);
		return true;
	}

	return false;
}

4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500
static void io_poll_remove_double(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;

	lockdep_assert_held(&req->ctx->completion_lock);

	if (poll && poll->head) {
		struct wait_queue_head *head = poll->head;

		spin_lock(&head->lock);
		list_del_init(&poll->wait.entry);
		if (poll->wait.private)
			refcount_dec(&req->refs);
		poll->head = NULL;
		spin_unlock(&head->lock);
	}
}

static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_poll_remove_double(req);
	req->poll.done = true;
	io_cqring_fill_event(req, error ? error : mangle_poll(mask));
	io_commit_cqring(ctx);
}

static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (io_poll_rewait(req, &req->poll)) {
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}

	hash_del(&req->hash_node);
	io_poll_complete(req, req->result, 0);
	req->flags |= REQ_F_COMP_LOCKED;
	io_put_req_find_next(req, nxt);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
}

static void io_poll_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct io_kiocb *nxt = NULL;

	io_poll_task_handler(req, &nxt);
	if (nxt) {
		struct io_ring_ctx *ctx = nxt->ctx;

		mutex_lock(&ctx->uring_lock);
		__io_queue_sqe(nxt, NULL);
		mutex_unlock(&ctx->uring_lock);
	}
}

static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
			       int sync, void *key)
{
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;
	__poll_t mask = key_to_poll(key);

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

	if (req->poll.head) {
		bool done;

		spin_lock(&req->poll.head->lock);
		done = list_empty(&req->poll.wait.entry);
		if (!done)
			list_del_init(&req->poll.wait.entry);
		spin_unlock(&req->poll.head->lock);
		if (!done)
			__io_async_wake(req, poll, mask, io_poll_task_func);
	}
	refcount_dec(&req->refs);
	return 1;
}

static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
			      wait_queue_func_t wake_func)
{
	poll->head = NULL;
	poll->done = false;
	poll->canceled = false;
	poll->events = events;
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, wake_func);
}

static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
			    struct wait_queue_head *head)
{
	struct io_kiocb *req = pt->req;

	/*
	 * If poll->head is already set, it's because the file being polled
	 * uses multiple waitqueues for poll handling (eg one for read, one
	 * for write). Setup a separate io_poll_iocb if this happens.
	 */
	if (unlikely(poll->head)) {
		/* already have a 2nd entry, fail a third attempt */
		if (req->io) {
			pt->error = -EINVAL;
			return;
		}
		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
		if (!poll) {
			pt->error = -ENOMEM;
			return;
		}
		io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);
		refcount_inc(&req->refs);
		poll->wait.private = req;
		req->io = (void *) poll;
	}

	pt->error = 0;
	poll->head = head;
4501 4502 4503 4504 4505

	if (poll->events & EPOLLEXCLUSIVE)
		add_wait_queue_exclusive(head, &poll->wait);
	else
		add_wait_queue(head, &poll->wait);
4506 4507 4508 4509 4510 4511 4512 4513 4514 4515
}

static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

	__io_queue_proc(&pt->req->apoll->poll, pt, head);
}

4516 4517 4518 4519 4520
static void io_async_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct async_poll *apoll = req->apoll;
	struct io_ring_ctx *ctx = req->ctx;
4521
	bool canceled = false;
4522 4523 4524

	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);

4525
	if (io_poll_rewait(req, &apoll->poll)) {
4526
		spin_unlock_irq(&ctx->completion_lock);
4527
		return;
4528 4529
	}

4530 4531
	/* If req is still hashed, it cannot have been canceled. Don't check. */
	if (hash_hashed(&req->hash_node)) {
4532
		hash_del(&req->hash_node);
4533 4534 4535 4536 4537 4538
	} else {
		canceled = READ_ONCE(apoll->poll.canceled);
		if (canceled) {
			io_cqring_fill_event(req, -ECANCELED);
			io_commit_cqring(ctx);
		}
4539 4540
	}

4541 4542
	spin_unlock_irq(&ctx->completion_lock);

4543
	/* restore ->work in case we need to retry again */
4544 4545
	if (req->flags & REQ_F_WORK_INITIALIZED)
		memcpy(&req->work, &apoll->work, sizeof(req->work));
4546
	kfree(apoll);
4547

4548 4549
	if (!canceled) {
		__set_current_state(TASK_RUNNING);
4550 4551 4552 4553
		if (io_sq_thread_acquire_mm(ctx, req)) {
			io_cqring_add_event(req, -EFAULT);
			goto end_req;
		}
4554 4555 4556 4557
		mutex_lock(&ctx->uring_lock);
		__io_queue_sqe(req, NULL);
		mutex_unlock(&ctx->uring_lock);
	} else {
4558
		io_cqring_ev_posted(ctx);
4559
end_req:
4560
		req_set_fail_links(req);
4561
		io_double_put_req(req);
4562
	}
4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594
}

static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->apoll->poll;

	trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
					key_to_poll(key));

	return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
}

static void io_poll_req_insert(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct hlist_head *list;

	list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	hlist_add_head(&req->hash_node, list);
}

static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
				      struct io_poll_iocb *poll,
				      struct io_poll_table *ipt, __poll_t mask,
				      wait_queue_func_t wake_func)
	__acquires(&ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;
	bool cancel = false;

4595
	io_init_poll_iocb(poll, mask, wake_func);
4596
	poll->file = req->file;
4597
	poll->wait.private = req;
4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632

	ipt->pt._key = mask;
	ipt->req = req;
	ipt->error = -EINVAL;

	mask = vfs_poll(req->file, &ipt->pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
		if (unlikely(list_empty(&poll->wait.entry))) {
			if (ipt->error)
				cancel = true;
			ipt->error = 0;
			mask = 0;
		}
		if (mask || ipt->error)
			list_del_init(&poll->wait.entry);
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
			io_poll_req_insert(req);
		spin_unlock(&poll->head->lock);
	}

	return mask;
}

static bool io_arm_poll_handler(struct io_kiocb *req)
{
	const struct io_op_def *def = &io_op_defs[req->opcode];
	struct io_ring_ctx *ctx = req->ctx;
	struct async_poll *apoll;
	struct io_poll_table ipt;
	__poll_t mask, ret;
4633
	bool had_io;
4634 4635 4636

	if (!req->file || !file_can_poll(req->file))
		return false;
4637
	if (req->flags & REQ_F_POLLED)
4638 4639 4640 4641 4642 4643 4644 4645 4646
		return false;
	if (!def->pollin && !def->pollout)
		return false;

	apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
	if (unlikely(!apoll))
		return false;

	req->flags |= REQ_F_POLLED;
4647 4648
	if (req->flags & REQ_F_WORK_INITIALIZED)
		memcpy(&apoll->work, &req->work, sizeof(req->work));
4649
	had_io = req->io != NULL;
4650

P
Pavel Begunkov 已提交
4651
	io_get_req_task(req);
4652 4653 4654
	req->apoll = apoll;
	INIT_HLIST_NODE(&req->hash_node);

4655
	mask = 0;
4656
	if (def->pollin)
4657
		mask |= POLLIN | POLLRDNORM;
4658 4659 4660 4661 4662 4663 4664 4665 4666 4667
	if (def->pollout)
		mask |= POLLOUT | POLLWRNORM;
	mask |= POLLERR | POLLPRI;

	ipt.pt._qproc = io_async_queue_proc;

	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
					io_async_wake);
	if (ret) {
		ipt.error = 0;
4668 4669 4670
		/* only remove double add if we did it here */
		if (!had_io)
			io_poll_remove_double(req);
4671
		spin_unlock_irq(&ctx->completion_lock);
4672 4673
		if (req->flags & REQ_F_WORK_INITIALIZED)
			memcpy(&req->work, &apoll->work, sizeof(req->work));
4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684
		kfree(apoll);
		return false;
	}
	spin_unlock_irq(&ctx->completion_lock);
	trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
					apoll->poll.events);
	return true;
}

static bool __io_poll_remove_one(struct io_kiocb *req,
				 struct io_poll_iocb *poll)
4685
{
4686
	bool do_complete = false;
4687 4688 4689

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
4690 4691
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
4692
		do_complete = true;
4693 4694
	}
	spin_unlock(&poll->head->lock);
4695
	hash_del(&req->hash_node);
4696 4697 4698 4699 4700 4701 4702 4703
	return do_complete;
}

static bool io_poll_remove_one(struct io_kiocb *req)
{
	bool do_complete;

	if (req->opcode == IORING_OP_POLL_ADD) {
4704
		io_poll_remove_double(req);
4705 4706
		do_complete = __io_poll_remove_one(req, &req->poll);
	} else {
4707 4708
		struct async_poll *apoll = req->apoll;

4709
		/* non-poll requests have submit ref still */
4710 4711
		do_complete = __io_poll_remove_one(req, &apoll->poll);
		if (do_complete) {
4712
			io_put_req(req);
4713 4714 4715 4716 4717
			/*
			 * restore ->work because we will call
			 * io_req_work_drop_env below when dropping the
			 * final reference.
			 */
4718 4719 4720
			if (req->flags & REQ_F_WORK_INITIALIZED)
				memcpy(&req->work, &apoll->work,
				       sizeof(req->work));
4721 4722
			kfree(apoll);
		}
4723 4724
	}

4725 4726 4727 4728 4729 4730 4731 4732
	if (do_complete) {
		io_cqring_fill_event(req, -ECANCELED);
		io_commit_cqring(req->ctx);
		req->flags |= REQ_F_COMP_LOCKED;
		io_put_req(req);
	}

	return do_complete;
4733 4734 4735 4736
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
4737
	struct hlist_node *tmp;
4738
	struct io_kiocb *req;
4739
	int posted = 0, i;
4740 4741

	spin_lock_irq(&ctx->completion_lock);
4742 4743 4744 4745 4746
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node)
4747
			posted += io_poll_remove_one(req);
4748 4749
	}
	spin_unlock_irq(&ctx->completion_lock);
4750

4751 4752
	if (posted)
		io_cqring_ev_posted(ctx);
4753 4754
}

4755 4756
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
4757
	struct hlist_head *list;
4758 4759
	struct io_kiocb *req;

4760 4761
	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	hlist_for_each_entry(req, list, hash_node) {
4762 4763 4764
		if (sqe_addr != req->user_data)
			continue;
		if (io_poll_remove_one(req))
4765
			return 0;
4766
		return -EALREADY;
4767 4768 4769 4770 4771
	}

	return -ENOENT;
}

4772 4773
static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	req->poll.addr = READ_ONCE(sqe->addr);
	return 0;
}

4785 4786 4787 4788
/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
4789
static int io_poll_remove(struct io_kiocb *req)
4790 4791
{
	struct io_ring_ctx *ctx = req->ctx;
4792
	u64 addr;
4793
	int ret;
4794

4795
	addr = req->poll.addr;
4796
	spin_lock_irq(&ctx->completion_lock);
4797
	ret = io_poll_cancel(ctx, addr);
4798 4799
	spin_unlock_irq(&ctx->completion_lock);

4800
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
4801 4802
	if (ret < 0)
		req_set_fail_links(req);
4803
	io_put_req(req);
4804 4805 4806 4807 4808 4809
	return 0;
}

static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
4810 4811
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->poll;
4812

4813
	return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
4814 4815 4816 4817 4818 4819 4820
}

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

4821
	__io_queue_proc(&pt->req->poll, pt, head);
4822 4823
}

4824
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4825 4826
{
	struct io_poll_iocb *poll = &req->poll;
4827
	u32 events;
4828 4829 4830 4831 4832

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
4833 4834
	if (!poll->file)
		return -EBADF;
4835

4836 4837 4838 4839
	events = READ_ONCE(sqe->poll32_events);
#ifdef __BIG_ENDIAN
	events = swahw32(events);
#endif
4840 4841
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
		       (events & EPOLLEXCLUSIVE);
4842

P
Pavel Begunkov 已提交
4843
	io_get_req_task(req);
4844 4845 4846
	return 0;
}

4847
static int io_poll_add(struct io_kiocb *req)
4848 4849 4850 4851 4852 4853
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	__poll_t mask;

4854
	INIT_HLIST_NODE(&req->hash_node);
4855
	INIT_LIST_HEAD(&req->list);
4856
	ipt.pt._qproc = io_poll_queue_proc;
4857

4858 4859
	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
					io_poll_wake);
4860

J
Jens Axboe 已提交
4861
	if (mask) { /* no async, we'd stolen it */
4862
		ipt.error = 0;
4863
		io_poll_complete(req, mask, 0);
4864 4865 4866
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
4867 4868
	if (mask) {
		io_cqring_ev_posted(ctx);
4869
		io_put_req(req);
4870
	}
J
Jens Axboe 已提交
4871
	return ipt.error;
4872 4873
}

J
Jens Axboe 已提交
4874 4875
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
4876 4877 4878 4879
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
4880 4881 4882 4883 4884
	unsigned long flags;

	atomic_inc(&ctx->cq_timeouts);

	spin_lock_irqsave(&ctx->completion_lock, flags);
4885
	/*
4886 4887
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
4888
	 */
4889
	if (!list_empty(&req->list))
4890
		list_del_init(&req->list);
J
Jens Axboe 已提交
4891

4892
	io_cqring_fill_event(req, -ETIME);
J
Jens Axboe 已提交
4893 4894 4895 4896
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
4897
	req_set_fail_links(req);
J
Jens Axboe 已提交
4898 4899 4900 4901
	io_put_req(req);
	return HRTIMER_NORESTART;
}

4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917
static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
{
	struct io_kiocb *req;
	int ret = -ENOENT;

	list_for_each_entry(req, &ctx->timeout_list, list) {
		if (user_data == req->user_data) {
			list_del_init(&req->list);
			ret = 0;
			break;
		}
	}

	if (ret == -ENOENT)
		return ret;

4918
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
4919 4920 4921
	if (ret == -1)
		return -EALREADY;

J
Jens Axboe 已提交
4922
	req_set_fail_links(req);
4923 4924 4925 4926 4927
	io_cqring_fill_event(req, -ECANCELED);
	io_put_req(req);
	return 0;
}

4928 4929
static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
		return -EINVAL;

	req->timeout.addr = READ_ONCE(sqe->addr);
	req->timeout.flags = READ_ONCE(sqe->timeout_flags);
	if (req->timeout.flags)
		return -EINVAL;

	return 0;
}

4944 4945 4946
/*
 * Remove or update an existing timeout command
 */
4947
static int io_timeout_remove(struct io_kiocb *req)
4948 4949
{
	struct io_ring_ctx *ctx = req->ctx;
4950
	int ret;
4951 4952

	spin_lock_irq(&ctx->completion_lock);
4953
	ret = io_timeout_cancel(ctx, req->timeout.addr);
4954

4955
	io_cqring_fill_event(req, ret);
4956 4957
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
4958
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
4959 4960
	if (ret < 0)
		req_set_fail_links(req);
4961
	io_put_req(req);
4962
	return 0;
J
Jens Axboe 已提交
4963 4964
}

4965
static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4966
			   bool is_timeout_link)
J
Jens Axboe 已提交
4967
{
4968
	struct io_timeout_data *data;
4969
	unsigned flags;
4970
	u32 off = READ_ONCE(sqe->off);
J
Jens Axboe 已提交
4971

4972
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
4973
		return -EINVAL;
4974
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
4975
		return -EINVAL;
4976
	if (off && is_timeout_link)
4977
		return -EINVAL;
4978 4979
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
4980
		return -EINVAL;
4981

4982
	req->timeout.off = off;
4983

4984
	if (!req->io && io_alloc_async_ctx(req))
4985 4986 4987
		return -ENOMEM;

	data = &req->io->timeout;
4988 4989 4990 4991
	data->req = req;
	req->flags |= REQ_F_TIMEOUT;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
4992 4993
		return -EFAULT;

4994
	if (flags & IORING_TIMEOUT_ABS)
4995
		data->mode = HRTIMER_MODE_ABS;
4996
	else
4997
		data->mode = HRTIMER_MODE_REL;
4998

4999 5000 5001 5002
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}

5003
static int io_timeout(struct io_kiocb *req)
5004 5005
{
	struct io_ring_ctx *ctx = req->ctx;
5006
	struct io_timeout_data *data = &req->io->timeout;
5007
	struct list_head *entry;
5008
	u32 tail, off = req->timeout.off;
5009

5010
	spin_lock_irq(&ctx->completion_lock);
5011

J
Jens Axboe 已提交
5012 5013
	/*
	 * sqe->off holds how many events that need to occur for this
5014 5015
	 * timeout event to be satisfied. If it isn't set, then this is
	 * a pure timeout request, sequence isn't used.
J
Jens Axboe 已提交
5016
	 */
5017
	if (!off) {
5018 5019 5020 5021
		req->flags |= REQ_F_TIMEOUT_NOSEQ;
		entry = ctx->timeout_list.prev;
		goto add;
	}
J
Jens Axboe 已提交
5022

5023 5024
	tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
	req->timeout.target_seq = tail + off;
J
Jens Axboe 已提交
5025 5026 5027 5028 5029 5030 5031 5032

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	list_for_each_prev(entry, &ctx->timeout_list) {
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);

5033 5034
		if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
			continue;
5035 5036
		/* nxt.seq is behind @tail, otherwise would've been completed */
		if (off >= nxt->timeout.target_seq - tail)
J
Jens Axboe 已提交
5037 5038
			break;
	}
5039
add:
J
Jens Axboe 已提交
5040
	list_add(&req->list, entry);
5041 5042
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
J
Jens Axboe 已提交
5043 5044 5045 5046
	spin_unlock_irq(&ctx->completion_lock);
	return 0;
}

5047 5048 5049 5050 5051 5052 5053
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

5054
static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
5055 5056 5057 5058
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;

5059
	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

5072 5073 5074
	return ret;
}

5075 5076
static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
5077
				     int success_ret)
5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
	if (ret != -ENOENT) {
		spin_lock_irqsave(&ctx->completion_lock, flags);
		goto done;
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr);
done:
5094 5095
	if (!ret)
		ret = success_ret;
5096 5097 5098 5099 5100
	io_cqring_fill_event(req, ret);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

J
Jens Axboe 已提交
5101 5102
	if (ret < 0)
		req_set_fail_links(req);
5103
	io_put_req(req);
5104 5105
}

5106 5107
static int io_async_cancel_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
5108
{
5109
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5110 5111 5112 5113 5114
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
	    sqe->cancel_flags)
		return -EINVAL;

5115 5116 5117 5118
	req->cancel.addr = READ_ONCE(sqe->addr);
	return 0;
}

5119
static int io_async_cancel(struct io_kiocb *req)
5120 5121 5122
{
	struct io_ring_ctx *ctx = req->ctx;

5123
	io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
J
Jens Axboe 已提交
5124 5125 5126
	return 0;
}

5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141
static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (sqe->flags || sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	req->files_update.offset = READ_ONCE(sqe->off);
	req->files_update.nr_args = READ_ONCE(sqe->len);
	if (!req->files_update.nr_args)
		return -EINVAL;
	req->files_update.arg = READ_ONCE(sqe->addr);
	return 0;
}

static int io_files_update(struct io_kiocb *req, bool force_nonblock)
5142 5143
{
	struct io_ring_ctx *ctx = req->ctx;
5144 5145
	struct io_uring_files_update up;
	int ret;
5146

5147
	if (force_nonblock)
5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
J
Jens Axboe 已提交
5161 5162 5163
	return 0;
}

5164 5165
static int io_req_defer_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
5166
{
J
Jens Axboe 已提交
5167
	ssize_t ret = 0;
5168

5169 5170 5171
	if (!sqe)
		return 0;

5172 5173
	io_req_init_async(req);

5174 5175 5176 5177 5178 5179
	if (io_op_defs[req->opcode].file_table) {
		ret = io_grab_files(req);
		if (unlikely(ret))
			return ret;
	}

5180 5181
	io_req_work_grab_env(req, &io_op_defs[req->opcode]);

5182
	switch (req->opcode) {
J
Jens Axboe 已提交
5183 5184
	case IORING_OP_NOP:
		break;
5185 5186
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
5187
	case IORING_OP_READ:
5188
		ret = io_read_prep(req, sqe, true);
5189 5190 5191
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
5192
	case IORING_OP_WRITE:
5193
		ret = io_write_prep(req, sqe, true);
5194
		break;
5195
	case IORING_OP_POLL_ADD:
5196
		ret = io_poll_add_prep(req, sqe);
5197 5198
		break;
	case IORING_OP_POLL_REMOVE:
5199
		ret = io_poll_remove_prep(req, sqe);
5200
		break;
5201
	case IORING_OP_FSYNC:
5202
		ret = io_prep_fsync(req, sqe);
5203 5204
		break;
	case IORING_OP_SYNC_FILE_RANGE:
5205
		ret = io_prep_sfr(req, sqe);
5206
		break;
5207
	case IORING_OP_SENDMSG:
5208
	case IORING_OP_SEND:
5209
		ret = io_sendmsg_prep(req, sqe);
5210 5211
		break;
	case IORING_OP_RECVMSG:
5212
	case IORING_OP_RECV:
5213
		ret = io_recvmsg_prep(req, sqe);
5214
		break;
5215
	case IORING_OP_CONNECT:
5216
		ret = io_connect_prep(req, sqe);
5217
		break;
5218
	case IORING_OP_TIMEOUT:
5219
		ret = io_timeout_prep(req, sqe, false);
5220
		break;
5221
	case IORING_OP_TIMEOUT_REMOVE:
5222
		ret = io_timeout_remove_prep(req, sqe);
5223
		break;
5224
	case IORING_OP_ASYNC_CANCEL:
5225
		ret = io_async_cancel_prep(req, sqe);
5226
		break;
5227
	case IORING_OP_LINK_TIMEOUT:
5228
		ret = io_timeout_prep(req, sqe, true);
5229
		break;
5230
	case IORING_OP_ACCEPT:
5231
		ret = io_accept_prep(req, sqe);
5232
		break;
5233 5234 5235
	case IORING_OP_FALLOCATE:
		ret = io_fallocate_prep(req, sqe);
		break;
5236 5237 5238
	case IORING_OP_OPENAT:
		ret = io_openat_prep(req, sqe);
		break;
5239 5240 5241
	case IORING_OP_CLOSE:
		ret = io_close_prep(req, sqe);
		break;
5242 5243 5244
	case IORING_OP_FILES_UPDATE:
		ret = io_files_update_prep(req, sqe);
		break;
5245 5246 5247
	case IORING_OP_STATX:
		ret = io_statx_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5248 5249 5250
	case IORING_OP_FADVISE:
		ret = io_fadvise_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5251 5252 5253
	case IORING_OP_MADVISE:
		ret = io_madvise_prep(req, sqe);
		break;
5254 5255 5256
	case IORING_OP_OPENAT2:
		ret = io_openat2_prep(req, sqe);
		break;
5257 5258 5259
	case IORING_OP_EPOLL_CTL:
		ret = io_epoll_ctl_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5260 5261 5262
	case IORING_OP_SPLICE:
		ret = io_splice_prep(req, sqe);
		break;
5263 5264 5265
	case IORING_OP_PROVIDE_BUFFERS:
		ret = io_provide_buffers_prep(req, sqe);
		break;
5266 5267 5268
	case IORING_OP_REMOVE_BUFFERS:
		ret = io_remove_buffers_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5269 5270 5271
	case IORING_OP_TEE:
		ret = io_tee_prep(req, sqe);
		break;
5272
	default:
J
Jens Axboe 已提交
5273 5274 5275
		printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
				req->opcode);
		ret = -EINVAL;
5276
		break;
5277 5278
	}

5279
	return ret;
5280 5281
}

5282
static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5283
{
5284
	struct io_ring_ctx *ctx = req->ctx;
5285
	int ret;
5286

B
Bob Liu 已提交
5287
	/* Still need defer if there is pending req in defer list. */
5288
	if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))
5289 5290
		return 0;

5291 5292 5293 5294 5295 5296 5297
	if (!req->io) {
		if (io_alloc_async_ctx(req))
			return -EAGAIN;
		ret = io_req_defer_prep(req, sqe);
		if (ret < 0)
			return ret;
	}
5298

5299
	spin_lock_irq(&ctx->completion_lock);
B
Bob Liu 已提交
5300
	if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
5301 5302 5303 5304
		spin_unlock_irq(&ctx->completion_lock);
		return 0;
	}

5305
	trace_io_uring_defer(ctx, req, req->user_data);
5306 5307 5308 5309 5310
	list_add_tail(&req->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

P
Pavel Begunkov 已提交
5311 5312 5313 5314 5315 5316 5317 5318
static void io_cleanup_req(struct io_kiocb *req)
{
	struct io_async_ctx *io = req->io;

	switch (req->opcode) {
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
5319 5320 5321
		if (req->flags & REQ_F_BUFFER_SELECTED)
			kfree((void *)(unsigned long)req->rw.addr);
		/* fallthrough */
P
Pavel Begunkov 已提交
5322 5323 5324 5325 5326 5327 5328
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		if (io->rw.iov != io->rw.fast_iov)
			kfree(io->rw.iov);
		break;
	case IORING_OP_RECVMSG:
5329 5330 5331 5332
		if (req->flags & REQ_F_BUFFER_SELECTED)
			kfree(req->sr_msg.kbuf);
		/* fallthrough */
	case IORING_OP_SENDMSG:
P
Pavel Begunkov 已提交
5333 5334 5335
		if (io->msg.iov != io->msg.fast_iov)
			kfree(io->msg.iov);
		break;
5336 5337 5338 5339
	case IORING_OP_RECV:
		if (req->flags & REQ_F_BUFFER_SELECTED)
			kfree(req->sr_msg.kbuf);
		break;
5340 5341 5342
	case IORING_OP_OPENAT:
	case IORING_OP_OPENAT2:
		break;
P
Pavel Begunkov 已提交
5343
	case IORING_OP_SPLICE:
P
Pavel Begunkov 已提交
5344
	case IORING_OP_TEE:
P
Pavel Begunkov 已提交
5345 5346 5347
		io_put_file(req, req->splice.file_in,
			    (req->splice.flags & SPLICE_F_FD_IN_FIXED));
		break;
P
Pavel Begunkov 已提交
5348 5349 5350 5351 5352
	}

	req->flags &= ~REQ_F_NEED_CLEANUP;
}

5353
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5354
			bool force_nonblock)
J
Jens Axboe 已提交
5355
{
5356
	struct io_ring_ctx *ctx = req->ctx;
5357
	int ret;
J
Jens Axboe 已提交
5358

5359
	switch (req->opcode) {
J
Jens Axboe 已提交
5360
	case IORING_OP_NOP:
5361
		ret = io_nop(req);
J
Jens Axboe 已提交
5362 5363
		break;
	case IORING_OP_READV:
5364
	case IORING_OP_READ_FIXED:
5365
	case IORING_OP_READ:
5366 5367 5368 5369 5370
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5371
		ret = io_read(req, force_nonblock);
5372
		break;
5373
	case IORING_OP_WRITEV:
5374
	case IORING_OP_WRITE_FIXED:
5375
	case IORING_OP_WRITE:
5376 5377 5378 5379 5380
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5381
		ret = io_write(req, force_nonblock);
J
Jens Axboe 已提交
5382
		break;
C
Christoph Hellwig 已提交
5383
	case IORING_OP_FSYNC:
5384 5385 5386 5387 5388
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
5389
		ret = io_fsync(req, force_nonblock);
C
Christoph Hellwig 已提交
5390
		break;
5391
	case IORING_OP_POLL_ADD:
5392 5393 5394 5395 5396
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
5397
		ret = io_poll_add(req);
5398 5399
		break;
	case IORING_OP_POLL_REMOVE:
5400 5401 5402 5403 5404
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
5405
		ret = io_poll_remove(req);
5406
		break;
5407
	case IORING_OP_SYNC_FILE_RANGE:
5408 5409 5410 5411 5412
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
5413
		ret = io_sync_file_range(req, force_nonblock);
5414
		break;
J
Jens Axboe 已提交
5415
	case IORING_OP_SENDMSG:
5416
	case IORING_OP_SEND:
5417 5418 5419 5420 5421
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
5422
		if (req->opcode == IORING_OP_SENDMSG)
5423
			ret = io_sendmsg(req, force_nonblock);
5424
		else
5425
			ret = io_send(req, force_nonblock);
J
Jens Axboe 已提交
5426
		break;
J
Jens Axboe 已提交
5427
	case IORING_OP_RECVMSG:
5428
	case IORING_OP_RECV:
5429 5430 5431 5432 5433
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
5434
		if (req->opcode == IORING_OP_RECVMSG)
5435
			ret = io_recvmsg(req, force_nonblock);
5436
		else
5437
			ret = io_recv(req, force_nonblock);
J
Jens Axboe 已提交
5438
		break;
J
Jens Axboe 已提交
5439
	case IORING_OP_TIMEOUT:
5440 5441 5442 5443 5444
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
5445
		ret = io_timeout(req);
J
Jens Axboe 已提交
5446
		break;
5447
	case IORING_OP_TIMEOUT_REMOVE:
5448 5449 5450 5451 5452
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
5453
		ret = io_timeout_remove(req);
5454
		break;
5455
	case IORING_OP_ACCEPT:
5456 5457 5458 5459 5460
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
5461
		ret = io_accept(req, force_nonblock);
5462
		break;
5463
	case IORING_OP_CONNECT:
5464 5465 5466 5467 5468
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
5469
		ret = io_connect(req, force_nonblock);
5470
		break;
5471
	case IORING_OP_ASYNC_CANCEL:
5472 5473 5474 5475 5476
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
5477
		ret = io_async_cancel(req);
5478
		break;
5479 5480 5481 5482 5483 5484
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
5485
		ret = io_fallocate(req, force_nonblock);
5486
		break;
5487 5488 5489 5490 5491 5492
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
5493
		ret = io_openat(req, force_nonblock);
5494
		break;
5495 5496 5497 5498 5499 5500
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
5501
		ret = io_close(req, force_nonblock);
5502
		break;
5503 5504 5505 5506 5507 5508 5509 5510
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_files_update(req, force_nonblock);
		break;
5511 5512 5513 5514 5515 5516
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
5517
		ret = io_statx(req, force_nonblock);
5518
		break;
J
Jens Axboe 已提交
5519 5520 5521 5522 5523 5524
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
5525
		ret = io_fadvise(req, force_nonblock);
J
Jens Axboe 已提交
5526
		break;
J
Jens Axboe 已提交
5527 5528 5529 5530 5531 5532
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
5533
		ret = io_madvise(req, force_nonblock);
J
Jens Axboe 已提交
5534
		break;
5535 5536 5537 5538 5539 5540
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
5541
		ret = io_openat2(req, force_nonblock);
5542
		break;
5543 5544 5545 5546 5547 5548
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
5549
		ret = io_epoll_ctl(req, force_nonblock);
5550
		break;
P
Pavel Begunkov 已提交
5551 5552 5553 5554 5555 5556
	case IORING_OP_SPLICE:
		if (sqe) {
			ret = io_splice_prep(req, sqe);
			if (ret < 0)
				break;
		}
5557
		ret = io_splice(req, force_nonblock);
P
Pavel Begunkov 已提交
5558
		break;
5559 5560 5561 5562 5563 5564 5565 5566
	case IORING_OP_PROVIDE_BUFFERS:
		if (sqe) {
			ret = io_provide_buffers_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_provide_buffers(req, force_nonblock);
		break;
5567 5568 5569 5570 5571 5572 5573
	case IORING_OP_REMOVE_BUFFERS:
		if (sqe) {
			ret = io_remove_buffers_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_remove_buffers(req, force_nonblock);
5574
		break;
P
Pavel Begunkov 已提交
5575 5576 5577 5578 5579 5580 5581 5582
	case IORING_OP_TEE:
		if (sqe) {
			ret = io_tee_prep(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_tee(req, force_nonblock);
		break;
J
Jens Axboe 已提交
5583 5584 5585 5586 5587
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
5588 5589 5590
	if (ret)
		return ret;

5591 5592
	/* If the op doesn't have a file, we're not polling for it */
	if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
5593 5594
		const bool in_async = io_wq_current_is_worker();

J
Jens Axboe 已提交
5595
		if (req->result == -EAGAIN)
J
Jens Axboe 已提交
5596 5597
			return -EAGAIN;

5598 5599 5600 5601
		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

J
Jens Axboe 已提交
5602
		io_iopoll_req_issued(req);
5603 5604 5605

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
5606 5607 5608
	}

	return 0;
J
Jens Axboe 已提交
5609 5610
}

5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622
static void io_arm_async_linked_timeout(struct io_kiocb *req)
{
	struct io_kiocb *link;

	/* link head's timeout is queued in io_queue_async_work() */
	if (!(req->flags & REQ_F_QUEUE_TIMEOUT))
		return;

	link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	io_queue_linked_timeout(link);
}

5623
static void io_wq_submit_work(struct io_wq_work **workptr)
J
Jens Axboe 已提交
5624
{
5625
	struct io_wq_work *work = *workptr;
J
Jens Axboe 已提交
5626
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5627
	int ret = 0;
J
Jens Axboe 已提交
5628

5629 5630
	io_arm_async_linked_timeout(req);

5631 5632 5633
	/* if NO_CANCEL is set, we must still run the work */
	if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
				IO_WQ_WORK_CANCEL) {
5634
		ret = -ECANCELED;
5635
	}
5636

5637 5638
	if (!ret) {
		do {
5639
			ret = io_issue_sqe(req, NULL, false);
5640 5641 5642 5643 5644 5645 5646 5647 5648 5649
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
5650

5651
	if (ret) {
J
Jens Axboe 已提交
5652
		req_set_fail_links(req);
5653
		io_cqring_add_event(req, ret);
5654
		io_put_req(req);
5655
	}
J
Jens Axboe 已提交
5656

5657
	io_steal_work(req, workptr);
J
Jens Axboe 已提交
5658 5659
}

5660 5661 5662 5663 5664
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

5665
	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
5666
	return table->files[index & IORING_FILE_TABLE_MASK];
5667 5668
}

5669 5670
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
			int fd, struct file **out_file, bool fixed)
J
Jens Axboe 已提交
5671
{
5672
	struct io_ring_ctx *ctx = req->ctx;
5673
	struct file *file;
J
Jens Axboe 已提交
5674

5675
	if (fixed) {
5676
		if (unlikely(!ctx->file_data ||
J
Jens Axboe 已提交
5677 5678
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
5679
		fd = array_index_nospec(fd, ctx->nr_user_files);
5680
		file = io_file_from_index(ctx, fd);
5681 5682 5683 5684
		if (file) {
			req->fixed_file_refs = ctx->file_data->cur_refs;
			percpu_ref_get(req->fixed_file_refs);
		}
J
Jens Axboe 已提交
5685
	} else {
5686
		trace_io_uring_file_get(ctx, fd);
5687
		file = __io_file_get(state, fd);
J
Jens Axboe 已提交
5688 5689
	}

5690 5691 5692 5693 5694
	if (file || io_op_defs[req->opcode].needs_file_no_error) {
		*out_file = file;
		return 0;
	}
	return -EBADF;
J
Jens Axboe 已提交
5695 5696
}

5697
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
5698
			   int fd)
5699 5700 5701
{
	bool fixed;

5702
	fixed = (req->flags & REQ_F_FIXED_FILE) != 0;
5703
	if (unlikely(!fixed && io_async_submit(req->ctx)))
5704 5705 5706 5707 5708
		return -EBADF;

	return io_file_get(state, req, fd, &req->file, fixed);
}

5709
static int io_grab_files(struct io_kiocb *req)
5710 5711
{
	int ret = -EBADF;
5712
	struct io_ring_ctx *ctx = req->ctx;
5713

5714
	if (req->work.files || (req->flags & REQ_F_NO_FILE_TABLE))
5715
		return 0;
P
Pavel Begunkov 已提交
5716
	if (!ctx->ring_file)
5717 5718
		return -EBADF;

5719 5720 5721 5722 5723 5724 5725 5726
	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
P
Pavel Begunkov 已提交
5727
	if (fcheck(ctx->ring_fd) == ctx->ring_file) {
5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

5739
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
J
Jens Axboe 已提交
5740
{
5741 5742 5743
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
5744 5745 5746 5747 5748 5749 5750 5751 5752 5753
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
5754 5755 5756
	if (!list_empty(&req->link_list)) {
		prev = list_entry(req->link_list.prev, struct io_kiocb,
				  link_list);
5757
		if (refcount_inc_not_zero(&prev->refs)) {
5758
			list_del_init(&req->link_list);
5759 5760
			prev->flags &= ~REQ_F_LINK_TIMEOUT;
		} else
5761
			prev = NULL;
5762 5763 5764 5765 5766
	}

	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
J
Jens Axboe 已提交
5767
		req_set_fail_links(prev);
5768
		io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
5769
		io_put_req(prev);
5770 5771 5772
	} else {
		io_cqring_add_event(req, -ETIME);
		io_put_req(req);
5773 5774 5775 5776
	}
	return HRTIMER_NORESTART;
}

5777
static void io_queue_linked_timeout(struct io_kiocb *req)
5778
{
5779
	struct io_ring_ctx *ctx = req->ctx;
5780

5781 5782 5783 5784 5785
	/*
	 * If the list is now empty, then our linked request finished before
	 * we got a chance to setup the timer
	 */
	spin_lock_irq(&ctx->completion_lock);
5786
	if (!list_empty(&req->link_list)) {
5787
		struct io_timeout_data *data = &req->io->timeout;
5788

5789 5790 5791
		data->timer.function = io_link_timeout_fn;
		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
				data->mode);
5792
	}
5793
	spin_unlock_irq(&ctx->completion_lock);
5794 5795

	/* drop submission reference */
5796 5797
	io_put_req(req);
}
5798

5799
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
5800 5801 5802
{
	struct io_kiocb *nxt;

5803
	if (!(req->flags & REQ_F_LINK_HEAD))
5804
		return NULL;
5805 5806 5807
	/* for polled retry, if flag is set, we already went through here */
	if (req->flags & REQ_F_POLLED)
		return NULL;
5808

5809 5810
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
					link_list);
5811
	if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
5812
		return NULL;
5813

5814 5815
	req->flags |= REQ_F_LINK_TIMEOUT;
	return nxt;
5816 5817
}

5818
static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
5819
{
5820
	struct io_kiocb *linked_timeout;
5821
	struct io_kiocb *nxt;
5822
	const struct cred *old_creds = NULL;
5823
	int ret;
J
Jens Axboe 已提交
5824

5825 5826 5827
again:
	linked_timeout = io_prep_linked_timeout(req);

5828 5829
	if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
	    req->work.creds != current_cred()) {
5830 5831 5832 5833 5834 5835 5836 5837
		if (old_creds)
			revert_creds(old_creds);
		if (old_creds == req->work.creds)
			old_creds = NULL; /* restored original creds */
		else
			old_creds = override_creds(req->work.creds);
	}

5838
	ret = io_issue_sqe(req, sqe, true);
5839 5840 5841 5842 5843

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
5844
	if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
5845 5846 5847
		if (io_arm_poll_handler(req)) {
			if (linked_timeout)
				io_queue_linked_timeout(linked_timeout);
5848
			goto exit;
5849
		}
5850
punt:
5851 5852
		io_req_init_async(req);

5853
		if (io_op_defs[req->opcode].file_table) {
5854 5855 5856
			ret = io_grab_files(req);
			if (ret)
				goto err;
J
Jens Axboe 已提交
5857
		}
5858 5859 5860 5861 5862 5863

		/*
		 * Queued up for async execution, worker will release
		 * submit reference when the iocb is actually submitted.
		 */
		io_queue_async_work(req);
5864
		goto exit;
J
Jens Axboe 已提交
5865
	}
5866

5867
err:
5868
	nxt = NULL;
5869
	/* drop submission reference */
5870
	io_put_req_find_next(req, &nxt);
5871

5872
	if (linked_timeout) {
5873
		if (!ret)
5874
			io_queue_linked_timeout(linked_timeout);
5875
		else
5876
			io_put_req(linked_timeout);
5877 5878
	}

5879
	/* and drop final reference, if we failed */
J
Jens Axboe 已提交
5880
	if (ret) {
5881
		io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
5882
		req_set_fail_links(req);
5883
		io_put_req(req);
J
Jens Axboe 已提交
5884
	}
5885 5886
	if (nxt) {
		req = nxt;
5887 5888 5889

		if (req->flags & REQ_F_FORCE_ASYNC)
			goto punt;
5890 5891
		goto again;
	}
5892
exit:
5893 5894
	if (old_creds)
		revert_creds(old_creds);
J
Jens Axboe 已提交
5895 5896
}

5897
static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5898 5899 5900
{
	int ret;

5901
	ret = io_req_defer(req, sqe);
5902 5903
	if (ret) {
		if (ret != -EIOCBQUEUED) {
5904
fail_req:
5905
			io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
5906
			req_set_fail_links(req);
5907
			io_double_put_req(req);
5908
		}
5909
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
5910 5911 5912 5913 5914 5915 5916 5917 5918
		if (!req->io) {
			ret = -EAGAIN;
			if (io_alloc_async_ctx(req))
				goto fail_req;
			ret = io_req_defer_prep(req, sqe);
			if (unlikely(ret < 0))
				goto fail_req;
		}

J
Jens Axboe 已提交
5919 5920 5921 5922 5923 5924 5925
		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
5926
		__io_queue_sqe(req, sqe);
J
Jens Axboe 已提交
5927
	}
5928 5929
}

5930
static inline void io_queue_link_head(struct io_kiocb *req)
5931
{
5932
	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
5933 5934 5935
		io_cqring_add_event(req, -ECANCELED);
		io_double_put_req(req);
	} else
5936
		io_queue_sqe(req, NULL);
5937 5938
}

5939
static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5940
			 struct io_kiocb **link)
J
Jens Axboe 已提交
5941
{
5942
	struct io_ring_ctx *ctx = req->ctx;
5943
	int ret;
J
Jens Axboe 已提交
5944 5945 5946 5947 5948 5949 5950 5951 5952

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
P
Pavel Begunkov 已提交
5953
		struct io_kiocb *head = *link;
J
Jens Axboe 已提交
5954

5955 5956 5957 5958 5959 5960 5961
		/*
		 * Taking sequential execution of a link, draining both sides
		 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
		 * requests in the link. So, it drains the head and the
		 * next after the link request. The last one is done via
		 * drain_next flag to persist the effect across calls.
		 */
5962
		if (req->flags & REQ_F_IO_DRAIN) {
5963 5964 5965
			head->flags |= REQ_F_IO_DRAIN;
			ctx->drain_next = 1;
		}
5966 5967
		if (io_alloc_async_ctx(req))
			return -EAGAIN;
J
Jens Axboe 已提交
5968

5969
		ret = io_req_defer_prep(req, sqe);
5970
		if (ret) {
J
Jens Axboe 已提交
5971
			/* fail even hard links since we don't submit */
P
Pavel Begunkov 已提交
5972
			head->flags |= REQ_F_FAIL_LINK;
5973
			return ret;
5974
		}
P
Pavel Begunkov 已提交
5975 5976
		trace_io_uring_link(ctx, req, head);
		list_add_tail(&req->link_list, &head->link_list);
5977 5978

		/* last request of a link, enqueue the link */
5979
		if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
5980 5981 5982
			io_queue_link_head(head);
			*link = NULL;
		}
J
Jens Axboe 已提交
5983
	} else {
5984 5985
		if (unlikely(ctx->drain_next)) {
			req->flags |= REQ_F_IO_DRAIN;
5986
			ctx->drain_next = 0;
5987
		}
5988
		if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
5989
			req->flags |= REQ_F_LINK_HEAD;
5990
			INIT_LIST_HEAD(&req->link_list);
5991

5992 5993 5994
			if (io_alloc_async_ctx(req))
				return -EAGAIN;

5995 5996 5997 5998 5999 6000 6001
			ret = io_req_defer_prep(req, sqe);
			if (ret)
				req->flags |= REQ_F_FAIL_LINK;
			*link = req;
		} else {
			io_queue_sqe(req, sqe);
		}
J
Jens Axboe 已提交
6002
	}
6003

6004
	return 0;
J
Jens Axboe 已提交
6005 6006
}

6007 6008 6009 6010 6011 6012
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
	blk_finish_plug(&state->plug);
P
Pavel Begunkov 已提交
6013
	io_state_file_put(state);
J
Jens Axboe 已提交
6014
	if (state->free_reqs)
6015
		kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
6016 6017 6018 6019 6020 6021
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
6022
				  unsigned int max_ios)
6023 6024
{
	blk_start_plug(&state->plug);
6025 6026 6027
#ifdef CONFIG_BLOCK
	state->plug.nowait = true;
#endif
J
Jens Axboe 已提交
6028
	state->free_reqs = 0;
6029 6030 6031 6032
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
6033 6034
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
6035
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
6036

6037 6038 6039 6040 6041 6042
	/*
	 * Ensure any loads from the SQEs are done at this point,
	 * since once we write the new head, the application could
	 * write new data to them.
	 */
	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
6043 6044 6045
}

/*
6046
 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
J
Jens Axboe 已提交
6047 6048 6049 6050 6051 6052
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
6053
static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
6054
{
6055
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
6056 6057 6058 6059 6060 6061 6062 6063 6064 6065
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
6066
	head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
6067 6068
	if (likely(head < ctx->sq_entries))
		return &ctx->sq_sqes[head];
J
Jens Axboe 已提交
6069 6070

	/* drop invalid entries */
6071
	ctx->cached_sq_dropped++;
6072
	WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
6073 6074 6075 6076 6077 6078
	return NULL;
}

static inline void io_consume_sqe(struct io_ring_ctx *ctx)
{
	ctx->cached_sq_head++;
J
Jens Axboe 已提交
6079 6080
}

6081 6082 6083 6084 6085 6086
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK|	\
				IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
				IOSQE_BUFFER_SELECT)

static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
		       const struct io_uring_sqe *sqe,
6087
		       struct io_submit_state *state)
6088
{
6089
	unsigned int sqe_flags;
6090
	int id;
6091

6092 6093 6094 6095 6096
	/*
	 * All io need record the previous position, if LINK vs DARIN,
	 * it can be used to mark the position of the first IO in the
	 * link list.
	 */
6097
	req->sequence = ctx->cached_sq_head - ctx->cached_sq_dropped;
6098 6099 6100 6101 6102 6103 6104 6105
	req->opcode = READ_ONCE(sqe->opcode);
	req->user_data = READ_ONCE(sqe->user_data);
	req->io = NULL;
	req->file = NULL;
	req->ctx = ctx;
	req->flags = 0;
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
P
Pavel Begunkov 已提交
6106
	req->task = current;
6107
	req->result = 0;
6108 6109 6110 6111

	if (unlikely(req->opcode >= IORING_OP_LAST))
		return -EINVAL;

6112 6113
	if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
		return -EFAULT;
6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125

	sqe_flags = READ_ONCE(sqe->flags);
	/* enforce forwards compatibility on users */
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
		return -EINVAL;

	if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
	    !io_op_defs[req->opcode].buffer_select)
		return -EOPNOTSUPP;

	id = READ_ONCE(sqe->personality);
	if (id) {
6126
		io_req_init_async(req);
6127 6128 6129 6130 6131 6132 6133
		req->work.creds = idr_find(&ctx->personality_idr, id);
		if (unlikely(!req->work.creds))
			return -EINVAL;
		get_cred(req->work.creds);
	}

	/* same numerical values with corresponding REQ_F_*, safe to copy */
6134
	req->flags |= sqe_flags;
6135

6136 6137 6138 6139
	if (!io_op_defs[req->opcode].needs_file)
		return 0;

	return io_req_set_file(state, req, READ_ONCE(sqe->fd));
6140 6141
}

6142
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
6143
			  struct file *ring_file, int ring_fd)
J
Jens Axboe 已提交
6144
{
6145
	struct io_submit_state state;
J
Jens Axboe 已提交
6146 6147
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
J
Jens Axboe 已提交
6148

6149
	/* if we have a backlog and couldn't flush it all, return BUSY */
6150 6151 6152 6153 6154
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}
J
Jens Axboe 已提交
6155

6156 6157
	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
6158

6159 6160
	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;
J
Jens Axboe 已提交
6161

6162
	io_submit_state_start(&state, nr);
J
Jens Axboe 已提交
6163

P
Pavel Begunkov 已提交
6164 6165 6166
	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

J
Jens Axboe 已提交
6167
	for (i = 0; i < nr; i++) {
6168
		const struct io_uring_sqe *sqe;
6169
		struct io_kiocb *req;
6170
		int err;
6171

6172 6173 6174 6175 6176
		sqe = io_get_sqe(ctx);
		if (unlikely(!sqe)) {
			io_consume_sqe(ctx);
			break;
		}
6177
		req = io_alloc_req(ctx, &state);
6178 6179 6180
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
6181
			break;
6182
		}
6183

6184
		err = io_init_req(ctx, req, sqe, &state);
6185
		io_consume_sqe(ctx);
6186 6187 6188
		/* will complete beyond this point, count as submitted */
		submitted++;

6189
		if (unlikely(err)) {
6190 6191
fail_req:
			io_cqring_add_event(req, err);
6192
			io_double_put_req(req);
6193 6194
			break;
		}
6195

6196
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6197
						true, io_async_submit(ctx));
6198
		err = io_submit_sqe(req, sqe, &link);
6199 6200
		if (err)
			goto fail_req;
J
Jens Axboe 已提交
6201 6202
	}

6203 6204 6205 6206 6207
	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
J
Jens Axboe 已提交
6208
	if (link)
6209
		io_queue_link_head(link);
6210
	io_submit_state_end(&state);
J
Jens Axboe 已提交
6211

6212 6213 6214
	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

J
Jens Axboe 已提交
6215 6216 6217 6218 6219 6220
	return submitted;
}

static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
6221
	const struct cred *old_cred;
J
Jens Axboe 已提交
6222 6223
	DEFINE_WAIT(wait);
	unsigned long timeout;
6224
	int ret = 0;
J
Jens Axboe 已提交
6225

6226
	complete(&ctx->sq_thread_comp);
6227

6228
	old_cred = override_creds(ctx->creds);
J
Jens Axboe 已提交
6229

6230
	timeout = jiffies + ctx->sq_thread_idle;
6231
	while (!kthread_should_park()) {
6232
		unsigned int to_submit;
J
Jens Axboe 已提交
6233

6234
		if (!list_empty(&ctx->poll_list)) {
J
Jens Axboe 已提交
6235 6236
			unsigned nr_events = 0;

6237 6238 6239 6240
			mutex_lock(&ctx->uring_lock);
			if (!list_empty(&ctx->poll_list))
				io_iopoll_getevents(ctx, &nr_events, 0);
			else
J
Jens Axboe 已提交
6241
				timeout = jiffies + ctx->sq_thread_idle;
6242
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
6243 6244
		}

6245
		to_submit = io_sqring_entries(ctx);
6246 6247 6248 6249 6250 6251

		/*
		 * If submit got -EBUSY, flag us as needing the application
		 * to enter the kernel to reap and flush events.
		 */
		if (!to_submit || ret == -EBUSY) {
6252 6253 6254 6255 6256 6257
			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
6258
			io_sq_thread_drop_mm(ctx);
6259

J
Jens Axboe 已提交
6260 6261 6262
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
6263 6264 6265
			 * to sleep. The exception is if we got EBUSY doing
			 * more IO, we should wait for the application to
			 * reap events and wake us up.
J
Jens Axboe 已提交
6266
			 */
6267
			if (!list_empty(&ctx->poll_list) ||
6268 6269
			    (!time_after(jiffies, timeout) && ret != -EBUSY &&
			    !percpu_ref_is_dying(&ctx->refs))) {
6270 6271
				if (current->task_works)
					task_work_run();
6272
				cond_resched();
J
Jens Axboe 已提交
6273 6274 6275 6276 6277 6278
				continue;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);

6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291
			/*
			 * While doing polled IO, before going to sleep, we need
			 * to check if there are new reqs added to poll_list, it
			 * is because reqs may have been punted to io worker and
			 * will be added to poll_list later, hence check the
			 * poll_list again.
			 */
			if ((ctx->flags & IORING_SETUP_IOPOLL) &&
			    !list_empty_careful(&ctx->poll_list)) {
				finish_wait(&ctx->sqo_wait, &wait);
				continue;
			}

J
Jens Axboe 已提交
6292
			/* Tell userspace we may need a wakeup call */
6293
			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
6294 6295
			/* make sure to read SQ tail after writing flags */
			smp_mb();
J
Jens Axboe 已提交
6296

6297
			to_submit = io_sqring_entries(ctx);
6298
			if (!to_submit || ret == -EBUSY) {
6299
				if (kthread_should_park()) {
J
Jens Axboe 已提交
6300 6301 6302
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
6303 6304
				if (current->task_works) {
					task_work_run();
6305
					finish_wait(&ctx->sqo_wait, &wait);
6306 6307
					continue;
				}
J
Jens Axboe 已提交
6308 6309 6310 6311 6312
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

6313
				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6314
				ret = 0;
J
Jens Axboe 已提交
6315 6316 6317 6318
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

6319
			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
6320 6321
		}

6322
		mutex_lock(&ctx->uring_lock);
6323 6324
		if (likely(!percpu_ref_is_dying(&ctx->refs)))
			ret = io_submit_sqes(ctx, to_submit, NULL, -1);
6325
		mutex_unlock(&ctx->uring_lock);
6326
		timeout = jiffies + ctx->sq_thread_idle;
J
Jens Axboe 已提交
6327 6328
	}

6329 6330 6331
	if (current->task_works)
		task_work_run();

6332
	io_sq_thread_drop_mm(ctx);
6333
	revert_creds(old_cred);
6334

6335
	kthread_parkme();
6336

J
Jens Axboe 已提交
6337 6338 6339
	return 0;
}

6340 6341 6342 6343 6344 6345 6346
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

6347
static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
6348 6349 6350 6351
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
6352
	 * Wake up if we have enough events, or if a timeout occurred since we
6353 6354 6355
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
6356
	return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
6357 6358 6359 6360 6361 6362 6363 6364 6365
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

6366 6367
	/* use noflush == true, as we can't safely rely on locking context */
	if (!io_should_wake(iowq, true))
6368 6369 6370 6371 6372
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
6373 6374 6375 6376 6377 6378 6379
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
6380 6381 6382 6383 6384 6385 6386 6387 6388
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
6389
	struct io_rings *rings = ctx->rings;
6390
	int ret = 0;
J
Jens Axboe 已提交
6391

6392 6393 6394 6395 6396 6397 6398
	do {
		if (io_cqring_events(ctx, false) >= min_events)
			return 0;
		if (!current->task_works)
			break;
		task_work_run();
	} while (1);
J
Jens Axboe 已提交
6399 6400

	if (sig) {
6401 6402 6403
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
6404
						      sigsz);
6405 6406
		else
#endif
6407
			ret = set_user_sigmask(sig, sigsz);
6408

J
Jens Axboe 已提交
6409 6410 6411 6412
		if (ret)
			return ret;
	}

6413
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
6414
	trace_io_uring_cqring_wait(ctx, min_events);
6415 6416 6417
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
6418 6419
		if (current->task_works)
			task_work_run();
6420
		if (io_should_wake(&iowq, false))
6421 6422 6423
			break;
		schedule();
		if (signal_pending(current)) {
6424
			ret = -EINTR;
6425 6426 6427 6428 6429
			break;
		}
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

6430
	restore_saved_sigmask_unless(ret == -EINTR);
J
Jens Axboe 已提交
6431

6432
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
6433 6434
}

J
Jens Axboe 已提交
6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

6448 6449 6450 6451 6452 6453 6454
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
6455 6456 6457
#endif
}

6458 6459 6460 6461 6462 6463 6464 6465
static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}

J
Jens Axboe 已提交
6466 6467
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
6468
	struct fixed_file_data *data = ctx->file_data;
6469
	struct fixed_file_ref_node *ref_node = NULL;
6470 6471
	unsigned nr_tables, i;

6472
	if (!data)
J
Jens Axboe 已提交
6473 6474
		return -ENXIO;

6475
	spin_lock(&data->lock);
6476 6477 6478
	if (!list_empty(&data->ref_list))
		ref_node = list_first_entry(&data->ref_list,
				struct fixed_file_ref_node, node);
6479
	spin_unlock(&data->lock);
6480 6481 6482 6483 6484 6485
	if (ref_node)
		percpu_ref_kill(&ref_node->refs);

	percpu_ref_kill(&data->refs);

	/* wait for all refs nodes to complete */
6486
	flush_delayed_work(&ctx->file_put_work);
6487
	wait_for_completion(&data->done);
6488

J
Jens Axboe 已提交
6489
	__io_sqe_files_unregister(ctx);
6490 6491
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
6492 6493
		kfree(data->table[i].files);
	kfree(data->table);
6494 6495
	percpu_ref_exit(&data->refs);
	kfree(data);
6496
	ctx->file_data = NULL;
J
Jens Axboe 已提交
6497 6498 6499 6500
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
6501 6502 6503
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
6504
		wait_for_completion(&ctx->sq_thread_comp);
6505 6506 6507 6508 6509
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
6510
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
6511 6512 6513 6514 6515
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
6516 6517
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
6518 6519
	io_sq_thread_stop(ctx);

6520 6521 6522
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536
	}
}

#if defined(CONFIG_UNIX)
/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
6537
	int i, nr_files;
J
Jens Axboe 已提交
6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

6551
	nr_files = 0;
J
Jens Axboe 已提交
6552 6553
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
6554 6555 6556
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
6557
			continue;
6558
		fpl->fp[nr_files] = get_file(file);
6559 6560
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
6561 6562
	}

6563 6564 6565 6566
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
6567
		skb->destructor = unix_destruct_scm;
6568 6569
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
6570

6571 6572 6573 6574 6575 6576
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
6607 6608 6609 6610
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

6623 6624 6625 6626 6627 6628
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
6629
		struct fixed_file_table *table = &ctx->file_data->table[i];
6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
6644
		struct fixed_file_table *table = &ctx->file_data->table[i];
6645 6646 6647 6648 6649
		kfree(table->files);
	}
	return 1;
}

6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712
static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
	fput(file);
#endif
}

struct io_file_put {
6713
	struct list_head list;
6714 6715 6716
	struct file *file;
};

6717
static void __io_file_put_work(struct fixed_file_ref_node *ref_node)
6718
{
6719 6720
	struct fixed_file_data *file_data = ref_node->file_data;
	struct io_ring_ctx *ctx = file_data->ctx;
6721
	struct io_file_put *pfile, *tmp;
6722 6723

	list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) {
6724
		list_del(&pfile->list);
6725 6726
		io_ring_file_put(ctx, pfile->file);
		kfree(pfile);
6727
	}
6728

6729 6730 6731
	spin_lock(&file_data->lock);
	list_del(&ref_node->node);
	spin_unlock(&file_data->lock);
6732 6733 6734 6735

	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
	percpu_ref_put(&file_data->refs);
6736
}
6737

6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755
static void io_file_put_work(struct work_struct *work)
{
	struct io_ring_ctx *ctx;
	struct llist_node *node;

	ctx = container_of(work, struct io_ring_ctx, file_put_work.work);
	node = llist_del_all(&ctx->file_put_llist);

	while (node) {
		struct fixed_file_ref_node *ref_node;
		struct llist_node *next = node->next;

		ref_node = llist_entry(node, struct fixed_file_ref_node, llist);
		__io_file_put_work(ref_node);
		node = next;
	}
}

6756
static void io_file_data_ref_zero(struct percpu_ref *ref)
6757
{
6758
	struct fixed_file_ref_node *ref_node;
6759 6760 6761
	struct io_ring_ctx *ctx;
	bool first_add;
	int delay = HZ;
6762

6763
	ref_node = container_of(ref, struct fixed_file_ref_node, refs);
6764
	ctx = ref_node->file_data->ctx;
6765

6766 6767
	if (percpu_ref_is_dying(&ctx->file_data->refs))
		delay = 0;
6768

6769 6770 6771 6772 6773
	first_add = llist_add(&ref_node->llist, &ctx->file_put_llist);
	if (!delay)
		mod_delayed_work(system_wq, &ctx->file_put_work, 0);
	else if (first_add)
		queue_delayed_work(system_wq, &ctx->file_put_work, delay);
6774
}
6775

6776 6777
static struct fixed_file_ref_node *alloc_fixed_file_ref_node(
			struct io_ring_ctx *ctx)
6778
{
6779
	struct fixed_file_ref_node *ref_node;
6780

6781 6782 6783
	ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
	if (!ref_node)
		return ERR_PTR(-ENOMEM);
6784

6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799
	if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero,
			    0, GFP_KERNEL)) {
		kfree(ref_node);
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&ref_node->node);
	INIT_LIST_HEAD(&ref_node->file_list);
	ref_node->file_data = ctx->file_data;
	return ref_node;
}

static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node)
{
	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
6800 6801
}

J
Jens Axboe 已提交
6802 6803 6804 6805
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
6806
	unsigned nr_tables;
6807
	struct file *file;
J
Jens Axboe 已提交
6808 6809
	int fd, ret = 0;
	unsigned i;
6810
	struct fixed_file_ref_node *ref_node;
J
Jens Axboe 已提交
6811

6812
	if (ctx->file_data)
J
Jens Axboe 已提交
6813 6814 6815 6816 6817 6818
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

6819 6820 6821 6822 6823
	ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
	if (!ctx->file_data)
		return -ENOMEM;
	ctx->file_data->ctx = ctx;
	init_completion(&ctx->file_data->done);
6824
	INIT_LIST_HEAD(&ctx->file_data->ref_list);
6825
	spin_lock_init(&ctx->file_data->lock);
6826

6827
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
6828 6829
	ctx->file_data->table = kcalloc(nr_tables,
					sizeof(struct fixed_file_table),
6830
					GFP_KERNEL);
6831 6832 6833
	if (!ctx->file_data->table) {
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
6834
		return -ENOMEM;
6835 6836
	}

6837
	if (percpu_ref_init(&ctx->file_data->refs, io_file_ref_kill,
6838 6839 6840 6841
				PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
6842
		return -ENOMEM;
6843
	}
J
Jens Axboe 已提交
6844

6845
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
6846 6847 6848 6849
		percpu_ref_exit(&ctx->file_data->refs);
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
6850 6851 6852
		return -ENOMEM;
	}

6853
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
6854 6855 6856
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
6857 6858 6859
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
6860 6861 6862 6863 6864
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
6865

6866
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
6867
		index = i & IORING_FILE_TABLE_MASK;
6868
		file = fget(fd);
J
Jens Axboe 已提交
6869 6870

		ret = -EBADF;
6871
		if (!file)
J
Jens Axboe 已提交
6872
			break;
6873

J
Jens Axboe 已提交
6874 6875 6876 6877 6878 6879 6880
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
6881 6882
		if (file->f_op == &io_uring_fops) {
			fput(file);
J
Jens Axboe 已提交
6883 6884 6885
			break;
		}
		ret = 0;
6886
		table->files[index] = file;
J
Jens Axboe 已提交
6887 6888 6889
	}

	if (ret) {
6890 6891 6892 6893 6894 6895
		for (i = 0; i < ctx->nr_user_files; i++) {
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
6896
			kfree(ctx->file_data->table[i].files);
J
Jens Axboe 已提交
6897

6898 6899 6900
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
6901 6902 6903 6904 6905
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
6906
	if (ret) {
J
Jens Axboe 已提交
6907
		io_sqe_files_unregister(ctx);
6908 6909
		return ret;
	}
J
Jens Axboe 已提交
6910

6911 6912 6913 6914 6915 6916 6917
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node)) {
		io_sqe_files_unregister(ctx);
		return PTR_ERR(ref_node);
	}

	ctx->file_data->cur_refs = &ref_node->refs;
6918
	spin_lock(&ctx->file_data->lock);
6919
	list_add(&ref_node->node, &ctx->file_data->ref_list);
6920
	spin_unlock(&ctx->file_data->lock);
6921
	percpu_ref_get(&ctx->file_data->refs);
J
Jens Axboe 已提交
6922 6923 6924
	return ret;
}

6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967
static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

6968
static int io_queue_file_removal(struct fixed_file_data *data,
6969
				 struct file *file)
6970
{
6971
	struct io_file_put *pfile;
6972 6973
	struct percpu_ref *refs = data->cur_refs;
	struct fixed_file_ref_node *ref_node;
6974 6975

	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
6976 6977
	if (!pfile)
		return -ENOMEM;
6978

6979
	ref_node = container_of(refs, struct fixed_file_ref_node, refs);
6980
	pfile->file = file;
6981 6982
	list_add(&pfile->list, &ref_node->file_list);

6983
	return 0;
6984 6985 6986 6987 6988 6989 6990
}

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *up,
				 unsigned nr_args)
{
	struct fixed_file_data *data = ctx->file_data;
6991
	struct fixed_file_ref_node *ref_node;
6992
	struct file *file;
6993 6994 6995
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;
6996
	bool needs_switch = false;
6997

6998
	if (check_add_overflow(up->offset, nr_args, &done))
6999 7000 7001 7002
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

7003 7004 7005 7006
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node))
		return PTR_ERR(ref_node);

7007
	done = 0;
7008
	fds = u64_to_user_ptr(up->fds);
7009
	while (nr_args) {
7010 7011 7012
		struct fixed_file_table *table;
		unsigned index;

7013 7014 7015 7016 7017
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
7018 7019
		i = array_index_nospec(up->offset, ctx->nr_user_files);
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7020 7021
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
7022
			file = io_file_from_index(ctx, index);
7023 7024 7025
			err = io_queue_file_removal(data, file);
			if (err)
				break;
7026
			table->files[index] = NULL;
7027
			needs_switch = true;
7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047
		}
		if (fd != -1) {
			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
7048
			table->files[index] = file;
7049 7050 7051 7052 7053 7054
			err = io_sqe_file_register(ctx, file, i);
			if (err)
				break;
		}
		nr_args--;
		done++;
7055 7056 7057
		up->offset++;
	}

7058 7059
	if (needs_switch) {
		percpu_ref_kill(data->cur_refs);
7060
		spin_lock(&data->lock);
7061 7062
		list_add(&ref_node->node, &data->ref_list);
		data->cur_refs = &ref_node->refs;
7063
		spin_unlock(&data->lock);
7064 7065 7066
		percpu_ref_get(&ctx->file_data->refs);
	} else
		destroy_fixed_file_ref_node(ref_node);
7067 7068 7069

	return done ? done : err;
}
7070

7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086
static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}
7087

7088
static void io_free_work(struct io_wq_work *work)
7089 7090 7091
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

7092
	/* Consider that io_steal_work() relies on this ref */
7093 7094 7095
	io_put_req(req);
}

7096 7097 7098 7099 7100 7101 7102 7103 7104 7105
static int io_init_wq_offload(struct io_ring_ctx *ctx,
			      struct io_uring_params *p)
{
	struct io_wq_data data;
	struct fd f;
	struct io_ring_ctx *ctx_attach;
	unsigned int concurrency;
	int ret = 0;

	data.user = ctx->user;
7106
	data.free_work = io_free_work;
7107
	data.do_work = io_wq_submit_work;
7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142

	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
		/* Do QD, or 4 * CPUS, whatever is smallest */
		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

		ctx->io_wq = io_wq_create(concurrency, &data);
		if (IS_ERR(ctx->io_wq)) {
			ret = PTR_ERR(ctx->io_wq);
			ctx->io_wq = NULL;
		}
		return ret;
	}

	f = fdget(p->wq_fd);
	if (!f.file)
		return -EBADF;

	if (f.file->f_op != &io_uring_fops) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx_attach = f.file->private_data;
	/* @io_wq is protected by holding the fd */
	if (!io_wq_get(ctx_attach->io_wq, &data)) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx->io_wq = ctx_attach->io_wq;
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
7143 7144
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
7145 7146 7147 7148 7149 7150
{
	int ret;

	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

J
Jens Axboe 已提交
7151
	if (ctx->flags & IORING_SETUP_SQPOLL) {
7152 7153 7154 7155
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

7156 7157 7158 7159
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
7160
		if (p->flags & IORING_SETUP_SQ_AFF) {
7161
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
7162

7163
			ret = -EINVAL;
7164 7165
			if (cpu >= nr_cpu_ids)
				goto err;
7166
			if (!cpu_online(cpu))
7167 7168
				goto err;

J
Jens Axboe 已提交
7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

7188 7189
	ret = io_init_wq_offload(ctx, p);
	if (ret)
J
Jens Axboe 已提交
7190 7191 7192 7193
		goto err;

	return 0;
err:
7194
	io_finish_async(ctx);
J
Jens Axboe 已提交
7195 7196 7197 7198 7199
	mmdrop(ctx->sqo_mm);
	ctx->sqo_mm = NULL;
	return ret;
}

7200 7201
static inline void __io_unaccount_mem(struct user_struct *user,
				      unsigned long nr_pages)
J
Jens Axboe 已提交
7202 7203 7204 7205
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

7206 7207
static inline int __io_account_mem(struct user_struct *user,
				   unsigned long nr_pages)
J
Jens Axboe 已提交
7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

7225 7226
static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			     enum io_mem_account acct)
7227
{
7228
	if (ctx->limit_mem)
7229
		__io_unaccount_mem(ctx->user, nr_pages);
7230

7231 7232 7233 7234 7235 7236
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm -= nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7237 7238
}

7239 7240
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			  enum io_mem_account acct)
7241
{
7242 7243 7244 7245 7246 7247 7248 7249
	int ret;

	if (ctx->limit_mem) {
		ret = __io_account_mem(ctx->user, nr_pages);
		if (ret)
			return ret;
	}

7250 7251 7252 7253 7254 7255
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm += nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7256 7257 7258 7259

	return 0;
}

J
Jens Axboe 已提交
7260 7261
static void io_mem_free(void *ptr)
{
7262 7263 7264 7265
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
7266

7267
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	if (sq_offset)
		*sq_offset = off;

	return off;
}

J
Jens Axboe 已提交
7309 7310
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
7311
	size_t pages;
J
Jens Axboe 已提交
7312

7313 7314 7315 7316
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
7317

7318
	return pages;
J
Jens Axboe 已提交
7319 7320
}

7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
7332
			unpin_user_page(imu->bvec[j].bv_page);
7333

7334
		io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
7335
		kvfree(imu->bvec);
7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

7359
		dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
7397
			goto err;
7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

7417
		ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
7418 7419
		if (ret)
			goto err;
7420 7421 7422

		ret = 0;
		if (!pages || nr_pages > got_pages) {
7423 7424
			kvfree(vmas);
			kvfree(pages);
7425
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
7426
						GFP_KERNEL);
7427
			vmas = kvmalloc_array(nr_pages,
7428 7429 7430 7431
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
7432
				io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7433 7434 7435 7436 7437
				goto err;
			}
			got_pages = nr_pages;
		}

7438
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
7439 7440 7441
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
7442
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7443 7444 7445 7446
			goto err;
		}

		ret = 0;
7447
		mmap_read_lock(current->mm);
7448
		pret = pin_user_pages(ubuf, nr_pages,
7449 7450
				      FOLL_WRITE | FOLL_LONGTERM,
				      pages, vmas);
7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
7465
		mmap_read_unlock(current->mm);
7466 7467 7468 7469 7470
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
7471
			if (pret > 0)
7472
				unpin_user_pages(pages, pret);
7473
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7474
			kvfree(imu->bvec);
7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
7497 7498
	kvfree(pages);
	kvfree(vmas);
7499 7500
	return 0;
err:
7501 7502
	kvfree(pages);
	kvfree(vmas);
7503 7504 7505 7506
	io_sqe_buffer_unregister(ctx);
	return ret;
}

7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

7539 7540 7541 7542 7543
static int __io_destroy_buffers(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	struct io_buffer *buf = p;

7544
	__io_remove_buffers(ctx, buf, id, -1U);
7545 7546 7547 7548 7549 7550 7551 7552 7553
	return 0;
}

static void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
	idr_destroy(&ctx->io_buffer_idr);
}

J
Jens Axboe 已提交
7554 7555
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
7556
	io_finish_async(ctx);
7557
	if (ctx->sqo_mm) {
J
Jens Axboe 已提交
7558
		mmdrop(ctx->sqo_mm);
7559 7560
		ctx->sqo_mm = NULL;
	}
J
Jens Axboe 已提交
7561 7562

	io_iopoll_reap_events(ctx);
7563
	io_sqe_buffer_unregister(ctx);
J
Jens Axboe 已提交
7564
	io_sqe_files_unregister(ctx);
7565
	io_eventfd_unregister(ctx);
7566
	io_destroy_buffers(ctx);
J
Jens Axboe 已提交
7567
	idr_destroy(&ctx->personality_idr);
J
Jens Axboe 已提交
7568

J
Jens Axboe 已提交
7569
#if defined(CONFIG_UNIX)
7570 7571
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
7572
		sock_release(ctx->ring_sock);
7573
	}
J
Jens Axboe 已提交
7574 7575
#endif

7576
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
7577 7578 7579
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
7580 7581
	io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
			 ACCT_LOCKED);
J
Jens Axboe 已提交
7582
	free_uid(ctx->user);
7583
	put_cred(ctx->creds);
7584
	kfree(ctx->cancel_hash);
7585
	kmem_cache_free(req_cachep, ctx->fallback_req);
J
Jens Axboe 已提交
7586 7587 7588 7589 7590 7591 7592 7593 7594
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
7595 7596 7597 7598
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
7599
	smp_rmb();
7600 7601
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
7602
		mask |= EPOLLOUT | EPOLLWRNORM;
7603
	if (io_cqring_events(ctx, false))
J
Jens Axboe 已提交
7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626
static int io_remove_personalities(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	const struct cred *cred;

	cred = idr_remove(&ctx->personality_idr, id);
	if (cred)
		put_cred(cred);
	return 0;
}

7627 7628 7629 7630 7631 7632 7633 7634
static void io_ring_exit_work(struct work_struct *work)
{
	struct io_ring_ctx *ctx;

	ctx = container_of(work, struct io_ring_ctx, exit_work);
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);

7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645
	/*
	 * If we're doing polled IO and end up having requests being
	 * submitted async (out-of-line), then completions can come in while
	 * we're waiting for refs to drop. We need to reap these manually,
	 * as nobody else will be looking for them.
	 */
	while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {
		io_iopoll_reap_events(ctx);
		if (ctx->rings)
			io_cqring_overflow_flush(ctx, true);
	}
7646 7647 7648
	io_ring_ctx_free(ctx);
}

J
Jens Axboe 已提交
7649 7650 7651 7652 7653 7654
static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
7655
	io_kill_timeouts(ctx);
7656
	io_poll_remove_all(ctx);
7657 7658 7659 7660

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

J
Jens Axboe 已提交
7661
	io_iopoll_reap_events(ctx);
7662 7663 7664
	/* if we failed setting up the ctx, we might not have any rings */
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);
7665
	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
7666 7667
	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
	queue_work(system_wq, &ctx->exit_work);
J
Jens Axboe 已提交
7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

7679 7680 7681 7682 7683 7684 7685
static bool io_wq_files_match(struct io_wq_work *work, void *data)
{
	struct files_struct *files = data;

	return work->files == files;
}

7686 7687 7688
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
7689 7690 7691 7692 7693 7694
	if (list_empty_careful(&ctx->inflight_list))
		return;

	/* cancel all at once, should be faster than doing it one by one*/
	io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);

7695
	while (!list_empty_careful(&ctx->inflight_list)) {
7696 7697
		struct io_kiocb *cancel_req = NULL, *req;
		DEFINE_WAIT(wait);
7698 7699 7700

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
7701 7702 7703 7704 7705 7706 7707
			if (req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
7708
		}
7709
		if (cancel_req)
7710
			prepare_to_wait(&ctx->inflight_wait, &wait,
7711
						TASK_UNINTERRUPTIBLE);
7712 7713
		spin_unlock_irq(&ctx->inflight_lock);

7714 7715
		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
7716
			break;
7717

7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735
		if (cancel_req->flags & REQ_F_OVERFLOW) {
			spin_lock_irq(&ctx->completion_lock);
			list_del(&cancel_req->list);
			cancel_req->flags &= ~REQ_F_OVERFLOW;
			if (list_empty(&ctx->cq_overflow_list)) {
				clear_bit(0, &ctx->sq_check_overflow);
				clear_bit(0, &ctx->cq_check_overflow);
			}
			spin_unlock_irq(&ctx->completion_lock);

			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));

			/*
			 * Put inflight ref and overflow ref. If that's
			 * all we had, then we're done with this request.
			 */
			if (refcount_sub_and_test(2, &cancel_req->refs)) {
7736
				io_free_req(cancel_req);
7737
				finish_wait(&ctx->inflight_wait, &wait);
7738 7739
				continue;
			}
7740 7741 7742
		} else {
			io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
			io_put_req(cancel_req);
7743 7744
		}

7745
		schedule();
7746
		finish_wait(&ctx->inflight_wait, &wait);
7747 7748 7749
	}
}

7750
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
7751
{
7752 7753
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct task_struct *task = data;
7754

7755
	return req->task == task;
7756 7757
}

7758 7759 7760 7761 7762
static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
7763 7764 7765 7766

	/*
	 * If the task is going away, cancel work it may have pending
	 */
7767 7768
	if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
		io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
7769

7770 7771 7772
	return 0;
}

7773 7774
static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
J
Jens Axboe 已提交
7775 7776
{
	struct io_ring_ctx *ctx = file->private_data;
7777
	loff_t offset = pgoff << PAGE_SHIFT;
J
Jens Axboe 已提交
7778 7779 7780 7781 7782
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
7783 7784
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
7785 7786 7787 7788 7789
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
7790
		return ERR_PTR(-EINVAL);
J
Jens Axboe 已提交
7791 7792 7793
	}

	page = virt_to_head_page(ptr);
7794
	if (sz > page_size(page))
7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810
		return ERR_PTR(-EINVAL);

	return ptr;
}

#ifdef CONFIG_MMU

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);
J
Jens Axboe 已提交
7811 7812 7813 7814 7815

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842
#else /* !CONFIG_MMU */

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
}

static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
{
	return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
}

static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len,
	unsigned long pgoff, unsigned long flags)
{
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, pgoff, len);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	return (unsigned long) ptr;
}

#endif /* !CONFIG_MMU */

J
Jens Axboe 已提交
7843 7844 7845 7846 7847 7848 7849 7850 7851
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

7852 7853 7854
	if (current->task_works)
		task_work_run();

J
Jens Axboe 已提交
7855
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
7871 7872 7873 7874 7875
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
7876
	ret = 0;
J
Jens Axboe 已提交
7877
	if (ctx->flags & IORING_SETUP_SQPOLL) {
7878 7879
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false);
J
Jens Axboe 已提交
7880 7881 7882
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
7883
	} else if (to_submit) {
J
Jens Axboe 已提交
7884
		mutex_lock(&ctx->uring_lock);
7885
		submitted = io_submit_sqes(ctx, to_submit, f.file, fd);
J
Jens Axboe 已提交
7886
		mutex_unlock(&ctx->uring_lock);
7887 7888 7889

		if (submitted != to_submit)
			goto out;
J
Jens Axboe 已提交
7890 7891
	}
	if (flags & IORING_ENTER_GETEVENTS) {
J
Jens Axboe 已提交
7892 7893
		unsigned nr_events = 0;

J
Jens Axboe 已提交
7894 7895
		min_complete = min(min_complete, ctx->cq_entries);

7896 7897 7898 7899 7900 7901 7902 7903
		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
J
Jens Axboe 已提交
7904 7905 7906 7907
			ret = io_iopoll_check(ctx, &nr_events, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
7908 7909
	}

7910
out:
7911
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
7912 7913 7914 7915 7916
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

7917
#ifdef CONFIG_PROC_FS
7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978
static int io_uring_show_cred(int id, void *p, void *data)
{
	const struct cred *cred = p;
	struct seq_file *m = data;
	struct user_namespace *uns = seq_user_ns(m);
	struct group_info *gi;
	kernel_cap_t cap;
	unsigned __capi;
	int g;

	seq_printf(m, "%5d\n", id);
	seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
	seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
	seq_puts(m, "\n\tGroups:\t");
	gi = cred->group_info;
	for (g = 0; g < gi->ngroups; g++) {
		seq_put_decimal_ull(m, g ? " " : "",
					from_kgid_munged(uns, gi->gid[g]));
	}
	seq_puts(m, "\n\tCapEff:\t");
	cap = cred->cap_effective;
	CAP_FOR_EACH_U32(__capi)
		seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
	seq_putc(m, '\n');
	return 0;
}

static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	int i;

	mutex_lock(&ctx->uring_lock);
	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (!idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989
	seq_printf(m, "PollList:\n");
	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list = &ctx->cancel_hash[i];
		struct io_kiocb *req;

		hlist_for_each_entry(req, list, hash_node)
			seq_printf(m, "  op=%d, task_works=%d\n", req->opcode,
					req->task->task_works != NULL);
	}
	spin_unlock_irq(&ctx->completion_lock);
7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001
	mutex_unlock(&ctx->uring_lock);
}

static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
{
	struct io_ring_ctx *ctx = f->private_data;

	if (percpu_ref_tryget(&ctx->refs)) {
		__io_uring_show_fdinfo(ctx, m);
		percpu_ref_put(&ctx->refs);
	}
}
8002
#endif
8003

J
Jens Axboe 已提交
8004 8005
static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
8006
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
8007
	.mmap		= io_uring_mmap,
8008 8009 8010 8011
#ifndef CONFIG_MMU
	.get_unmapped_area = io_uring_nommu_get_unmapped_area,
	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
#endif
J
Jens Axboe 已提交
8012 8013
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
8014
#ifdef CONFIG_PROC_FS
8015
	.show_fdinfo	= io_uring_show_fdinfo,
8016
#endif
J
Jens Axboe 已提交
8017 8018 8019 8020 8021
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
8022 8023
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
8024

8025 8026 8027 8028 8029 8030
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
8031 8032
		return -ENOMEM;

8033 8034 8035 8036 8037 8038 8039 8040 8041 8042
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
	ctx->sq_entries = rings->sq_ring_entries;
	ctx->cq_entries = rings->cq_ring_entries;
J
Jens Axboe 已提交
8043 8044

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
8045 8046 8047
	if (size == SIZE_MAX) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8048
		return -EOVERFLOW;
8049
	}
J
Jens Axboe 已提交
8050 8051

	ctx->sq_sqes = io_mem_alloc(size);
8052 8053 8054
	if (!ctx->sq_sqes) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8055
		return -ENOMEM;
8056
	}
J
Jens Axboe 已提交
8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

8104 8105
static int io_uring_create(unsigned entries, struct io_uring_params *p,
			   struct io_uring_params __user *params)
J
Jens Axboe 已提交
8106 8107 8108
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
8109
	bool limit_mem;
J
Jens Axboe 已提交
8110 8111
	int ret;

8112
	if (!entries)
J
Jens Axboe 已提交
8113
		return -EINVAL;
8114 8115 8116 8117 8118
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}
J
Jens Axboe 已提交
8119 8120 8121 8122 8123

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
8124 8125 8126
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
8127 8128
	 */
	p->sq_entries = roundup_pow_of_two(entries);
8129 8130 8131 8132 8133 8134
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
8135
		if (p->cq_entries < p->sq_entries)
8136
			return -EINVAL;
8137 8138 8139 8140 8141
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
8142 8143 8144 8145
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
8146 8147

	user = get_uid(current_user());
8148
	limit_mem = !capable(CAP_IPC_LOCK);
J
Jens Axboe 已提交
8149

8150
	if (limit_mem) {
8151
		ret = __io_account_mem(user,
J
Jens Axboe 已提交
8152 8153 8154 8155 8156 8157 8158 8159 8160
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
8161
		if (limit_mem)
8162
			__io_unaccount_mem(user, ring_pages(p->sq_entries,
J
Jens Axboe 已提交
8163 8164 8165 8166 8167 8168
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->user = user;
8169
	ctx->creds = get_current_cred();
J
Jens Axboe 已提交
8170 8171 8172 8173 8174

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
8175
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
8176 8177 8178 8179
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
8180 8181 8182 8183 8184 8185 8186
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
8187 8188

	memset(&p->cq_off, 0, sizeof(p->cq_off));
8189 8190 8191 8192 8193 8194
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
8195
	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
8196

8197 8198
	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
8199 8200
			IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
			IORING_FEAT_POLL_32BITS;
8201 8202 8203 8204 8205

	if (copy_to_user(params, p, sizeof(*p))) {
		ret = -EFAULT;
		goto err;
	}
8206 8207 8208 8209 8210 8211 8212 8213
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

8214
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
8215 8216
	io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
		       ACCT_LOCKED);
8217
	ctx->limit_mem = limit_mem;
J
Jens Axboe 已提交
8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
8241
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8242
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
8243
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
J
Jens Axboe 已提交
8244 8245
		return -EINVAL;

8246
	return  io_uring_create(entries, &p, params);
J
Jens Axboe 已提交
8247 8248 8249 8250 8251 8252 8253 8254
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293
static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}

8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332
static int io_register_personality(struct io_ring_ctx *ctx)
{
	const struct cred *creds = get_current_cred();
	int id;

	id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
				USHRT_MAX, GFP_KERNEL);
	if (id < 0)
		put_cred(creds);
	return id;
}

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}

static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}

8333 8334
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
8335 8336
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
8337 8338 8339
{
	int ret;

8340 8341 8342 8343 8344 8345 8346 8347
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

8348
	if (io_register_op_must_quiesce(opcode)) {
8349
		percpu_ref_kill(&ctx->refs);
8350

8351 8352 8353 8354 8355 8356 8357 8358 8359
		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
8360
		ret = wait_for_completion_interruptible(&ctx->ref_comp);
8361
		mutex_lock(&ctx->uring_lock);
8362 8363 8364 8365 8366
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
8367
	}
8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
8379 8380 8381 8382 8383 8384 8385 8386 8387
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
8388 8389 8390
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
8391
	case IORING_REGISTER_EVENTFD:
8392
	case IORING_REGISTER_EVENTFD_ASYNC:
8393 8394 8395 8396
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
8397 8398 8399 8400 8401 8402
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
8403 8404 8405 8406 8407 8408 8409
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
8410 8411 8412 8413 8414 8415
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
8428 8429 8430 8431 8432
	default:
		ret = -EINVAL;
		break;
	}

8433
	if (io_register_op_must_quiesce(opcode)) {
8434 8435
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
8436
out:
8437
		reinit_completion(&ctx->ref_comp);
8438
	}
8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
8462 8463
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
8464 8465 8466 8467 8468
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
8469 8470
static int __init io_uring_init(void)
{
8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
P
Pavel Begunkov 已提交
8486
	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
8487 8488 8489 8490 8491
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
8492 8493
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
8494 8495 8496 8497 8498 8499 8500 8501
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
P
Pavel Begunkov 已提交
8502
	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
8503 8504 8505
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
P
Pavel Begunkov 已提交
8506
	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);
8507

8508
	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
8509
	BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
J
Jens Axboe 已提交
8510 8511 8512 8513
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);