io_uring.c 210.3 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
47
#include <net/compat.h>
J
Jens Axboe 已提交
48 49
#include <linux/refcount.h>
#include <linux/uio.h>
50
#include <linux/bits.h>
J
Jens Axboe 已提交
51 52 53 54 55 56 57 58 59

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
60
#include <linux/kthread.h>
J
Jens Axboe 已提交
61
#include <linux/blkdev.h>
62
#include <linux/bvec.h>
J
Jens Axboe 已提交
63 64 65
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
66
#include <net/scm.h>
J
Jens Axboe 已提交
67 68 69 70
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
71 72
#include <linux/sizes.h>
#include <linux/hugetlb.h>
73
#include <linux/highmem.h>
74 75
#include <linux/namei.h>
#include <linux/fsnotify.h>
J
Jens Axboe 已提交
76
#include <linux/fadvise.h>
77
#include <linux/eventpoll.h>
78
#include <linux/fs_struct.h>
P
Pavel Begunkov 已提交
79
#include <linux/splice.h>
80
#include <linux/task_work.h>
81
#include <linux/pagemap.h>
J
Jens Axboe 已提交
82

83 84 85
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
86 87 88
#include <uapi/linux/io_uring.h>

#include "internal.h"
89
#include "io-wq.h"
J
Jens Axboe 已提交
90

91
#define IORING_MAX_ENTRIES	32768
92
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
93 94 95 96 97 98 99 100

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
101 102 103 104 105 106

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
107
/*
108 109
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
110 111 112 113
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
114
struct io_rings {
S
Stefan Bühler 已提交
115 116 117 118
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
119 120 121
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
122
	 */
123
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
124
	/*
125
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
126 127
	 * ring_entries - 1)
	 */
128 129 130
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
131 132 133 134 135 136 137 138 139 140 141 142
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
143
	u32			sq_dropped;
S
Stefan Bühler 已提交
144
	/*
145
	 * Runtime SQ flags
S
Stefan Bühler 已提交
146 147 148 149 150 151 152
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
153
	u32			sq_flags;
154 155 156 157 158 159 160
	/*
	 * Runtime CQ flags
	 *
	 * Written by the application, shouldn't be modified by the
	 * kernel.
	 */
	u32                     cq_flags;
S
Stefan Bühler 已提交
161 162 163
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
L
LimingWu 已提交
164
	 * there are not more requests pending than there is space in
S
Stefan Bühler 已提交
165 166 167 168 169 170 171 172 173
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
174
	u32			cq_overflow;
S
Stefan Bühler 已提交
175 176 177 178 179 180 181
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
182
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
183 184
};

185 186 187 188 189 190 191
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

192 193
struct fixed_file_table {
	struct file		**files;
194 195
};

196 197 198 199 200
struct fixed_file_ref_node {
	struct percpu_ref		refs;
	struct list_head		node;
	struct list_head		file_list;
	struct fixed_file_data		*file_data;
201
	struct llist_node		llist;
202 203
};

204 205 206 207
struct fixed_file_data {
	struct fixed_file_table		*table;
	struct io_ring_ctx		*ctx;

208
	struct percpu_ref		*cur_refs;
209 210
	struct percpu_ref		refs;
	struct completion		done;
211 212
	struct list_head		ref_list;
	spinlock_t			lock;
213 214
};

215 216 217 218 219 220 221
struct io_buffer {
	struct list_head list;
	__u64 addr;
	__s32 len;
	__u16 bid;
};

J
Jens Axboe 已提交
222 223 224 225 226 227 228
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
229
		unsigned int		compat: 1;
230
		unsigned int		limit_mem: 1;
231 232 233
		unsigned int		cq_overflow_flushed: 1;
		unsigned int		drain_next: 1;
		unsigned int		eventfd_async: 1;
J
Jens Axboe 已提交
234

235 236 237 238 239 240 241 242 243 244 245 246
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
247 248 249
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
250
		unsigned		sq_thread_idle;
251
		unsigned		cached_sq_dropped;
252
		atomic_t		cached_cq_overflow;
253
		unsigned long		sq_check_overflow;
254 255

		struct list_head	defer_list;
J
Jens Axboe 已提交
256
		struct list_head	timeout_list;
257
		struct list_head	cq_overflow_list;
258 259

		wait_queue_head_t	inflight_wait;
260
		struct io_uring_sqe	*sq_sqes;
J
Jens Axboe 已提交
261 262
	} ____cacheline_aligned_in_smp;

263 264
	struct io_rings	*rings;

J
Jens Axboe 已提交
265
	/* IO offload */
266
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
267
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
268
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
269
	wait_queue_head_t	sqo_wait;
270

J
Jens Axboe 已提交
271 272 273 274 275
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
276
	struct fixed_file_data	*file_data;
J
Jens Axboe 已提交
277
	unsigned		nr_user_files;
P
Pavel Begunkov 已提交
278 279
	int 			ring_fd;
	struct file 		*ring_file;
J
Jens Axboe 已提交
280

281 282 283 284
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
285 286
	struct user_struct	*user;

287
	const struct cred	*creds;
288

289 290
	struct completion	ref_comp;
	struct completion	sq_thread_comp;
291

292 293 294
	/* if all else fails... */
	struct io_kiocb		*fallback_req;

295 296 297 298
#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif

299 300
	struct idr		io_buffer_idr;

301 302
	struct idr		personality_idr;

303 304 305 306 307
	struct {
		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		unsigned		cq_mask;
		atomic_t		cq_timeouts;
308
		unsigned long		cq_check_overflow;
309 310 311 312
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
		struct eventfd_ctx	*cq_ev_fd;
	} ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
313 314 315 316 317 318 319 320

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
321

J
Jens Axboe 已提交
322
		/*
323
		 * ->iopoll_list is protected by the ctx->uring_lock for
J
Jens Axboe 已提交
324 325 326 327
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
328
		struct list_head	iopoll_list;
329 330
		struct hlist_head	*cancel_hash;
		unsigned		cancel_hash_bits;
331
		bool			poll_multi_file;
332

333 334
		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
335
	} ____cacheline_aligned_in_smp;
336

337 338 339
	struct delayed_work		file_put_work;
	struct llist_head		file_put_llist;

340
	struct work_struct		exit_work;
J
Jens Axboe 已提交
341 342
};

J
Jens Axboe 已提交
343 344 345 346
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
347 348
struct io_poll_iocb {
	struct file			*file;
349 350 351 352
	union {
		struct wait_queue_head	*head;
		u64			addr;
	};
353
	__poll_t			events;
J
Jens Axboe 已提交
354
	bool				done;
355
	bool				canceled;
356
	struct wait_queue_entry		wait;
357 358
};

359 360 361 362 363 364
struct io_close {
	struct file			*file;
	struct file			*put_file;
	int				fd;
};

365 366 367 368 369 370 371
struct io_timeout_data {
	struct io_kiocb			*req;
	struct hrtimer			timer;
	struct timespec64		ts;
	enum hrtimer_mode		mode;
};

372 373 374 375 376
struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
377
	unsigned long			nofile;
378 379 380 381 382 383 384
};

struct io_sync {
	struct file			*file;
	loff_t				len;
	loff_t				off;
	int				flags;
385
	int				mode;
386 387
};

388 389 390 391 392
struct io_cancel {
	struct file			*file;
	u64				addr;
};

393 394 395 396
struct io_timeout {
	struct file			*file;
	u64				addr;
	int				flags;
397 398
	u32				off;
	u32				target_seq;
P
Pavel Begunkov 已提交
399
	struct list_head		list;
400 401
};

402 403 404 405 406 407 408
struct io_rw {
	/* NOTE: kiocb has the file as the first member, so don't do it here */
	struct kiocb			kiocb;
	u64				addr;
	u64				len;
};

409 410 411 412 413 414
struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
};

415 416
struct io_sr_msg {
	struct file			*file;
417
	union {
418
		struct user_msghdr __user *umsg;
419 420
		void __user		*buf;
	};
421
	int				msg_flags;
422
	int				bgid;
423
	size_t				len;
424
	struct io_buffer		*kbuf;
425 426
};

427 428 429 430
struct io_open {
	struct file			*file;
	int				dfd;
	struct filename			*filename;
431
	struct open_how			how;
432
	unsigned long			nofile;
433 434
};

435 436 437 438 439 440 441
struct io_files_update {
	struct file			*file;
	u64				arg;
	u32				nr_args;
	u32				offset;
};

J
Jens Axboe 已提交
442 443 444 445 446 447 448
struct io_fadvise {
	struct file			*file;
	u64				offset;
	u32				len;
	u32				advice;
};

J
Jens Axboe 已提交
449 450 451 452 453 454 455
struct io_madvise {
	struct file			*file;
	u64				addr;
	u32				len;
	u32				advice;
};

456 457 458 459 460 461
struct io_epoll {
	struct file			*file;
	int				epfd;
	int				op;
	int				fd;
	struct epoll_event		event;
462 463
};

P
Pavel Begunkov 已提交
464 465 466 467 468 469 470 471 472
struct io_splice {
	struct file			*file_out;
	struct file			*file_in;
	loff_t				off_out;
	loff_t				off_in;
	u64				len;
	unsigned int			flags;
};

473 474 475 476 477 478 479 480 481
struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__s32				len;
	__u32				bgid;
	__u16				nbufs;
	__u16				bid;
};

482 483 484 485 486
struct io_statx {
	struct file			*file;
	int				dfd;
	unsigned int			mask;
	unsigned int			flags;
B
Bijan Mottahedeh 已提交
487
	const char __user		*filename;
488 489 490
	struct statx __user		*buffer;
};

491 492 493
struct io_completion {
	struct file			*file;
	struct list_head		list;
494
	int				cflags;
495 496
};

497 498 499 500
struct io_async_connect {
	struct sockaddr_storage		address;
};

501 502 503 504 505
struct io_async_msghdr {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	struct sockaddr __user		*uaddr;
	struct msghdr			msg;
506
	struct sockaddr_storage		addr;
507 508
};

509 510
struct io_async_rw {
	struct iovec			fast_iov[UIO_FASTIOV];
511 512
	const struct iovec		*free_iovec;
	struct iov_iter			iter;
513
	size_t				bytes_done;
514
	struct wait_page_queue		wpq;
515 516
};

517
struct io_async_ctx {
518 519
	union {
		struct io_async_rw	rw;
520
		struct io_async_msghdr	msg;
521
		struct io_async_connect	connect;
522
		struct io_timeout_data	timeout;
523
	};
524 525
};

526 527 528 529 530 531
enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
532
	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
533

534
	REQ_F_LINK_HEAD_BIT,
535 536 537 538 539 540 541
	REQ_F_FAIL_LINK_BIT,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_ISREG_BIT,
	REQ_F_COMP_LOCKED_BIT,
P
Pavel Begunkov 已提交
542
	REQ_F_NEED_CLEANUP_BIT,
543
	REQ_F_POLLED_BIT,
544
	REQ_F_BUFFER_SELECTED_BIT,
545
	REQ_F_NO_FILE_TABLE_BIT,
546
	REQ_F_WORK_INITIALIZED_BIT,
P
Pavel Begunkov 已提交
547
	REQ_F_TASK_PINNED_BIT,
548 549 550

	/* not a real bit, just to check we're not overflowing the space */
	__REQ_F_LAST_BIT,
551 552 553 554 555 556 557 558 559 560 561 562 563
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),
564 565
	/* IOSQE_BUFFER_SELECT */
	REQ_F_BUFFER_SELECT	= BIT(REQ_F_BUFFER_SELECT_BIT),
566

567 568
	/* head of a link */
	REQ_F_LINK_HEAD		= BIT(REQ_F_LINK_HEAD_BIT),
569 570 571 572 573 574 575 576 577 578 579 580 581 582
	/* fail rest of links */
	REQ_F_FAIL_LINK		= BIT(REQ_F_FAIL_LINK_BIT),
	/* on inflight list */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* has linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* completion under lock */
	REQ_F_COMP_LOCKED	= BIT(REQ_F_COMP_LOCKED_BIT),
P
Pavel Begunkov 已提交
583 584
	/* needs cleanup */
	REQ_F_NEED_CLEANUP	= BIT(REQ_F_NEED_CLEANUP_BIT),
585 586
	/* already went through poll handler */
	REQ_F_POLLED		= BIT(REQ_F_POLLED_BIT),
587 588
	/* buffer already selected */
	REQ_F_BUFFER_SELECTED	= BIT(REQ_F_BUFFER_SELECTED_BIT),
589 590
	/* doesn't need file table for this request */
	REQ_F_NO_FILE_TABLE	= BIT(REQ_F_NO_FILE_TABLE_BIT),
591 592
	/* io_wq_work is initialized */
	REQ_F_WORK_INITIALIZED	= BIT(REQ_F_WORK_INITIALIZED_BIT),
P
Pavel Begunkov 已提交
593 594
	/* req->task is refcounted */
	REQ_F_TASK_PINNED	= BIT(REQ_F_TASK_PINNED_BIT),
595 596 597 598
};

struct async_poll {
	struct io_poll_iocb	poll;
599
	struct io_poll_iocb	*double_poll;
600 601
};

J
Jens Axboe 已提交
602 603 604 605 606 607
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
608
struct io_kiocb {
609
	union {
J
Jens Axboe 已提交
610
		struct file		*file;
611
		struct io_rw		rw;
612
		struct io_poll_iocb	poll;
613 614
		struct io_accept	accept;
		struct io_sync		sync;
615
		struct io_cancel	cancel;
616
		struct io_timeout	timeout;
617
		struct io_connect	connect;
618
		struct io_sr_msg	sr_msg;
619
		struct io_open		open;
620
		struct io_close		close;
621
		struct io_files_update	files_update;
J
Jens Axboe 已提交
622
		struct io_fadvise	fadvise;
J
Jens Axboe 已提交
623
		struct io_madvise	madvise;
624
		struct io_epoll		epoll;
P
Pavel Begunkov 已提交
625
		struct io_splice	splice;
626
		struct io_provide_buf	pbuf;
627
		struct io_statx		statx;
628 629
		/* use only after cleaning per-op data, see io_clean_op() */
		struct io_completion	compl;
630
	};
J
Jens Axboe 已提交
631

632
	struct io_async_ctx		*io;
633
	u8				opcode;
634 635
	/* polled IO has completed */
	u8				iopoll_completed;
J
Jens Axboe 已提交
636

637
	u16				buf_index;
638
	u32				result;
639

P
Pavel Begunkov 已提交
640 641 642 643 644
	struct io_ring_ctx		*ctx;
	unsigned int			flags;
	refcount_t			refs;
	struct task_struct		*task;
	u64				user_data;
645

P
Pavel Begunkov 已提交
646
	struct list_head		link_list;
647

648 649 650 651
	/*
	 * 1. used with ctx->iopoll_list with reads/writes
	 * 2. to track reqs with ->files (see io_op_def::file_table)
	 */
P
Pavel Begunkov 已提交
652 653 654 655 656 657 658 659
	struct list_head		inflight_entry;

	struct percpu_ref		*fixed_file_refs;
	struct callback_head		task_work;
	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
	struct hlist_node		hash_node;
	struct async_poll		*apoll;
	struct io_wq_work		work;
J
Jens Axboe 已提交
660
};
661

662 663 664
struct io_defer_entry {
	struct list_head	list;
	struct io_kiocb		*req;
665
	u32			seq;
J
Jens Axboe 已提交
666 667
};

J
Jens Axboe 已提交
668
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
669

670 671 672 673 674 675
struct io_comp_state {
	unsigned int		nr;
	struct list_head	list;
	struct io_ring_ctx	*ctx;
};

676 677 678
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
679 680 681 682
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
683
	unsigned int		free_reqs;
J
Jens Axboe 已提交
684

685 686 687 688 689
	/*
	 * Batch completion logic
	 */
	struct io_comp_state	comp;

690 691 692 693 694 695 696 697 698
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		ios_left;
};

699 700 701 702 703 704 705
struct io_op_def {
	/* needs req->io allocated for deferral/async */
	unsigned		async_ctx : 1;
	/* needs current->mm setup, does mm access */
	unsigned		needs_mm : 1;
	/* needs req->file assigned */
	unsigned		needs_file : 1;
706 707
	/* don't fail if file grab fails */
	unsigned		needs_file_no_error : 1;
708 709 710 711
	/* hash wq insertion if file is a regular file */
	unsigned		hash_reg_file : 1;
	/* unbound wq insertion if file is a non-regular file */
	unsigned		unbound_nonreg_file : 1;
712 713
	/* opcode is not supported by this kernel */
	unsigned		not_supported : 1;
714 715
	/* needs file table */
	unsigned		file_table : 1;
716 717
	/* needs ->fs */
	unsigned		needs_fs : 1;
718 719 720
	/* set if opcode supports polled "wait" */
	unsigned		pollin : 1;
	unsigned		pollout : 1;
721 722
	/* op supports buffer selection */
	unsigned		buffer_select : 1;
723
	unsigned		needs_fsize : 1;
724 725 726
};

static const struct io_op_def io_op_defs[] = {
727 728
	[IORING_OP_NOP] = {},
	[IORING_OP_READV] = {
729 730 731 732
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
733
		.pollin			= 1,
734
		.buffer_select		= 1,
735
	},
736
	[IORING_OP_WRITEV] = {
737 738 739 740 741
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
742
		.pollout		= 1,
743
		.needs_fsize		= 1,
744
	},
745
	[IORING_OP_FSYNC] = {
746 747
		.needs_file		= 1,
	},
748
	[IORING_OP_READ_FIXED] = {
749 750
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
751
		.pollin			= 1,
752
	},
753
	[IORING_OP_WRITE_FIXED] = {
754 755 756
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
757
		.pollout		= 1,
758
		.needs_fsize		= 1,
759
	},
760
	[IORING_OP_POLL_ADD] = {
761 762 763
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
764 765
	[IORING_OP_POLL_REMOVE] = {},
	[IORING_OP_SYNC_FILE_RANGE] = {
766 767
		.needs_file		= 1,
	},
768
	[IORING_OP_SENDMSG] = {
769 770 771 772
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
773
		.needs_fs		= 1,
774
		.pollout		= 1,
775
	},
776
	[IORING_OP_RECVMSG] = {
777 778 779 780
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
781
		.needs_fs		= 1,
782
		.pollin			= 1,
783
		.buffer_select		= 1,
784
	},
785
	[IORING_OP_TIMEOUT] = {
786 787 788
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
789 790
	[IORING_OP_TIMEOUT_REMOVE] = {},
	[IORING_OP_ACCEPT] = {
791 792 793
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
794
		.file_table		= 1,
795
		.pollin			= 1,
796
	},
797 798
	[IORING_OP_ASYNC_CANCEL] = {},
	[IORING_OP_LINK_TIMEOUT] = {
799 800 801
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
802
	[IORING_OP_CONNECT] = {
803 804 805 806
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
807
		.pollout		= 1,
808
	},
809
	[IORING_OP_FALLOCATE] = {
810
		.needs_file		= 1,
811
		.needs_fsize		= 1,
812
	},
813
	[IORING_OP_OPENAT] = {
814
		.file_table		= 1,
815
		.needs_fs		= 1,
816
	},
817
	[IORING_OP_CLOSE] = {
818 819
		.needs_file		= 1,
		.needs_file_no_error	= 1,
820
		.file_table		= 1,
821
	},
822
	[IORING_OP_FILES_UPDATE] = {
823
		.needs_mm		= 1,
824
		.file_table		= 1,
825
	},
826
	[IORING_OP_STATX] = {
827
		.needs_mm		= 1,
828
		.needs_fs		= 1,
829
		.file_table		= 1,
830
	},
831
	[IORING_OP_READ] = {
832 833 834
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
835
		.pollin			= 1,
836
		.buffer_select		= 1,
837
	},
838
	[IORING_OP_WRITE] = {
839 840 841
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
842
		.pollout		= 1,
843
		.needs_fsize		= 1,
844
	},
845
	[IORING_OP_FADVISE] = {
J
Jens Axboe 已提交
846 847
		.needs_file		= 1,
	},
848
	[IORING_OP_MADVISE] = {
J
Jens Axboe 已提交
849 850
		.needs_mm		= 1,
	},
851
	[IORING_OP_SEND] = {
852 853 854
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
855
		.pollout		= 1,
856
	},
857
	[IORING_OP_RECV] = {
858 859 860
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
861
		.pollin			= 1,
862
		.buffer_select		= 1,
863
	},
864
	[IORING_OP_OPENAT2] = {
865
		.file_table		= 1,
866
		.needs_fs		= 1,
867
	},
868 869 870 871
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.file_table		= 1,
	},
P
Pavel Begunkov 已提交
872 873 874 875
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
876 877
	},
	[IORING_OP_PROVIDE_BUFFERS] = {},
878
	[IORING_OP_REMOVE_BUFFERS] = {},
P
Pavel Begunkov 已提交
879 880 881 882 883
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
884 885
};

886 887 888 889 890
enum io_mem_account {
	ACCT_LOCKED,
	ACCT_PINNED,
};

891 892
static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
			     struct io_comp_state *cs);
893
static void io_cqring_fill_event(struct io_kiocb *req, long res);
894
static void io_put_req(struct io_kiocb *req);
895
static void io_double_put_req(struct io_kiocb *req);
896
static void __io_double_put_req(struct io_kiocb *req);
897
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
898
static void __io_queue_linked_timeout(struct io_kiocb *req);
899
static void io_queue_linked_timeout(struct io_kiocb *req);
900 901 902
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *ip,
				 unsigned nr_args);
903
static int io_prep_work_files(struct io_kiocb *req);
904
static void __io_clean_op(struct io_kiocb *req);
905 906 907
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
		       int fd, struct file **out_file, bool fixed);
static void __io_queue_sqe(struct io_kiocb *req,
908 909
			   const struct io_uring_sqe *sqe,
			   struct io_comp_state *cs);
910
static void io_file_put_work(struct work_struct *work);
911

912 913 914
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock);
915 916
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
			     const struct iovec *fast_iov,
917
			     struct iov_iter *iter, bool force);
918

J
Jens Axboe 已提交
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

P
Pavel Begunkov 已提交
936 937 938 939 940 941 942 943
static void io_get_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		return;
	get_task_struct(req->task);
	req->flags |= REQ_F_TASK_PINNED;
}

944 945
static inline void io_clean_op(struct io_kiocb *req)
{
946 947
	if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED |
			  REQ_F_INFLIGHT))
948 949 950
		__io_clean_op(req);
}

P
Pavel Begunkov 已提交
951 952 953 954 955 956 957
/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
static void __io_put_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		put_task_struct(req->task);
}

958
static void io_sq_thread_drop_mm(void)
959 960 961 962 963 964 965 966 967 968 969 970
{
	struct mm_struct *mm = current->mm;

	if (mm) {
		kthread_unuse_mm(mm);
		mmput(mm);
	}
}

static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
{
	if (!current->mm) {
971 972
		if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) ||
			     !mmget_not_zero(ctx->sqo_mm)))
973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
			return -EFAULT;
		kthread_use_mm(ctx->sqo_mm);
	}

	return 0;
}

static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	if (!io_op_defs[req->opcode].needs_mm)
		return 0;
	return __io_sq_thread_acquire_mm(ctx);
}

static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}
993

994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
/*
 * Note: must call io_req_init_async() for the first time you
 * touch any members of io_wq_work.
 */
static inline void io_req_init_async(struct io_kiocb *req)
{
	if (req->flags & REQ_F_WORK_INITIALIZED)
		return;

	memset(&req->work, 0, sizeof(req->work));
	req->flags |= REQ_F_WORK_INITIALIZED;
}

1007 1008 1009 1010 1011
static inline bool io_async_submit(struct io_ring_ctx *ctx)
{
	return ctx->flags & IORING_SETUP_SQPOLL;
}

J
Jens Axboe 已提交
1012 1013 1014 1015
static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

1016
	complete(&ctx->ref_comp);
J
Jens Axboe 已提交
1017 1018
}

1019 1020 1021 1022 1023
static inline bool io_is_timeout_noseq(struct io_kiocb *req)
{
	return !req->timeout.off;
}

J
Jens Axboe 已提交
1024 1025 1026
static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;
1027
	int hash_bits;
J
Jens Axboe 已提交
1028 1029 1030 1031 1032

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

1033 1034 1035 1036
	ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
	if (!ctx->fallback_req)
		goto err;

1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
	/*
	 * Use 5 bits less than the max cq entries, that should give us around
	 * 32 entries per hash list if totally full and uniformly spread.
	 */
	hash_bits = ilog2(p->cq_entries);
	hash_bits -= 5;
	if (hash_bits <= 0)
		hash_bits = 1;
	ctx->cancel_hash_bits = hash_bits;
	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
					GFP_KERNEL);
	if (!ctx->cancel_hash)
		goto err;
	__hash_init(ctx->cancel_hash, 1U << hash_bits);

1052
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1053 1054
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
		goto err;
J
Jens Axboe 已提交
1055 1056

	ctx->flags = p->flags;
1057
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
1058
	init_waitqueue_head(&ctx->cq_wait);
1059
	INIT_LIST_HEAD(&ctx->cq_overflow_list);
1060 1061
	init_completion(&ctx->ref_comp);
	init_completion(&ctx->sq_thread_comp);
1062
	idr_init(&ctx->io_buffer_idr);
1063
	idr_init(&ctx->personality_idr);
J
Jens Axboe 已提交
1064 1065 1066
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
1067
	INIT_LIST_HEAD(&ctx->iopoll_list);
1068
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
1069
	INIT_LIST_HEAD(&ctx->timeout_list);
1070 1071 1072
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
1073 1074
	INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work);
	init_llist_head(&ctx->file_put_llist);
J
Jens Axboe 已提交
1075
	return ctx;
1076
err:
1077 1078
	if (ctx->fallback_req)
		kmem_cache_free(req_cachep, ctx->fallback_req);
1079
	kfree(ctx->cancel_hash);
1080 1081
	kfree(ctx);
	return NULL;
J
Jens Axboe 已提交
1082 1083
}

1084
static bool req_need_defer(struct io_kiocb *req, u32 seq)
1085
{
1086 1087
	if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
		struct io_ring_ctx *ctx = req->ctx;
1088

1089
		return seq != ctx->cached_cq_tail
1090
				+ atomic_read(&ctx->cached_cq_overflow);
1091
	}
1092

B
Bob Liu 已提交
1093
	return false;
1094 1095 1096
}

static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
1097
{
1098
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1099

1100 1101
	/* order cqe stores with ring update */
	smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
1102

1103 1104 1105
	if (wq_has_sleeper(&ctx->cq_wait)) {
		wake_up_interruptible(&ctx->cq_wait);
		kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
J
Jens Axboe 已提交
1106 1107 1108
	}
}

1109 1110 1111 1112 1113
/*
 * Returns true if we need to defer file table putting. This can only happen
 * from the error path with REQ_F_COMP_LOCKED set.
 */
static bool io_req_clean_work(struct io_kiocb *req)
1114
{
1115
	if (!(req->flags & REQ_F_WORK_INITIALIZED))
1116 1117 1118
		return false;

	req->flags &= ~REQ_F_WORK_INITIALIZED;
1119

1120 1121 1122 1123 1124 1125 1126 1127
	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
1128 1129 1130
	if (req->work.fs) {
		struct fs_struct *fs = req->work.fs;

1131 1132 1133
		if (req->flags & REQ_F_COMP_LOCKED)
			return true;

1134 1135 1136 1137 1138 1139
		spin_lock(&req->work.fs->lock);
		if (--fs->users)
			fs = NULL;
		spin_unlock(&req->work.fs->lock);
		if (fs)
			free_fs_struct(fs);
1140
		req->work.fs = NULL;
1141
	}
1142 1143

	return false;
1144 1145
}

1146
static void io_prep_async_work(struct io_kiocb *req)
1147
{
1148
	const struct io_op_def *def = &io_op_defs[req->opcode];
1149

1150 1151
	io_req_init_async(req);

1152 1153
	if (req->flags & REQ_F_ISREG) {
		if (def->hash_reg_file)
1154
			io_wq_hash_work(&req->work, file_inode(req->file));
1155 1156
	} else {
		if (def->unbound_nonreg_file)
1157
			req->work.flags |= IO_WQ_WORK_UNBOUND;
1158
	}
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	if (!req->work.mm && def->needs_mm) {
		mmgrab(current->mm);
		req->work.mm = current->mm;
	}
	if (!req->work.creds)
		req->work.creds = get_current_cred();
	if (!req->work.fs && def->needs_fs) {
		spin_lock(&current->fs->lock);
		if (!current->fs->in_exec) {
			req->work.fs = current->fs;
			req->work.fs->users++;
		} else {
			req->work.flags |= IO_WQ_WORK_CANCEL;
		}
		spin_unlock(&current->fs->lock);
	}
1175 1176 1177 1178
	if (def->needs_fsize)
		req->work.fsize = rlimit(RLIMIT_FSIZE);
	else
		req->work.fsize = RLIM_INFINITY;
1179
}
1180

1181
static void io_prep_async_link(struct io_kiocb *req)
1182
{
1183
	struct io_kiocb *cur;
1184

1185 1186 1187 1188
	io_prep_async_work(req);
	if (req->flags & REQ_F_LINK_HEAD)
		list_for_each_entry(cur, &req->link_list, link_list)
			io_prep_async_work(cur);
1189 1190
}

1191
static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req)
1192
{
1193
	struct io_ring_ctx *ctx = req->ctx;
1194
	struct io_kiocb *link = io_prep_linked_timeout(req);
1195

1196 1197 1198
	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
					&req->work, req->flags);
	io_wq_enqueue(ctx->io_wq, &req->work);
1199
	return link;
1200 1201
}

1202 1203
static void io_queue_async_work(struct io_kiocb *req)
{
1204 1205
	struct io_kiocb *link;

1206 1207
	/* init ->work of the whole link before punting */
	io_prep_async_link(req);
1208 1209 1210 1211
	link = __io_queue_async_work(req);

	if (link)
		io_queue_linked_timeout(link);
1212 1213
}

J
Jens Axboe 已提交
1214 1215 1216 1217
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

1218
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
J
Jens Axboe 已提交
1219
	if (ret != -1) {
1220 1221
		atomic_set(&req->ctx->cq_timeouts,
			atomic_read(&req->ctx->cq_timeouts) + 1);
P
Pavel Begunkov 已提交
1222
		list_del_init(&req->timeout.list);
1223
		req->flags |= REQ_F_COMP_LOCKED;
1224
		io_cqring_fill_event(req, 0);
1225
		io_put_req(req);
J
Jens Axboe 已提交
1226 1227 1228 1229 1230 1231 1232 1233
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
P
Pavel Begunkov 已提交
1234
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list)
J
Jens Axboe 已提交
1235 1236 1237 1238
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

1239
static void __io_queue_deferred(struct io_ring_ctx *ctx)
1240
{
1241
	do {
1242 1243
		struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
						struct io_defer_entry, list);
1244
		struct io_kiocb *link;
1245

1246
		if (req_need_defer(de->req, de->seq))
1247
			break;
1248
		list_del_init(&de->list);
1249
		/* punt-init is done before queueing for defer */
1250 1251 1252 1253 1254 1255 1256
		link = __io_queue_async_work(de->req);
		if (link) {
			__io_queue_linked_timeout(link);
			/* drop submission reference */
			link->flags |= REQ_F_COMP_LOCKED;
			io_put_req(link);
		}
1257
		kfree(de);
1258 1259 1260
	} while (!list_empty(&ctx->defer_list));
}

1261
static void io_flush_timeouts(struct io_ring_ctx *ctx)
1262
{
1263 1264
	while (!list_empty(&ctx->timeout_list)) {
		struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
P
Pavel Begunkov 已提交
1265
						struct io_kiocb, timeout.list);
1266

1267
		if (io_is_timeout_noseq(req))
1268
			break;
1269 1270
		if (req->timeout.target_seq != ctx->cached_cq_tail
					- atomic_read(&ctx->cq_timeouts))
1271
			break;
1272

P
Pavel Begunkov 已提交
1273
		list_del_init(&req->timeout.list);
J
Jens Axboe 已提交
1274
		io_kill_timeout(req);
1275 1276
	}
}
J
Jens Axboe 已提交
1277

1278 1279 1280
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	io_flush_timeouts(ctx);
1281 1282
	__io_commit_cqring(ctx);

1283 1284
	if (unlikely(!list_empty(&ctx->defer_list)))
		__io_queue_deferred(ctx);
1285 1286
}

J
Jens Axboe 已提交
1287 1288
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
1289
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1290 1291 1292
	unsigned tail;

	tail = ctx->cached_cq_tail;
1293 1294 1295 1296 1297
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
1298
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
1299 1300 1301
		return NULL;

	ctx->cached_cq_tail++;
1302
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
1303 1304
}

1305 1306
static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
{
1307 1308
	if (!ctx->cq_ev_fd)
		return false;
1309 1310
	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
		return false;
1311 1312
	if (!ctx->eventfd_async)
		return true;
1313
	return io_wq_current_is_worker();
1314 1315
}

1316
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1317 1318 1319 1320 1321
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
1322
	if (io_should_trigger_evfd(ctx))
1323 1324 1325
		eventfd_signal(ctx->cq_ev_fd, 1);
}

1326 1327 1328 1329 1330 1331 1332 1333 1334
static void io_cqring_mark_overflow(struct io_ring_ctx *ctx)
{
	if (list_empty(&ctx->cq_overflow_list)) {
		clear_bit(0, &ctx->sq_check_overflow);
		clear_bit(0, &ctx->cq_check_overflow);
		ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
	}
}

1335 1336
/* Returns true if there are no backlogged entries after the flush */
static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1337 1338 1339 1340 1341 1342 1343 1344 1345
{
	struct io_rings *rings = ctx->rings;
	struct io_uring_cqe *cqe;
	struct io_kiocb *req;
	unsigned long flags;
	LIST_HEAD(list);

	if (!force) {
		if (list_empty_careful(&ctx->cq_overflow_list))
1346
			return true;
1347 1348
		if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
		    rings->cq_ring_entries))
1349
			return false;
1350 1351 1352 1353 1354 1355
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/* if force is set, the ring is going away. always drop after that */
	if (force)
1356
		ctx->cq_overflow_flushed = 1;
1357

1358
	cqe = NULL;
1359 1360 1361 1362 1363 1364
	while (!list_empty(&ctx->cq_overflow_list)) {
		cqe = io_get_cqring(ctx);
		if (!cqe && !force)
			break;

		req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1365 1366
						compl.list);
		list_move(&req->compl.list, &list);
1367 1368 1369
		if (cqe) {
			WRITE_ONCE(cqe->user_data, req->user_data);
			WRITE_ONCE(cqe->res, req->result);
1370
			WRITE_ONCE(cqe->flags, req->compl.cflags);
1371 1372 1373 1374 1375 1376 1377
		} else {
			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
		}
	}

	io_commit_cqring(ctx);
1378 1379
	io_cqring_mark_overflow(ctx);

1380 1381 1382 1383
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	while (!list_empty(&list)) {
1384 1385
		req = list_first_entry(&list, struct io_kiocb, compl.list);
		list_del(&req->compl.list);
1386
		io_put_req(req);
1387
	}
1388 1389

	return cqe != NULL;
1390 1391
}

1392
static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1393
{
1394
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1395 1396
	struct io_uring_cqe *cqe;

1397
	trace_io_uring_complete(ctx, req->user_data, res);
J
Jens Axboe 已提交
1398

J
Jens Axboe 已提交
1399 1400 1401 1402 1403 1404
	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
1405
	if (likely(cqe)) {
1406
		WRITE_ONCE(cqe->user_data, req->user_data);
J
Jens Axboe 已提交
1407
		WRITE_ONCE(cqe->res, res);
1408
		WRITE_ONCE(cqe->flags, cflags);
1409
	} else if (ctx->cq_overflow_flushed) {
1410 1411
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
1412
	} else {
1413 1414 1415
		if (list_empty(&ctx->cq_overflow_list)) {
			set_bit(0, &ctx->sq_check_overflow);
			set_bit(0, &ctx->cq_check_overflow);
1416
			ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
1417
		}
1418
		io_clean_op(req);
1419
		req->result = res;
1420
		req->compl.cflags = cflags;
1421 1422
		refcount_inc(&req->refs);
		list_add_tail(&req->compl.list, &ctx->cq_overflow_list);
J
Jens Axboe 已提交
1423 1424 1425
	}
}

1426 1427 1428 1429 1430
static void io_cqring_fill_event(struct io_kiocb *req, long res)
{
	__io_cqring_fill_event(req, res, 0);
}

1431
static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1432
{
1433
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1434 1435 1436
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
1437
	__io_cqring_fill_event(req, res, cflags);
J
Jens Axboe 已提交
1438 1439 1440
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
1441
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1442 1443
}

1444
static void io_submit_flush_completions(struct io_comp_state *cs)
1445
{
1446 1447 1448 1449 1450 1451
	struct io_ring_ctx *ctx = cs->ctx;

	spin_lock_irq(&ctx->completion_lock);
	while (!list_empty(&cs->list)) {
		struct io_kiocb *req;

1452 1453
		req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
		list_del(&req->compl.list);
1454
		__io_cqring_fill_event(req, req->result, req->compl.cflags);
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
		if (!(req->flags & REQ_F_LINK_HEAD)) {
			req->flags |= REQ_F_COMP_LOCKED;
			io_put_req(req);
		} else {
			spin_unlock_irq(&ctx->completion_lock);
			io_put_req(req);
			spin_lock_irq(&ctx->completion_lock);
		}
	}
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	cs->nr = 0;
}

static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
			      struct io_comp_state *cs)
{
	if (!cs) {
		io_cqring_add_event(req, res, cflags);
		io_put_req(req);
	} else {
1478
		io_clean_op(req);
1479
		req->result = res;
1480
		req->compl.cflags = cflags;
1481
		list_add_tail(&req->compl.list, &cs->list);
1482 1483 1484
		if (++cs->nr >= 32)
			io_submit_flush_completions(cs);
	}
1485 1486 1487
}

static void io_req_complete(struct io_kiocb *req, long res)
1488
{
1489
	__io_req_complete(req, res, 0, NULL);
1490 1491
}

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
static inline bool io_is_fallback_req(struct io_kiocb *req)
{
	return req == (struct io_kiocb *)
			((unsigned long) req->ctx->fallback_req & ~1UL);
}

static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	req = ctx->fallback_req;
1503
	if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req))
1504 1505 1506 1507 1508
		return req;

	return NULL;
}

1509 1510
static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
				     struct io_submit_state *state)
J
Jens Axboe 已提交
1511
{
1512
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
1513 1514
	struct io_kiocb *req;

1515
	if (!state->free_reqs) {
J
Jens Axboe 已提交
1516 1517 1518 1519
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
1520 1521 1522 1523 1524 1525 1526 1527 1528
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
1529
				goto fallback;
1530 1531
			ret = 1;
		}
J
Jens Axboe 已提交
1532
		state->free_reqs = ret - 1;
1533
		req = state->reqs[ret - 1];
J
Jens Axboe 已提交
1534 1535
	} else {
		state->free_reqs--;
1536
		req = state->reqs[state->free_reqs];
J
Jens Axboe 已提交
1537 1538
	}

J
Jens Axboe 已提交
1539
	return req;
1540
fallback:
1541
	return io_get_fallback_req(ctx);
J
Jens Axboe 已提交
1542 1543
}

1544 1545 1546 1547
static inline void io_put_file(struct io_kiocb *req, struct file *file,
			  bool fixed)
{
	if (fixed)
1548
		percpu_ref_put(req->fixed_file_refs);
1549 1550 1551 1552
	else
		fput(file);
}

1553
static bool io_dismantle_req(struct io_kiocb *req)
J
Jens Axboe 已提交
1554
{
1555
	io_clean_op(req);
1556

1557 1558
	if (req->io)
		kfree(req->io);
1559 1560
	if (req->file)
		io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1561

1562
	return io_req_clean_work(req);
1563 1564
}

1565
static void __io_free_req_finish(struct io_kiocb *req)
1566
{
1567
	struct io_ring_ctx *ctx = req->ctx;
1568

1569
	__io_put_req_task(req);
1570 1571 1572
	if (likely(!io_is_fallback_req(req)))
		kmem_cache_free(req_cachep, req);
	else
1573 1574
		clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req);
	percpu_ref_put(&ctx->refs);
1575 1576
}

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
static void io_req_task_file_table_put(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct fs_struct *fs = req->work.fs;

	spin_lock(&req->work.fs->lock);
	if (--fs->users)
		fs = NULL;
	spin_unlock(&req->work.fs->lock);
	if (fs)
		free_fs_struct(fs);
	req->work.fs = NULL;
	__io_free_req_finish(req);
}

static void __io_free_req(struct io_kiocb *req)
{
	if (!io_dismantle_req(req)) {
		__io_free_req_finish(req);
	} else {
		int ret;

		init_task_work(&req->task_work, io_req_task_file_table_put);
		ret = task_work_add(req->task, &req->task_work, TWA_RESUME);
		if (unlikely(ret)) {
			struct task_struct *tsk;

			tsk = io_wq_get_task(req->ctx->io_wq);
			task_work_add(tsk, &req->task_work, 0);
		}
	}
}

1610
static bool io_link_cancel_timeout(struct io_kiocb *req)
1611
{
1612
	struct io_ring_ctx *ctx = req->ctx;
1613 1614
	int ret;

1615
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1616
	if (ret != -1) {
1617
		io_cqring_fill_event(req, -ECANCELED);
1618
		io_commit_cqring(ctx);
1619
		req->flags &= ~REQ_F_LINK_HEAD;
1620
		io_put_req(req);
1621 1622 1623 1624
		return true;
	}

	return false;
1625 1626
}

1627
static bool __io_kill_linked_timeout(struct io_kiocb *req)
J
Jens Axboe 已提交
1628
{
1629
	struct io_kiocb *link;
1630
	bool wake_ev;
1631 1632

	if (list_empty(&req->link_list))
1633
		return false;
1634 1635
	link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	if (link->opcode != IORING_OP_LINK_TIMEOUT)
1636
		return false;
1637 1638

	list_del_init(&link->link_list);
1639
	link->flags |= REQ_F_COMP_LOCKED;
1640 1641
	wake_ev = io_link_cancel_timeout(link);
	req->flags &= ~REQ_F_LINK_TIMEOUT;
1642 1643 1644 1645
	return wake_ev;
}

static void io_kill_linked_timeout(struct io_kiocb *req)
J
Jens Axboe 已提交
1646
{
1647
	struct io_ring_ctx *ctx = req->ctx;
1648
	bool wake_ev;
J
Jens Axboe 已提交
1649

1650 1651 1652 1653 1654
	if (!(req->flags & REQ_F_COMP_LOCKED)) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->completion_lock, flags);
		wake_ev = __io_kill_linked_timeout(req);
1655
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
1656 1657 1658 1659
	} else {
		wake_ev = __io_kill_linked_timeout(req);
	}

1660 1661 1662 1663
	if (wake_ev)
		io_cqring_ev_posted(ctx);
}

1664
static struct io_kiocb *io_req_link_next(struct io_kiocb *req)
1665 1666
{
	struct io_kiocb *nxt;
1667

J
Jens Axboe 已提交
1668 1669 1670 1671 1672
	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
1673
	if (unlikely(list_empty(&req->link_list)))
1674
		return NULL;
1675

1676 1677 1678 1679
	nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	list_del_init(&req->link_list);
	if (!list_empty(&nxt->link_list))
		nxt->flags |= REQ_F_LINK_HEAD;
1680
	return nxt;
J
Jens Axboe 已提交
1681 1682 1683
}

/*
1684
 * Called if REQ_F_LINK_HEAD is set, and we fail the head request
J
Jens Axboe 已提交
1685
 */
1686
static void __io_fail_links(struct io_kiocb *req)
J
Jens Axboe 已提交
1687
{
1688
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1689 1690

	while (!list_empty(&req->link_list)) {
1691 1692
		struct io_kiocb *link = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
J
Jens Axboe 已提交
1693

1694
		list_del_init(&link->link_list);
1695
		trace_io_uring_fail_link(req, link);
1696

1697
		io_cqring_fill_event(link, -ECANCELED);
1698
		link->flags |= REQ_F_COMP_LOCKED;
1699
		__io_double_put_req(link);
1700
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
1701
	}
1702 1703 1704

	io_commit_cqring(ctx);
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1705 1706
}

1707
static void io_fail_links(struct io_kiocb *req)
J
Jens Axboe 已提交
1708
{
1709
	struct io_ring_ctx *ctx = req->ctx;
1710

1711
	if (!(req->flags & REQ_F_COMP_LOCKED)) {
1712 1713 1714
		unsigned long flags;

		spin_lock_irqsave(&ctx->completion_lock, flags);
1715
		__io_fail_links(req);
1716 1717
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
1718
		__io_fail_links(req);
J
Jens Axboe 已提交
1719 1720
	}

1721
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1722 1723
}

1724
static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
1725
{
P
Pavel Begunkov 已提交
1726
	req->flags &= ~REQ_F_LINK_HEAD;
1727 1728
	if (req->flags & REQ_F_LINK_TIMEOUT)
		io_kill_linked_timeout(req);
1729

J
Jens Axboe 已提交
1730 1731 1732 1733 1734 1735
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
1736 1737 1738 1739
	if (likely(!(req->flags & REQ_F_FAIL_LINK)))
		return io_req_link_next(req);
	io_fail_links(req);
	return NULL;
1740
}
J
Jens Axboe 已提交
1741

1742 1743 1744 1745 1746 1747 1748
static struct io_kiocb *io_req_find_next(struct io_kiocb *req)
{
	if (likely(!(req->flags & REQ_F_LINK_HEAD)))
		return NULL;
	return __io_req_find_next(req);
}

1749 1750 1751 1752
static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb)
{
	struct task_struct *tsk = req->task;
	struct io_ring_ctx *ctx = req->ctx;
1753
	int ret, notify;
1754 1755

	/*
1756 1757 1758 1759
	 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
	 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
	 * processing task_work. There's no reliable way to tell if TWA_RESUME
	 * will do the job.
1760
	 */
1761 1762
	notify = 0;
	if (!(ctx->flags & IORING_SETUP_SQPOLL))
1763 1764 1765 1766 1767
		notify = TWA_SIGNAL;

	ret = task_work_add(tsk, cb, notify);
	if (!ret)
		wake_up_process(tsk);
1768

1769 1770 1771
	return ret;
}

1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
static void __io_req_task_cancel(struct io_kiocb *req, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	io_cqring_fill_event(req, error);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	req_set_fail_links(req);
	io_double_put_req(req);
}

static void io_req_task_cancel(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);

	__io_req_task_cancel(req, -ECANCELED);
}

static void __io_req_task_submit(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!__io_sq_thread_acquire_mm(ctx)) {
		mutex_lock(&ctx->uring_lock);
		__io_queue_sqe(req, NULL, NULL);
		mutex_unlock(&ctx->uring_lock);
	} else {
		__io_req_task_cancel(req, -EFAULT);
	}
}

static void io_req_task_submit(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
1809
	struct io_ring_ctx *ctx = req->ctx;
1810 1811

	__io_req_task_submit(req);
1812
	percpu_ref_put(&ctx->refs);
1813 1814 1815 1816 1817 1818 1819
}

static void io_req_task_queue(struct io_kiocb *req)
{
	int ret;

	init_task_work(&req->task_work, io_req_task_submit);
1820
	percpu_ref_get(&req->ctx->refs);
1821

1822
	ret = io_req_task_work_add(req, &req->task_work);
1823
	if (unlikely(ret)) {
1824 1825
		struct task_struct *tsk;

1826 1827
		init_task_work(&req->task_work, io_req_task_cancel);
		tsk = io_wq_get_task(req->ctx->io_wq);
1828 1829
		task_work_add(tsk, &req->task_work, 0);
		wake_up_process(tsk);
1830 1831 1832
	}
}

1833
static void io_queue_next(struct io_kiocb *req)
1834
{
1835
	struct io_kiocb *nxt = io_req_find_next(req);
1836 1837

	if (nxt)
1838
		io_req_task_queue(nxt);
1839 1840
}

1841
static void io_free_req(struct io_kiocb *req)
1842
{
1843 1844 1845
	io_queue_next(req);
	__io_free_req(req);
}
1846

1847 1848 1849
struct req_batch {
	void *reqs[IO_IOPOLL_BATCH];
	int to_free;
1850

1851 1852
	struct task_struct	*task;
	int			task_refs;
1853 1854
};

1855 1856 1857 1858 1859 1860 1861
static inline void io_init_req_batch(struct req_batch *rb)
{
	rb->to_free = 0;
	rb->task_refs = 0;
	rb->task = NULL;
}

1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
static void __io_req_free_batch_flush(struct io_ring_ctx *ctx,
				      struct req_batch *rb)
{
	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
	percpu_ref_put_many(&ctx->refs, rb->to_free);
	rb->to_free = 0;
}

static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
				     struct req_batch *rb)
{
	if (rb->to_free)
		__io_req_free_batch_flush(ctx, rb);
1875 1876 1877 1878
	if (rb->task) {
		put_task_struct_many(rb->task, rb->task_refs);
		rb->task = NULL;
	}
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
}

static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req)
{
	if (unlikely(io_is_fallback_req(req))) {
		io_free_req(req);
		return;
	}
	if (req->flags & REQ_F_LINK_HEAD)
		io_queue_next(req);

1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
	if (req->flags & REQ_F_TASK_PINNED) {
		if (req->task != rb->task) {
			if (rb->task)
				put_task_struct_many(rb->task, rb->task_refs);
			rb->task = req->task;
			rb->task_refs = 0;
		}
		rb->task_refs++;
		req->flags &= ~REQ_F_TASK_PINNED;
	}

1901
	WARN_ON_ONCE(io_dismantle_req(req));
1902 1903 1904
	rb->reqs[rb->to_free++] = req;
	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		__io_req_free_batch_flush(req->ctx, rb);
1905 1906
}

1907 1908 1909 1910
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
1911
static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
1912
{
1913 1914
	struct io_kiocb *nxt = NULL;

1915
	if (refcount_dec_and_test(&req->refs)) {
1916
		nxt = io_req_find_next(req);
1917
		__io_free_req(req);
1918
	}
1919
	return nxt;
J
Jens Axboe 已提交
1920 1921
}

1922 1923 1924 1925
static void io_put_req(struct io_kiocb *req)
{
	if (refcount_dec_and_test(&req->refs))
		io_free_req(req);
J
Jens Axboe 已提交
1926 1927
}

1928
static struct io_wq_work *io_steal_work(struct io_kiocb *req)
1929
{
1930
	struct io_kiocb *nxt;
1931

1932
	/*
1933 1934 1935
	 * A ref is owned by io-wq in which context we're. So, if that's the
	 * last one, it's safe to steal next work. False negatives are Ok,
	 * it just will be re-punted async in io_put_work()
1936
	 */
1937 1938
	if (refcount_read(&req->refs) != 1)
		return NULL;
1939

1940
	nxt = io_req_find_next(req);
1941
	return nxt ? &nxt->work : NULL;
1942 1943
}

1944 1945 1946 1947 1948
/*
 * Must only be used if we don't need to care about links, usually from
 * within the completion handling itself.
 */
static void __io_double_put_req(struct io_kiocb *req)
1949 1950 1951 1952 1953 1954
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		__io_free_req(req);
}

1955 1956 1957 1958 1959 1960 1961
static void io_double_put_req(struct io_kiocb *req)
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		io_free_req(req);
}

1962
static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
1963
{
1964 1965
	struct io_rings *rings = ctx->rings;

1966 1967 1968 1969 1970 1971 1972 1973
	if (test_bit(0, &ctx->cq_check_overflow)) {
		/*
		 * noflush == true is from the waitqueue handler, just ensure
		 * we wake up the task, and the next invocation will flush the
		 * entries. We cannot safely to it from here.
		 */
		if (noflush && !list_empty(&ctx->cq_overflow_list))
			return -1U;
1974

1975 1976
		io_cqring_overflow_flush(ctx, false);
	}
1977

1978 1979
	/* See comment at the top of this file */
	smp_rmb();
1980
	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
1981 1982
}

1983 1984 1985 1986 1987 1988 1989 1990
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

1991
static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
1992
{
1993
	unsigned int cflags;
1994

1995 1996
	cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	cflags |= IORING_CQE_F_BUFFER;
1997
	req->flags &= ~REQ_F_BUFFER_SELECTED;
1998 1999
	kfree(kbuf);
	return cflags;
2000 2001
}

2002
static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2003
{
2004
	struct io_buffer *kbuf;
2005

2006
	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2007 2008 2009
	return io_put_kbuf(req, kbuf);
}

2010 2011 2012 2013 2014 2015 2016 2017 2018
static inline bool io_run_task_work(void)
{
	if (current->task_works) {
		__set_current_state(TASK_RUNNING);
		task_work_run();
		return true;
	}

	return false;
2019 2020
}

2021 2022 2023 2024 2025
static void io_iopoll_queue(struct list_head *again)
{
	struct io_kiocb *req;

	do {
2026 2027
		req = list_first_entry(again, struct io_kiocb, inflight_entry);
		list_del(&req->inflight_entry);
2028
		__io_complete_rw(req, -EAGAIN, 0, NULL);
2029 2030 2031
	} while (!list_empty(again));
}

J
Jens Axboe 已提交
2032 2033 2034 2035 2036 2037
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
2038
	struct req_batch rb;
J
Jens Axboe 已提交
2039
	struct io_kiocb *req;
2040 2041 2042 2043
	LIST_HEAD(again);

	/* order with ->result store in io_complete_rw_iopoll() */
	smp_rmb();
J
Jens Axboe 已提交
2044

2045
	io_init_req_batch(&rb);
J
Jens Axboe 已提交
2046
	while (!list_empty(done)) {
2047 2048
		int cflags = 0;

2049
		req = list_first_entry(done, struct io_kiocb, inflight_entry);
2050 2051
		if (READ_ONCE(req->result) == -EAGAIN) {
			req->iopoll_completed = 0;
2052
			list_move_tail(&req->inflight_entry, &again);
2053 2054
			continue;
		}
2055
		list_del(&req->inflight_entry);
J
Jens Axboe 已提交
2056

2057
		if (req->flags & REQ_F_BUFFER_SELECTED)
2058
			cflags = io_put_rw_kbuf(req);
2059 2060

		__io_cqring_fill_event(req, req->result, cflags);
J
Jens Axboe 已提交
2061 2062
		(*nr_events)++;

2063
		if (refcount_dec_and_test(&req->refs))
2064
			io_req_free_batch(&rb, req);
J
Jens Axboe 已提交
2065 2066
	}

J
Jens Axboe 已提交
2067
	io_commit_cqring(ctx);
2068 2069
	if (ctx->flags & IORING_SETUP_SQPOLL)
		io_cqring_ev_posted(ctx);
2070
	io_req_free_batch_finish(ctx, &rb);
2071

2072 2073
	if (!list_empty(&again))
		io_iopoll_queue(&again);
2074 2075
}

J
Jens Axboe 已提交
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
2091
	list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
2092
		struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2093 2094

		/*
2095 2096 2097
		 * Move completed and retryable entries to our local lists.
		 * If we find a request that requires polling, break out
		 * and complete those lists first, if we have entries there.
J
Jens Axboe 已提交
2098
		 */
2099
		if (READ_ONCE(req->iopoll_completed)) {
2100
			list_move_tail(&req->inflight_entry, &done);
J
Jens Axboe 已提交
2101 2102 2103 2104 2105 2106 2107 2108 2109
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

2110 2111
		/* iopoll may have completed current req */
		if (READ_ONCE(req->iopoll_completed))
2112
			list_move_tail(&req->inflight_entry, &done);
2113

J
Jens Axboe 已提交
2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
2126
 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
J
Jens Axboe 已提交
2127 2128 2129 2130 2131 2132
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
2133
	while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
J
Jens Axboe 已提交
2134 2135 2136 2137 2138
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
2139
		if (*nr_events >= min)
J
Jens Axboe 已提交
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
2150
static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
2151 2152 2153 2154 2155
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
2156
	while (!list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
2157 2158
		unsigned int nr_events = 0;

2159
		io_do_iopoll(ctx, &nr_events, 0);
2160

2161 2162 2163
		/* let it sleep and repeat later if can't complete a request */
		if (nr_events == 0)
			break;
2164 2165 2166
		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
2167
		 * Also let task_work, etc. to progress by releasing the mutex
2168
		 */
2169 2170 2171 2172 2173
		if (need_resched()) {
			mutex_unlock(&ctx->uring_lock);
			cond_resched();
			mutex_lock(&ctx->uring_lock);
		}
J
Jens Axboe 已提交
2174 2175 2176 2177
	}
	mutex_unlock(&ctx->uring_lock);
}

2178
static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
J
Jens Axboe 已提交
2179
{
2180
	unsigned int nr_events = 0;
2181
	int iters = 0, ret = 0;
2182

2183 2184 2185 2186 2187 2188
	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
J
Jens Axboe 已提交
2189
	do {
2190 2191 2192 2193 2194
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
2195
		if (io_cqring_events(ctx, false))
2196 2197
			break;

2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
2210
			io_run_task_work();
2211 2212 2213
			mutex_lock(&ctx->uring_lock);
		}

2214
		ret = io_iopoll_getevents(ctx, &nr_events, min);
J
Jens Axboe 已提交
2215 2216 2217
		if (ret <= 0)
			break;
		ret = 0;
2218
	} while (min && !nr_events && !need_resched());
J
Jens Axboe 已提交
2219

2220
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
2221 2222 2223
	return ret;
}

2224
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
2225
{
2226 2227 2228 2229 2230 2231
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
2232

2233
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
2234
	}
2235
	file_end_write(req->file);
J
Jens Axboe 已提交
2236 2237
}

2238 2239
static void io_complete_rw_common(struct kiocb *kiocb, long res,
				  struct io_comp_state *cs)
J
Jens Axboe 已提交
2240
{
2241
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2242
	int cflags = 0;
J
Jens Axboe 已提交
2243

2244 2245
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
2246

J
Jens Axboe 已提交
2247 2248
	if (res != req->result)
		req_set_fail_links(req);
2249
	if (req->flags & REQ_F_BUFFER_SELECTED)
2250
		cflags = io_put_rw_kbuf(req);
2251
	__io_req_complete(req, res, cflags, cs);
2252 2253
}

2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
#ifdef CONFIG_BLOCK
static bool io_resubmit_prep(struct io_kiocb *req, int error)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	ssize_t ret = -ECANCELED;
	struct iov_iter iter;
	int rw;

	if (error) {
		ret = error;
		goto end_req;
	}

	switch (req->opcode) {
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
		rw = READ;
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		rw = WRITE;
		break;
	default:
		printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
				req->opcode);
		goto end_req;
	}

	ret = io_import_iovec(rw, req, &iovec, &iter, false);
	if (ret < 0)
		goto end_req;
2287
	ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false);
2288 2289 2290 2291 2292
	if (!ret)
		return true;
	kfree(iovec);
end_req:
	req_set_fail_links(req);
2293
	io_req_complete(req, ret);
2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
	return false;
}

static void io_rw_resubmit(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct io_ring_ctx *ctx = req->ctx;
	int err;

	err = io_sq_thread_acquire_mm(ctx, req);

	if (io_resubmit_prep(req, err)) {
		refcount_inc(&req->refs);
		io_queue_async_work(req);
	}
2309 2310

	percpu_ref_put(&ctx->refs);
2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322
}
#endif

static bool io_rw_reissue(struct io_kiocb *req, long res)
{
#ifdef CONFIG_BLOCK
	int ret;

	if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker())
		return false;

	init_task_work(&req->task_work, io_rw_resubmit);
2323 2324
	percpu_ref_get(&req->ctx->refs);

2325 2326
	ret = io_req_task_work_add(req, &req->task_work);
	if (!ret)
2327 2328 2329 2330 2331
		return true;
#endif
	return false;
}

2332 2333 2334 2335 2336
static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
			     struct io_comp_state *cs)
{
	if (!io_rw_reissue(req, res))
		io_complete_rw_common(&req->rw.kiocb, res, cs);
2337 2338 2339 2340
}

static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
2341
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2342

2343
	__io_complete_rw(req, res, res2, NULL);
J
Jens Axboe 已提交
2344 2345
}

J
Jens Axboe 已提交
2346 2347
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
2348
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
2349

2350 2351
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
2352

2353
	if (res != -EAGAIN && res != req->result)
J
Jens Axboe 已提交
2354
		req_set_fail_links(req);
2355 2356 2357

	WRITE_ONCE(req->result, res);
	/* order with io_poll_complete() checking ->result */
2358 2359
	smp_wmb();
	WRITE_ONCE(req->iopoll_completed, 1);
J
Jens Axboe 已提交
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
2377
	if (list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
2378 2379 2380 2381
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

2382
		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
2383
						inflight_entry);
2384
		if (list_req->file != req->file)
J
Jens Axboe 已提交
2385 2386 2387 2388 2389 2390 2391
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
2392
	if (READ_ONCE(req->iopoll_completed))
2393
		list_add(&req->inflight_entry, &ctx->iopoll_list);
J
Jens Axboe 已提交
2394
	else
2395
		list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
2396 2397 2398 2399

	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
	    wq_has_sleeper(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
J
Jens Axboe 已提交
2400 2401
}

P
Pavel Begunkov 已提交
2402
static void __io_state_file_put(struct io_submit_state *state)
2403
{
2404 2405
	if (state->has_refs)
		fput_many(state->file, state->has_refs);
P
Pavel Begunkov 已提交
2406 2407 2408 2409 2410 2411 2412
	state->file = NULL;
}

static inline void io_state_file_put(struct io_submit_state *state)
{
	if (state->file)
		__io_state_file_put(state);
2413 2414 2415 2416 2417 2418 2419
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
2420
static struct file *__io_file_get(struct io_submit_state *state, int fd)
2421 2422 2423 2424 2425 2426
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
2427
			state->has_refs--;
2428 2429 2430
			state->ios_left--;
			return state->file;
		}
P
Pavel Begunkov 已提交
2431
		__io_state_file_put(state);
2432 2433 2434 2435 2436 2437 2438
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->ios_left--;
2439
	state->has_refs = state->ios_left;
2440 2441 2442
	return state->file;
}

2443 2444 2445 2446 2447 2448 2449 2450 2451
static bool io_bdev_nowait(struct block_device *bdev)
{
#ifdef CONFIG_BLOCK
	return !bdev || queue_is_mq(bdev_get_queue(bdev));
#else
	return true;
#endif
}

J
Jens Axboe 已提交
2452 2453 2454 2455 2456
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
2457
static bool io_file_supports_async(struct file *file, int rw)
J
Jens Axboe 已提交
2458 2459 2460
{
	umode_t mode = file_inode(file)->i_mode;

2461 2462 2463 2464 2465 2466
	if (S_ISBLK(mode)) {
		if (io_bdev_nowait(file->f_inode->i_bdev))
			return true;
		return false;
	}
	if (S_ISCHR(mode) || S_ISSOCK(mode))
J
Jens Axboe 已提交
2467
		return true;
2468 2469 2470 2471 2472 2473
	if (S_ISREG(mode)) {
		if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
		    file->f_op != &io_uring_fops)
			return true;
		return false;
	}
J
Jens Axboe 已提交
2474

2475 2476 2477 2478
	/* any ->read/write should understand O_NONBLOCK */
	if (file->f_flags & O_NONBLOCK)
		return true;

2479 2480 2481 2482 2483 2484 2485
	if (!(file->f_mode & FMODE_NOWAIT))
		return false;

	if (rw == READ)
		return file->f_op->read_iter != NULL;

	return file->f_op->write_iter != NULL;
J
Jens Axboe 已提交
2486 2487
}

2488 2489
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		      bool force_nonblock)
J
Jens Axboe 已提交
2490
{
J
Jens Axboe 已提交
2491
	struct io_ring_ctx *ctx = req->ctx;
2492
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2493 2494
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
2495

2496 2497 2498
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

J
Jens Axboe 已提交
2499
	kiocb->ki_pos = READ_ONCE(sqe->off);
2500 2501 2502 2503
	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
		req->flags |= REQ_F_CUR_POS;
		kiocb->ki_pos = req->file->f_pos;
	}
J
Jens Axboe 已提交
2504
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
2505 2506 2507 2508
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
		return ret;
J
Jens Axboe 已提交
2509 2510 2511 2512 2513

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
2514
			return ret;
J
Jens Axboe 已提交
2515 2516 2517 2518 2519

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

2520
	/* don't allow async punt if RWF_NOWAIT was requested */
2521
	if (kiocb->ki_flags & IOCB_NOWAIT)
2522 2523
		req->flags |= REQ_F_NOWAIT;

2524 2525 2526
	if (kiocb->ki_flags & IOCB_DIRECT)
		io_get_req_task(req);

2527
	if (force_nonblock)
J
Jens Axboe 已提交
2528
		kiocb->ki_flags |= IOCB_NOWAIT;
2529

J
Jens Axboe 已提交
2530 2531 2532
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
2533
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
2534

J
Jens Axboe 已提交
2535 2536
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
2537
		req->iopoll_completed = 0;
2538
		io_get_req_task(req);
J
Jens Axboe 已提交
2539
	} else {
J
Jens Axboe 已提交
2540 2541
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
2542 2543
		kiocb->ki_complete = io_complete_rw;
	}
2544

2545 2546
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
2547
	req->buf_index = READ_ONCE(sqe->buf_index);
J
Jens Axboe 已提交
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

2572 2573
static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
		       struct io_comp_state *cs)
2574
{
2575 2576
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

2577 2578 2579 2580 2581 2582 2583 2584
	/* add previously done IO, if any */
	if (req->io && req->io->rw.bytes_done > 0) {
		if (ret < 0)
			ret = req->io->rw.bytes_done;
		else
			ret += req->io->rw.bytes_done;
	}

2585 2586
	if (req->flags & REQ_F_CUR_POS)
		req->file->f_pos = kiocb->ki_pos;
P
Pavel Begunkov 已提交
2587
	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
2588
		__io_complete_rw(req, ret, 0, cs);
2589 2590 2591 2592
	else
		io_rw_done(kiocb, ret);
}

2593
static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
2594
			       struct iov_iter *iter)
2595
{
2596 2597
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
2598
	struct io_mapped_ubuf *imu;
2599
	u16 index, buf_index;
2600 2601 2602 2603 2604 2605 2606
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

2607
	buf_index = req->buf_index;
2608 2609 2610 2611 2612
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
2613
	buf_addr = req->rw.addr;
2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
2659
			iter->count -= bvec->bv_len + offset;
2660 2661 2662 2663
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

2664
	return len;
2665 2666
}

2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
{
	if (needs_lock)
		mutex_unlock(&ctx->uring_lock);
}

static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
{
	/*
	 * "Normal" inline submissions always hold the uring_lock, since we
	 * grab it from the system call. Same is true for the SQPOLL offload.
	 * The only exception is when we've detached the request and issue it
	 * from an async worker thread, grab the lock for that case.
	 */
	if (needs_lock)
		mutex_lock(&ctx->uring_lock);
}

static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
					  int bgid, struct io_buffer *kbuf,
					  bool needs_lock)
{
	struct io_buffer *head;

	if (req->flags & REQ_F_BUFFER_SELECTED)
		return kbuf;

	io_ring_submit_lock(req->ctx, needs_lock);

	lockdep_assert_held(&req->ctx->uring_lock);

	head = idr_find(&req->ctx->io_buffer_idr, bgid);
	if (head) {
		if (!list_empty(&head->list)) {
			kbuf = list_last_entry(&head->list, struct io_buffer,
							list);
			list_del(&kbuf->list);
		} else {
			kbuf = head;
			idr_remove(&req->ctx->io_buffer_idr, bgid);
		}
		if (*len > kbuf->len)
			*len = kbuf->len;
	} else {
		kbuf = ERR_PTR(-ENOBUFS);
	}

	io_ring_submit_unlock(req->ctx, needs_lock);

	return kbuf;
}

2719 2720 2721 2722
static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
					bool needs_lock)
{
	struct io_buffer *kbuf;
2723
	u16 bgid;
2724 2725

	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2726
	bgid = req->buf_index;
2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
	kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;
	req->rw.addr = (u64) (unsigned long) kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;
	return u64_to_user_ptr(kbuf->addr);
}

#ifdef CONFIG_COMPAT
static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
				bool needs_lock)
{
	struct compat_iovec __user *uiov;
	compat_ssize_t clen;
	void __user *buf;
	ssize_t len;

	uiov = u64_to_user_ptr(req->rw.addr);
	if (!access_ok(uiov, sizeof(*uiov)))
		return -EFAULT;
	if (__get_user(clen, &uiov->iov_len))
		return -EFAULT;
	if (clen < 0)
		return -EINVAL;

	len = clen;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = (compat_size_t) len;
	return 0;
}
#endif

static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				      bool needs_lock)
{
	struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
	void __user *buf;
	ssize_t len;

	if (copy_from_user(iov, uiov, sizeof(*uiov)))
		return -EFAULT;

	len = iov[0].iov_len;
	if (len < 0)
		return -EINVAL;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = len;
	return 0;
}

static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				    bool needs_lock)
{
2786 2787 2788 2789 2790 2791
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		struct io_buffer *kbuf;

		kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
		iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		iov[0].iov_len = kbuf->len;
2792
		return 0;
2793
	}
2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806
	if (!req->rw.len)
		return 0;
	else if (req->rw.len > 1)
		return -EINVAL;

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		return io_compat_import(req, iov, needs_lock);
#endif

	return __io_iov_buffer_select(req, iov, needs_lock);
}

2807 2808 2809
static ssize_t __io_import_iovec(int rw, struct io_kiocb *req,
				 struct iovec **iovec, struct iov_iter *iter,
				 bool needs_lock)
J
Jens Axboe 已提交
2810
{
2811 2812
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
2813
	ssize_t ret;
2814 2815
	u8 opcode;

2816
	opcode = req->opcode;
2817
	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2818
		*iovec = NULL;
2819
		return io_import_fixed(req, rw, iter);
2820
	}
J
Jens Axboe 已提交
2821

2822
	/* buffer index only valid with fixed read/write, or buffer select  */
2823
	if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
2824 2825
		return -EINVAL;

2826
	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2827
		if (req->flags & REQ_F_BUFFER_SELECT) {
2828
			buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
2829
			if (IS_ERR(buf))
2830
				return PTR_ERR(buf);
2831
			req->rw.len = sqe_len;
2832 2833
		}

2834 2835
		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
2836
		return ret < 0 ? ret : sqe_len;
2837 2838
	}

2839 2840
	if (req->flags & REQ_F_BUFFER_SELECT) {
		ret = io_iov_buffer_select(req, *iovec, needs_lock);
2841 2842 2843 2844
		if (!ret) {
			ret = (*iovec)->iov_len;
			iov_iter_init(iter, rw, *iovec, 1, ret);
		}
2845 2846 2847 2848
		*iovec = NULL;
		return ret;
	}

J
Jens Axboe 已提交
2849
#ifdef CONFIG_COMPAT
2850
	if (req->ctx->compat)
J
Jens Axboe 已提交
2851 2852 2853 2854 2855 2856 2857
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

2858 2859 2860 2861 2862 2863 2864 2865 2866 2867
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock)
{
	if (!req->io)
		return __io_import_iovec(rw, req, iovec, iter, needs_lock);
	*iovec = NULL;
	return iov_iter_count(&req->io->rw.iter);
}

2868
/*
2869 2870
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
2871
 */
2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
2888
		struct iovec iovec;
2889 2890
		ssize_t nr;

2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

2901 2902 2903 2904 2905 2906 2907 2908
		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, &kiocb->ki_pos);
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, &kiocb->ki_pos);
		}

2909 2910 2911
		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925
		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

2926 2927
static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
			  const struct iovec *fast_iov, struct iov_iter *iter)
2928
{
2929 2930
	struct io_async_rw *rw = &req->io->rw;

2931 2932
	memcpy(&rw->iter, iter, sizeof(*iter));
	rw->free_iovec = NULL;
2933
	rw->bytes_done = 0;
2934 2935 2936
	/* can only be fixed buffers, no need to do anything */
	if (iter->type == ITER_BVEC)
		return;
2937
	if (!iovec) {
2938 2939 2940 2941 2942 2943 2944 2945 2946
		unsigned iov_off = 0;

		rw->iter.iov = rw->fast_iov;
		if (iter->iov != fast_iov) {
			iov_off = iter->iov - fast_iov;
			rw->iter.iov += iov_off;
		}
		if (rw->fast_iov != fast_iov)
			memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
2947
			       sizeof(struct iovec) * iter->nr_segs);
P
Pavel Begunkov 已提交
2948
	} else {
2949
		rw->free_iovec = iovec;
P
Pavel Begunkov 已提交
2950
		req->flags |= REQ_F_NEED_CLEANUP;
2951 2952 2953
	}
}

2954 2955 2956 2957 2958 2959
static inline int __io_alloc_async_ctx(struct io_kiocb *req)
{
	req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
	return req->io == NULL;
}

2960
static int io_alloc_async_ctx(struct io_kiocb *req)
2961
{
2962 2963
	if (!io_op_defs[req->opcode].async_ctx)
		return 0;
2964 2965

	return  __io_alloc_async_ctx(req);
2966 2967
}

2968 2969
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
			     const struct iovec *fast_iov,
2970
			     struct iov_iter *iter, bool force)
2971
{
2972
	if (!force && !io_op_defs[req->opcode].async_ctx)
2973
		return 0;
2974
	if (!req->io) {
2975
		if (__io_alloc_async_ctx(req))
2976
			return -ENOMEM;
2977

2978
		io_req_map_rw(req, iovec, fast_iov, iter);
2979
	}
2980
	return 0;
2981 2982
}

2983 2984 2985
static inline int io_rw_prep_async(struct io_kiocb *req, int rw,
				   bool force_nonblock)
{
2986
	struct io_async_rw *iorw = &req->io->rw;
2987 2988
	ssize_t ret;

2989
	iorw->iter.iov = iorw->fast_iov;
2990
	ret = __io_import_iovec(rw, req, (struct iovec **) &iorw->iter.iov,
2991
				&iorw->iter, !force_nonblock);
2992 2993 2994
	if (unlikely(ret < 0))
		return ret;

2995
	io_req_map_rw(req, iorw->iter.iov, iorw->fast_iov, &iorw->iter);
2996 2997 2998
	return 0;
}

2999 3000
static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
3001 3002 3003
{
	ssize_t ret;

3004 3005 3006
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
3007

3008 3009
	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;
3010

3011 3012
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
3013
		return 0;
3014
	return io_rw_prep_async(req, READ, force_nonblock);
3015 3016
}

3017 3018 3019 3020 3021 3022 3023 3024 3025 3026
/*
 * This is our waitqueue callback handler, registered through lock_page_async()
 * when we initially tried to do the IO with the iocb armed our waitqueue.
 * This gets called when the page is unlocked, and we generally expect that to
 * happen when the page IO is completed and the page is now uptodate. This will
 * queue a task_work based retry of the operation, attempting to copy the data
 * again. If the latter fails because the page was NOT uptodate, then we will
 * do a thread based blocking retry of the operation. That's the unexpected
 * slow path.
 */
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036
static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
			     int sync, void *arg)
{
	struct wait_page_queue *wpq;
	struct io_kiocb *req = wait->private;
	struct wait_page_key *key = arg;
	int ret;

	wpq = container_of(wait, struct wait_page_queue, wait);

3037 3038 3039
	if (!wake_page_match(wpq, key))
		return 0;

3040 3041
	list_del_init(&wait->entry);

3042
	init_task_work(&req->task_work, io_req_task_submit);
3043 3044
	percpu_ref_get(&req->ctx->refs);

3045 3046
	/* submit ref gets dropped, acquire a new one */
	refcount_inc(&req->refs);
3047
	ret = io_req_task_work_add(req, &req->task_work);
3048
	if (unlikely(ret)) {
3049 3050
		struct task_struct *tsk;

3051
		/* queue just for cancelation */
3052
		init_task_work(&req->task_work, io_req_task_cancel);
3053
		tsk = io_wq_get_task(req->ctx->io_wq);
3054
		task_work_add(tsk, &req->task_work, 0);
3055
		wake_up_process(tsk);
3056 3057 3058 3059
	}
	return 1;
}

3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
/*
 * This controls whether a given IO request should be armed for async page
 * based retry. If we return false here, the request is handed to the async
 * worker threads for retry. If we're doing buffered reads on a regular file,
 * we prepare a private wait_page_queue entry and retry the operation. This
 * will either succeed because the page is now uptodate and unlocked, or it
 * will register a callback when the page is unlocked at IO completion. Through
 * that callback, io_uring uses task_work to setup a retry of the operation.
 * That retry will attempt the buffered read again. The retry will generally
 * succeed, or in rare cases where it fails, we then fall back to using the
 * async worker threads for a blocking retry.
 */
3072
static bool io_rw_should_retry(struct io_kiocb *req)
3073
{
3074
	struct wait_page_queue *wait = &req->io->rw.wpq;
3075
	struct kiocb *kiocb = &req->rw.kiocb;
3076

3077 3078 3079
	/* never retry for NOWAIT, we just complete with -EAGAIN */
	if (req->flags & REQ_F_NOWAIT)
		return false;
3080

3081
	/* Only for buffered IO */
3082
	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
3083
		return false;
3084

3085 3086 3087 3088 3089 3090
	/*
	 * just use poll if we can, and don't attempt if the fs doesn't
	 * support callback based unlocks
	 */
	if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
		return false;
3091

3092 3093 3094 3095 3096 3097
	wait->wait.func = io_async_buf_func;
	wait->wait.private = req;
	wait->wait.flags = 0;
	INIT_LIST_HEAD(&wait->wait.entry);
	kiocb->ki_flags |= IOCB_WAITQ;
	kiocb->ki_waitq = wait;
3098

3099 3100
	io_get_req_task(req);
	return true;
3101 3102 3103 3104 3105 3106
}

static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (req->file->f_op->read_iter)
		return call_read_iter(req->file, &req->rw.kiocb, iter);
3107 3108 3109 3110
	else if (req->file->f_op->read)
		return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
	else
		return -EINVAL;
3111 3112
}

3113 3114
static int io_read(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
J
Jens Axboe 已提交
3115 3116
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3117
	struct kiocb *kiocb = &req->rw.kiocb;
3118
	struct iov_iter __iter, *iter = &__iter;
3119
	ssize_t io_size, ret, ret2;
3120
	size_t iov_count;
3121 3122 3123

	if (req->io)
		iter = &req->io->rw.iter;
J
Jens Axboe 已提交
3124

3125
	ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3126 3127
	if (ret < 0)
		return ret;
3128 3129
	io_size = ret;
	req->result = io_size;
3130
	ret = 0;
J
Jens Axboe 已提交
3131

3132 3133
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
3134
		kiocb->ki_flags &= ~IOCB_NOWAIT;
3135

3136
	/* If the file doesn't support async, just async punt */
3137
	if (force_nonblock && !io_file_supports_async(req->file, READ))
3138
		goto copy_iov;
J
Jens Axboe 已提交
3139

3140
	iov_count = iov_iter_count(iter);
3141
	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
3142 3143
	if (unlikely(ret))
		goto out_free;
J
Jens Axboe 已提交
3144

3145
	ret = io_iter_do_read(req, iter);
3146

3147 3148 3149 3150 3151 3152
	if (!ret) {
		goto done;
	} else if (ret == -EIOCBQUEUED) {
		ret = 0;
		goto out_free;
	} else if (ret == -EAGAIN) {
3153 3154
		if (!force_nonblock)
			goto done;
3155
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
3156 3157
		if (ret)
			goto out_free;
3158 3159 3160 3161 3162 3163
		return -EAGAIN;
	} else if (ret < 0) {
		goto out_free;
	}

	/* read it all, or we did blocking attempt. no retry. */
3164 3165
	if (!iov_iter_count(iter) || !force_nonblock ||
	    (req->file->f_flags & O_NONBLOCK))
3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182
		goto done;

	io_size -= ret;
copy_iov:
	ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
	if (ret2) {
		ret = ret2;
		goto out_free;
	}
	/* it's copied and will be cleaned with ->io */
	iovec = NULL;
	/* now use our persistent iterator, if we aren't already */
	iter = &req->io->rw.iter;
retry:
	req->io->rw.bytes_done += ret;
	/* if we can retry, do so with the callbacks armed */
	if (!io_rw_should_retry(req)) {
3183 3184
		kiocb->ki_flags &= ~IOCB_WAITQ;
		return -EAGAIN;
J
Jens Axboe 已提交
3185
	}
3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203

	/*
	 * Now retry read with the IOCB_WAITQ parts set in the iocb. If we
	 * get -EIOCBQUEUED, then we'll get a notification when the desired
	 * page gets unlocked. We can also get a partial read here, and if we
	 * do, then just retry at the new offset.
	 */
	ret = io_iter_do_read(req, iter);
	if (ret == -EIOCBQUEUED) {
		ret = 0;
		goto out_free;
	} else if (ret > 0 && ret < io_size) {
		/* we got some bytes, but not all. retry. */
		goto retry;
	}
done:
	kiocb_done(kiocb, ret, cs);
	ret = 0;
3204
out_free:
3205
	/* it's reportedly faster than delegating the null check to kfree() */
3206
	if (iovec)
3207
		kfree(iovec);
J
Jens Axboe 已提交
3208 3209 3210
	return ret;
}

3211 3212
static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 bool force_nonblock)
3213 3214 3215
{
	ssize_t ret;

3216 3217 3218
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
3219

3220 3221
	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
		return -EBADF;
3222

3223 3224
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
3225
		return 0;
3226
	return io_rw_prep_async(req, WRITE, force_nonblock);
3227 3228
}

3229 3230
static int io_write(struct io_kiocb *req, bool force_nonblock,
		    struct io_comp_state *cs)
J
Jens Axboe 已提交
3231 3232
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3233
	struct kiocb *kiocb = &req->rw.kiocb;
3234
	struct iov_iter __iter, *iter = &__iter;
3235
	size_t iov_count;
3236
	ssize_t ret, ret2, io_size;
J
Jens Axboe 已提交
3237

3238 3239 3240 3241
	if (req->io)
		iter = &req->io->rw.iter;

	ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3242 3243
	if (ret < 0)
		return ret;
3244 3245
	io_size = ret;
	req->result = io_size;
J
Jens Axboe 已提交
3246

3247 3248
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
3249
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
3250

3251
	/* If the file doesn't support async, just async punt */
3252
	if (force_nonblock && !io_file_supports_async(req->file, WRITE))
3253
		goto copy_iov;
3254

3255 3256 3257
	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
3258
		goto copy_iov;
3259

3260
	iov_count = iov_iter_count(iter);
3261
	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
3262 3263
	if (unlikely(ret))
		goto out_free;
3264

3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
	/*
	 * Open-code file_start_write here to grab freeze protection,
	 * which will be released by another thread in
	 * io_complete_rw().  Fool lockdep by telling it the lock got
	 * released so that it doesn't complain about the held lock when
	 * we return to userspace.
	 */
	if (req->flags & REQ_F_ISREG) {
		__sb_start_write(file_inode(req->file)->i_sb,
					SB_FREEZE_WRITE, true);
		__sb_writers_release(file_inode(req->file)->i_sb,
					SB_FREEZE_WRITE);
	}
	kiocb->ki_flags |= IOCB_WRITE;
3279

3280
	if (req->file->f_op->write_iter)
3281
		ret2 = call_write_iter(req->file, kiocb, iter);
3282
	else if (req->file->f_op->write)
3283
		ret2 = loop_rw_iter(WRITE, req->file, kiocb, iter);
3284 3285
	else
		ret2 = -EINVAL;
3286

3287 3288 3289 3290 3291 3292 3293 3294 3295
	/*
	 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
	 * retry them without IOCB_NOWAIT.
	 */
	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
		ret2 = -EAGAIN;
	if (!force_nonblock || ret2 != -EAGAIN) {
		kiocb_done(kiocb, ret2, cs);
	} else {
3296
copy_iov:
3297
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
3298 3299
		if (!ret)
			return -EAGAIN;
J
Jens Axboe 已提交
3300
	}
3301
out_free:
3302
	/* it's reportedly faster than delegating the null check to kfree() */
3303
	if (iovec)
3304
		kfree(iovec);
J
Jens Axboe 已提交
3305 3306 3307
	return ret;
}

P
Pavel Begunkov 已提交
3308 3309
static int __io_splice_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
P
Pavel Begunkov 已提交
3310 3311 3312 3313 3314 3315 3316
{
	struct io_splice* sp = &req->splice;
	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
	int ret;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3317 3318
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
P
Pavel Begunkov 已提交
3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332

	sp->file_in = NULL;
	sp->len = READ_ONCE(sqe->len);
	sp->flags = READ_ONCE(sqe->splice_flags);

	if (unlikely(sp->flags & ~valid_flags))
		return -EINVAL;

	ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
			  (sp->flags & SPLICE_F_FD_IN_FIXED));
	if (ret)
		return ret;
	req->flags |= REQ_F_NEED_CLEANUP;

3333 3334 3335 3336 3337 3338
	if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
		/*
		 * Splice operation will be punted aync, and here need to
		 * modify io_wq_work.flags, so initialize io_wq_work firstly.
		 */
		io_req_init_async(req);
P
Pavel Begunkov 已提交
3339
		req->work.flags |= IO_WQ_WORK_UNBOUND;
3340
	}
P
Pavel Begunkov 已提交
3341 3342 3343 3344

	return 0;
}

P
Pavel Begunkov 已提交
3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370
static int io_tee_prep(struct io_kiocb *req,
		       const struct io_uring_sqe *sqe)
{
	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
		return -EINVAL;
	return __io_splice_prep(req, sqe);
}

static int io_tee(struct io_kiocb *req, bool force_nonblock)
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	long ret = 0;

	if (force_nonblock)
		return -EAGAIN;
	if (sp->len)
		ret = do_tee(in, out, sp->len, flags);

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail_links(req);
3371
	io_req_complete(req, ret);
P
Pavel Begunkov 已提交
3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383
	return 0;
}

static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_splice* sp = &req->splice;

	sp->off_in = READ_ONCE(sqe->splice_off_in);
	sp->off_out = READ_ONCE(sqe->off);
	return __io_splice_prep(req, sqe);
}

3384
static int io_splice(struct io_kiocb *req, bool force_nonblock)
P
Pavel Begunkov 已提交
3385 3386 3387 3388 3389 3390
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	loff_t *poff_in, *poff_out;
3391
	long ret = 0;
P
Pavel Begunkov 已提交
3392

3393 3394
	if (force_nonblock)
		return -EAGAIN;
P
Pavel Begunkov 已提交
3395 3396 3397

	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
3398

3399
	if (sp->len)
3400
		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
P
Pavel Begunkov 已提交
3401 3402 3403 3404 3405 3406

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail_links(req);
3407
	io_req_complete(req, ret);
P
Pavel Begunkov 已提交
3408 3409 3410
	return 0;
}

J
Jens Axboe 已提交
3411 3412 3413
/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
3414
static int io_nop(struct io_kiocb *req, struct io_comp_state *cs)
J
Jens Axboe 已提交
3415 3416 3417
{
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
3418 3419 3420
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

3421
	__io_req_complete(req, 0, 0, cs);
J
Jens Axboe 已提交
3422 3423 3424
	return 0;
}

3425
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
C
Christoph Hellwig 已提交
3426
{
J
Jens Axboe 已提交
3427
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
3428

J
Jens Axboe 已提交
3429 3430
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
3431

J
Jens Axboe 已提交
3432
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
3433
		return -EINVAL;
3434
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
3435 3436
		return -EINVAL;

3437 3438 3439 3440 3441 3442
	req->sync.flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
C
Christoph Hellwig 已提交
3443 3444 3445
	return 0;
}

3446
static int io_fsync(struct io_kiocb *req, bool force_nonblock)
3447 3448 3449 3450
{
	loff_t end = req->sync.off + req->sync.len;
	int ret;

3451 3452 3453 3454
	/* fsync always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

3455
	ret = vfs_fsync_range(req->file, req->sync.off,
3456 3457 3458 3459
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
3460
	io_req_complete(req, ret);
C
Christoph Hellwig 已提交
3461 3462 3463
	return 0;
}

3464 3465 3466 3467 3468
static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;
3469 3470
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3471 3472 3473 3474 3475 3476 3477

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
	return 0;
}

3478
static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
3479
{
3480 3481
	int ret;

3482
	/* fallocate always requiring blocking context */
3483
	if (force_nonblock)
3484
		return -EAGAIN;
3485 3486 3487 3488
	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
				req->sync.len);
	if (ret < 0)
		req_set_fail_links(req);
3489
	io_req_complete(req, ret);
3490 3491 3492
	return 0;
}

3493
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3494
{
3495
	const char __user *fname;
3496
	int ret;
3497

3498
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3499
		return -EINVAL;
3500
	if (unlikely(sqe->ioprio || sqe->buf_index))
3501
		return -EINVAL;
3502
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
3503
		return -EBADF;
3504

3505 3506
	/* open.how should be already initialised */
	if (!(req->open.how.flags & O_PATH) && force_o_largefile())
3507
		req->open.how.flags |= O_LARGEFILE;
3508

3509 3510
	req->open.dfd = READ_ONCE(sqe->fd);
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3511
	req->open.filename = getname(fname);
3512 3513 3514 3515 3516
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}
3517
	req->open.nofile = rlimit(RLIMIT_NOFILE);
3518
	req->flags |= REQ_F_NEED_CLEANUP;
3519
	return 0;
3520 3521
}

3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	u64 flags, mode;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
	mode = READ_ONCE(sqe->len);
	flags = READ_ONCE(sqe->open_flags);
	req->open.how = build_open_how(flags, mode);
	return __io_openat_prep(req, sqe);
}

3534
static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
3535
{
3536 3537
	struct open_how __user *how;
	size_t len;
J
Jens Axboe 已提交
3538 3539
	int ret;

3540 3541
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3542 3543 3544 3545
	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	len = READ_ONCE(sqe->len);
	if (len < OPEN_HOW_SIZE_VER0)
		return -EINVAL;
3546

3547 3548 3549 3550
	ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
					len);
	if (ret)
		return ret;
3551

3552
	return __io_openat_prep(req, sqe);
3553 3554
}

3555
static int io_openat2(struct io_kiocb *req, bool force_nonblock)
3556 3557 3558 3559 3560
{
	struct open_flags op;
	struct file *file;
	int ret;

3561
	if (force_nonblock)
3562 3563
		return -EAGAIN;

3564
	ret = build_open_flags(&req->open.how, &op);
3565 3566 3567
	if (ret)
		goto err;

3568
	ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
	if (ret < 0)
		goto err;

	file = do_filp_open(req->open.dfd, req->open.filename, &op);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
	} else {
		fsnotify_open(file);
		fd_install(ret, file);
	}
err:
	putname(req->open.filename);
3582
	req->flags &= ~REQ_F_NEED_CLEANUP;
3583 3584
	if (ret < 0)
		req_set_fail_links(req);
3585
	io_req_complete(req, ret);
3586 3587 3588
	return 0;
}

3589
static int io_openat(struct io_kiocb *req, bool force_nonblock)
3590
{
3591
	return io_openat2(req, force_nonblock);
3592 3593
}

3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638
static int io_remove_buffers_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}

static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
			       int bgid, unsigned nbufs)
{
	unsigned i = 0;

	/* shouldn't happen */
	if (!nbufs)
		return 0;

	/* the head kbuf is the list itself */
	while (!list_empty(&buf->list)) {
		struct io_buffer *nxt;

		nxt = list_first_entry(&buf->list, struct io_buffer, list);
		list_del(&nxt->list);
		kfree(nxt);
		if (++i == nbufs)
			return i;
	}
	i++;
	kfree(buf);
	idr_remove(&ctx->io_buffer_idr, bgid);

	return i;
}

3639 3640
static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock,
			     struct io_comp_state *cs)
3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	ret = -ENOENT;
	head = idr_find(&ctx->io_buffer_idr, p->bgid);
	if (head)
		ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);

	io_ring_submit_lock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
3659
	__io_req_complete(req, ret, 0, cs);
3660 3661 3662
	return 0;
}

3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678
static int io_provide_buffers_prep(struct io_kiocb *req,
				   const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

3679
	if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}

static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
{
	struct io_buffer *buf;
	u64 addr = pbuf->addr;
	int i, bid = pbuf->bid;

	for (i = 0; i < pbuf->nbufs; i++) {
		buf = kmalloc(sizeof(*buf), GFP_KERNEL);
		if (!buf)
			break;

		buf->addr = addr;
		buf->len = pbuf->len;
		buf->bid = bid;
		addr += pbuf->len;
		bid++;
		if (!*head) {
			INIT_LIST_HEAD(&buf->list);
			*head = buf;
		} else {
			list_add_tail(&buf->list, &(*head)->list);
		}
	}

	return i ? i : -ENOMEM;
}

3717 3718
static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock,
			      struct io_comp_state *cs)
3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head, *list;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	list = head = idr_find(&ctx->io_buffer_idr, p->bgid);

	ret = io_add_buffers(p, &head);
	if (ret < 0)
		goto out;

	if (!list) {
		ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
					GFP_KERNEL);
		if (ret < 0) {
3739
			__io_remove_buffers(ctx, head, p->bgid, -1U);
3740 3741 3742 3743 3744 3745 3746
			goto out;
		}
	}
out:
	io_ring_submit_unlock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
3747
	__io_req_complete(req, ret, 0, cs);
3748
	return 0;
3749 3750
}

3751 3752 3753 3754 3755 3756
static int io_epoll_ctl_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_EPOLL)
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3757 3758
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777

	req->epoll.epfd = READ_ONCE(sqe->fd);
	req->epoll.op = READ_ONCE(sqe->len);
	req->epoll.fd = READ_ONCE(sqe->off);

	if (ep_op_has_event(req->epoll.op)) {
		struct epoll_event __user *ev;

		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
		if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
			return -EFAULT;
	}

	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3778 3779
static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock,
			struct io_comp_state *cs)
3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790
{
#if defined(CONFIG_EPOLL)
	struct io_epoll *ie = &req->epoll;
	int ret;

	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;

	if (ret < 0)
		req_set_fail_links(req);
3791
	__io_req_complete(req, ret, 0, cs);
3792 3793 3794 3795 3796 3797
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3798 3799 3800 3801 3802
static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	if (sqe->ioprio || sqe->buf_index || sqe->off)
		return -EINVAL;
3803 3804
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3805 3806 3807 3808 3809 3810 3811 3812 3813 3814

	req->madvise.addr = READ_ONCE(sqe->addr);
	req->madvise.len = READ_ONCE(sqe->len);
	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3815
static int io_madvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
3827
	io_req_complete(req, ret);
J
Jens Axboe 已提交
3828 3829 3830 3831 3832 3833
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3834 3835 3836 3837
static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr)
		return -EINVAL;
3838 3839
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3840 3841 3842 3843 3844 3845 3846

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}

3847
static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3848 3849 3850 3851
{
	struct io_fadvise *fa = &req->fadvise;
	int ret;

3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
	if (force_nonblock) {
		switch (fa->advice) {
		case POSIX_FADV_NORMAL:
		case POSIX_FADV_RANDOM:
		case POSIX_FADV_SEQUENTIAL:
			break;
		default:
			return -EAGAIN;
		}
	}
J
Jens Axboe 已提交
3862 3863 3864 3865

	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
	if (ret < 0)
		req_set_fail_links(req);
3866
	io_req_complete(req, ret);
J
Jens Axboe 已提交
3867 3868 3869
	return 0;
}

3870 3871
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
3872 3873
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3874 3875
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3876
	if (req->flags & REQ_F_FIXED_FILE)
3877
		return -EBADF;
3878

3879 3880
	req->statx.dfd = READ_ONCE(sqe->fd);
	req->statx.mask = READ_ONCE(sqe->len);
B
Bijan Mottahedeh 已提交
3881
	req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
3882 3883
	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	req->statx.flags = READ_ONCE(sqe->statx_flags);
3884 3885 3886 3887

	return 0;
}

3888
static int io_statx(struct io_kiocb *req, bool force_nonblock)
3889
{
3890
	struct io_statx *ctx = &req->statx;
3891 3892
	int ret;

3893 3894 3895 3896
	if (force_nonblock) {
		/* only need file table for an actual valid fd */
		if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD)
			req->flags |= REQ_F_NO_FILE_TABLE;
3897
		return -EAGAIN;
3898
	}
3899

B
Bijan Mottahedeh 已提交
3900 3901
	ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
		       ctx->buffer);
3902 3903 3904

	if (ret < 0)
		req_set_fail_links(req);
3905
	io_req_complete(req, ret);
3906 3907 3908
	return 0;
}

3909 3910 3911 3912
static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	/*
	 * If we queue this for async, it must not be cancellable. That would
3913 3914
	 * leave the 'file' in an undeterminate state, and here need to modify
	 * io_wq_work.flags, so initialize io_wq_work firstly.
3915
	 */
3916
	io_req_init_async(req);
3917 3918
	req->work.flags |= IO_WQ_WORK_NO_CANCEL;

3919 3920
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
3921 3922 3923
	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
	    sqe->rw_flags || sqe->buf_index)
		return -EINVAL;
3924
	if (req->flags & REQ_F_FIXED_FILE)
3925
		return -EBADF;
3926 3927

	req->close.fd = READ_ONCE(sqe->fd);
3928 3929 3930
	if ((req->file && req->file->f_op == &io_uring_fops) ||
	    req->close.fd == req->ctx->ring_fd)
		return -EBADF;
3931

3932
	req->close.put_file = NULL;
3933 3934 3935
	return 0;
}

3936 3937
static int io_close(struct io_kiocb *req, bool force_nonblock,
		    struct io_comp_state *cs)
3938
{
3939
	struct io_close *close = &req->close;
3940 3941
	int ret;

3942 3943 3944 3945 3946 3947
	/* might be already done during nonblock submission */
	if (!close->put_file) {
		ret = __close_fd_get_file(close->fd, &close->put_file);
		if (ret < 0)
			return (ret == -ENOENT) ? -EBADF : ret;
	}
3948 3949

	/* if the file has a flush method, be safe and punt to async */
3950
	if (close->put_file->f_op->flush && force_nonblock) {
3951 3952
		/* was never set, but play safe */
		req->flags &= ~REQ_F_NOWAIT;
3953
		/* avoid grabbing files - we don't need the files */
3954
		req->flags |= REQ_F_NO_FILE_TABLE;
3955
		return -EAGAIN;
P
Pavel Begunkov 已提交
3956
	}
3957

3958 3959 3960 3961 3962 3963
	/* No ->flush() or already async, safely close from here */
	ret = filp_close(close->put_file, req->work.files);
	if (ret < 0)
		req_set_fail_links(req);
	fput(close->put_file);
	close->put_file = NULL;
3964
	__io_req_complete(req, ret, 0, cs);
3965
	return 0;
3966 3967
}

3968
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

3980 3981 3982 3983 3984 3985
	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
	req->sync.flags = READ_ONCE(sqe->sync_range_flags);
	return 0;
}

3986
static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
3987 3988 3989
{
	int ret;

3990 3991 3992 3993
	/* sync_file_range always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

3994
	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
3995 3996 3997
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
3998
	io_req_complete(req, ret);
3999 4000 4001
	return 0;
}

4002
#if defined(CONFIG_NET)
4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017
static int io_setup_async_msg(struct io_kiocb *req,
			      struct io_async_msghdr *kmsg)
{
	if (req->io)
		return -EAGAIN;
	if (io_alloc_async_ctx(req)) {
		if (kmsg->iov != kmsg->fast_iov)
			kfree(kmsg->iov);
		return -ENOMEM;
	}
	req->flags |= REQ_F_NEED_CLEANUP;
	memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
	return -EAGAIN;
}

4018 4019 4020 4021 4022 4023 4024 4025 4026
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
{
	iomsg->iov = iomsg->fast_iov;
	iomsg->msg.msg_name = &iomsg->addr;
	return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
				   req->sr_msg.msg_flags, &iomsg->iov);
}

4027
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4028
{
4029
	struct io_sr_msg *sr = &req->sr_msg;
4030
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
4031
	int ret;
4032

4033 4034 4035
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

4036
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
4037
	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4038
	sr->len = READ_ONCE(sqe->len);
4039

4040 4041 4042 4043 4044
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

4045
	if (!io || req->opcode == IORING_OP_SEND)
4046
		return 0;
4047 4048 4049
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
4050

4051
	ret = io_sendmsg_copy_hdr(req, &io->msg);
P
Pavel Begunkov 已提交
4052 4053 4054
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
4055 4056
}

4057 4058
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
J
Jens Axboe 已提交
4059
{
4060
	struct io_async_msghdr iomsg, *kmsg;
J
Jens Axboe 已提交
4061
	struct socket *sock;
4062
	unsigned flags;
J
Jens Axboe 已提交
4063 4064 4065
	int ret;

	sock = sock_from_file(req->file, &ret);
4066 4067
	if (unlikely(!sock))
		return ret;
4068

4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080
	if (req->io) {
		kmsg = &req->io->msg;
		kmsg->msg.msg_name = &req->io->msg.addr;
		/* if iov is set, it's allocated already */
		if (!kmsg->iov)
			kmsg->iov = kmsg->fast_iov;
		kmsg->msg.msg_iter.iov = kmsg->iov;
	} else {
		ret = io_sendmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
J
Jens Axboe 已提交
4081 4082
	}

4083 4084 4085 4086 4087
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4088

4089 4090 4091 4092 4093
	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
	if (force_nonblock && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
J
Jens Axboe 已提交
4094

4095
	if (kmsg->iov != kmsg->fast_iov)
4096
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
4097
	req->flags &= ~REQ_F_NEED_CLEANUP;
J
Jens Axboe 已提交
4098 4099
	if (ret < 0)
		req_set_fail_links(req);
4100
	__io_req_complete(req, ret, 0, cs);
4101
	return 0;
4102
}
J
Jens Axboe 已提交
4103

4104 4105
static int io_send(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4106
{
4107 4108 4109
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	struct iovec iov;
4110
	struct socket *sock;
4111
	unsigned flags;
4112 4113 4114
	int ret;

	sock = sock_from_file(req->file, &ret);
4115 4116
	if (unlikely(!sock))
		return ret;
4117

4118 4119
	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
	if (unlikely(ret))
4120
		return ret;;
4121

4122 4123 4124 4125
	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;
4126

4127 4128 4129 4130 4131
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4132

4133 4134 4135 4136 4137 4138
	msg.msg_flags = flags;
	ret = sock_sendmsg(sock, &msg);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4139 4140 4141

	if (ret < 0)
		req_set_fail_links(req);
4142
	__io_req_complete(req, ret, 0, cs);
4143 4144 4145
	return 0;
}

4146 4147
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
				 struct io_async_msghdr *iomsg)
4148 4149 4150 4151 4152 4153
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct iovec __user *uiov;
	size_t iov_len;
	int ret;

4154 4155
	ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
					&iomsg->uaddr, &uiov, &iov_len);
4156 4157 4158 4159 4160 4161
	if (ret)
		return ret;

	if (req->flags & REQ_F_BUFFER_SELECT) {
		if (iov_len > 1)
			return -EINVAL;
4162
		if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov)))
4163
			return -EFAULT;
4164 4165
		sr->len = iomsg->iov[0].iov_len;
		iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1,
4166
				sr->len);
4167
		iomsg->iov = NULL;
4168 4169
	} else {
		ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
4170
					&iomsg->iov, &iomsg->msg.msg_iter);
4171 4172 4173 4174 4175 4176 4177 4178 4179
		if (ret > 0)
			ret = 0;
	}

	return ret;
}

#ifdef CONFIG_COMPAT
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
4180
					struct io_async_msghdr *iomsg)
4181 4182 4183 4184 4185 4186 4187 4188
{
	struct compat_msghdr __user *msg_compat;
	struct io_sr_msg *sr = &req->sr_msg;
	struct compat_iovec __user *uiov;
	compat_uptr_t ptr;
	compat_size_t len;
	int ret;

4189
	msg_compat = (struct compat_msghdr __user *) sr->umsg;
4190
	ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206
					&ptr, &len);
	if (ret)
		return ret;

	uiov = compat_ptr(ptr);
	if (req->flags & REQ_F_BUFFER_SELECT) {
		compat_ssize_t clen;

		if (len > 1)
			return -EINVAL;
		if (!access_ok(uiov, sizeof(*uiov)))
			return -EFAULT;
		if (__get_user(clen, &uiov->iov_len))
			return -EFAULT;
		if (clen < 0)
			return -EINVAL;
4207 4208
		sr->len = iomsg->iov[0].iov_len;
		iomsg->iov = NULL;
4209 4210
	} else {
		ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
4211 4212
						&iomsg->iov,
						&iomsg->msg.msg_iter);
4213 4214 4215 4216 4217 4218 4219 4220
		if (ret < 0)
			return ret;
	}

	return 0;
}
#endif

4221 4222
static int io_recvmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
4223
{
4224 4225
	iomsg->msg.msg_name = &iomsg->addr;
	iomsg->iov = iomsg->fast_iov;
4226 4227 4228

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
4229
		return __io_compat_recvmsg_copy_hdr(req, iomsg);
4230
#endif
4231

4232
	return __io_recvmsg_copy_hdr(req, iomsg);
4233 4234
}

4235
static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
4236
					       bool needs_lock)
4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct io_buffer *kbuf;

	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;

	sr->kbuf = kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;
	return kbuf;
4248 4249
}

4250 4251 4252 4253 4254
static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
{
	return io_put_kbuf(req, req->sr_msg.kbuf);
}

4255 4256
static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4257
{
4258
	struct io_sr_msg *sr = &req->sr_msg;
4259
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
4260
	int ret;
4261

4262 4263 4264
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

4265
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
4266
	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4267
	sr->len = READ_ONCE(sqe->len);
4268
	sr->bgid = READ_ONCE(sqe->buf_group);
4269

4270 4271 4272 4273 4274
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

4275
	if (!io || req->opcode == IORING_OP_RECV)
4276
		return 0;
4277 4278 4279
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
4280

4281
	ret = io_recvmsg_copy_hdr(req, &io->msg);
P
Pavel Begunkov 已提交
4282 4283 4284
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
J
Jens Axboe 已提交
4285 4286
}

4287 4288
static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
J
Jens Axboe 已提交
4289
{
4290
	struct io_async_msghdr iomsg, *kmsg;
4291
	struct socket *sock;
4292
	struct io_buffer *kbuf;
4293
	unsigned flags;
4294
	int ret, cflags = 0;
4295 4296

	sock = sock_from_file(req->file, &ret);
4297 4298
	if (unlikely(!sock))
		return ret;
4299

4300 4301 4302 4303 4304 4305 4306 4307 4308 4309
	if (req->io) {
		kmsg = &req->io->msg;
		kmsg->msg.msg_name = &req->io->msg.addr;
		/* if iov is set, it's allocated already */
		if (!kmsg->iov)
			kmsg->iov = kmsg->fast_iov;
		kmsg->msg.msg_iter.iov = kmsg->iov;
	} else {
		ret = io_recvmsg_copy_hdr(req, &iomsg);
		if (ret)
4310
			return ret;
4311 4312
		kmsg = &iomsg;
	}
4313

4314
	if (req->flags & REQ_F_BUFFER_SELECT) {
4315
		kbuf = io_recv_buffer_select(req, !force_nonblock);
4316
		if (IS_ERR(kbuf))
4317
			return PTR_ERR(kbuf);
4318 4319 4320 4321
		kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
				1, req->sr_msg.len);
	}
4322

4323 4324 4325 4326 4327
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4328

4329 4330
	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
					kmsg->uaddr, flags);
4331 4332
	if (force_nonblock && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
4333 4334
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4335

4336 4337
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
4338
	if (kmsg->iov != kmsg->fast_iov)
4339
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
4340
	req->flags &= ~REQ_F_NEED_CLEANUP;
J
Jens Axboe 已提交
4341 4342
	if (ret < 0)
		req_set_fail_links(req);
4343
	__io_req_complete(req, ret, cflags, cs);
4344
	return 0;
J
Jens Axboe 已提交
4345
}
4346

4347 4348
static int io_recv(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4349
{
4350
	struct io_buffer *kbuf;
4351 4352 4353
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	void __user *buf = sr->buf;
4354
	struct socket *sock;
4355 4356
	struct iovec iov;
	unsigned flags;
4357
	int ret, cflags = 0;
4358 4359

	sock = sock_from_file(req->file, &ret);
4360 4361
	if (unlikely(!sock))
		return ret;
4362

4363
	if (req->flags & REQ_F_BUFFER_SELECT) {
4364
		kbuf = io_recv_buffer_select(req, !force_nonblock);
4365 4366
		if (IS_ERR(kbuf))
			return PTR_ERR(kbuf);
4367
		buf = u64_to_user_ptr(kbuf->addr);
4368
	}
4369

4370
	ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
4371 4372
	if (unlikely(ret))
		goto out_free;
4373

4374 4375 4376 4377 4378 4379
	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;
	msg.msg_iocb = NULL;
	msg.msg_flags = 0;
4380

4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;

	ret = sock_recvmsg(sock, &msg, flags);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4392
out_free:
4393 4394
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
4395 4396
	if (ret < 0)
		req_set_fail_links(req);
4397
	__io_req_complete(req, ret, cflags, cs);
4398 4399 4400
	return 0;
}

4401
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4402
{
4403 4404
	struct io_accept *accept = &req->accept;

4405 4406
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
4407
	if (sqe->ioprio || sqe->len || sqe->buf_index)
4408 4409
		return -EINVAL;

4410 4411
	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4412
	accept->flags = READ_ONCE(sqe->accept_flags);
4413
	accept->nofile = rlimit(RLIMIT_NOFILE);
4414 4415
	return 0;
}
4416

4417 4418
static int io_accept(struct io_kiocb *req, bool force_nonblock,
		     struct io_comp_state *cs)
4419 4420
{
	struct io_accept *accept = &req->accept;
4421
	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
4422 4423
	int ret;

4424 4425 4426
	if (req->file->f_flags & O_NONBLOCK)
		req->flags |= REQ_F_NOWAIT;

4427
	ret = __sys_accept4_file(req->file, file_flags, accept->addr,
4428 4429
					accept->addr_len, accept->flags,
					accept->nofile);
4430
	if (ret == -EAGAIN && force_nonblock)
4431
		return -EAGAIN;
4432 4433 4434
	if (ret < 0) {
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
4435
		req_set_fail_links(req);
4436
	}
4437
	__io_req_complete(req, ret, 0, cs);
4438
	return 0;
4439 4440
}

4441
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4442
{
4443 4444
	struct io_connect *conn = &req->connect;
	struct io_async_ctx *io = req->io;
4445

4446 4447 4448 4449 4450
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

4451 4452 4453 4454 4455 4456 4457
	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	conn->addr_len =  READ_ONCE(sqe->addr2);

	if (!io)
		return 0;

	return move_addr_to_kernel(conn->addr, conn->addr_len,
4458
					&io->connect.address);
4459 4460
}

4461 4462
static int io_connect(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4463
{
4464
	struct io_async_ctx __io, *io;
4465
	unsigned file_flags;
4466
	int ret;
4467

4468 4469 4470
	if (req->io) {
		io = req->io;
	} else {
4471 4472 4473
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
4474 4475 4476 4477 4478
		if (ret)
			goto out;
		io = &__io;
	}

4479 4480 4481 4482
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
4483
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
4484 4485 4486
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
4487 4488 4489
			ret = -ENOMEM;
			goto out;
		}
4490
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
4491
		return -EAGAIN;
4492
	}
4493 4494
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4495
out:
J
Jens Axboe 已提交
4496 4497
	if (ret < 0)
		req_set_fail_links(req);
4498
	__io_req_complete(req, ret, 0, cs);
4499
	return 0;
4500 4501 4502 4503
}
#else /* !CONFIG_NET */
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
4504 4505 4506
	return -EOPNOTSUPP;
}

4507 4508
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4509 4510 4511 4512
{
	return -EOPNOTSUPP;
}

4513 4514
static int io_send(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4515 4516 4517 4518 4519 4520 4521 4522 4523 4524
{
	return -EOPNOTSUPP;
}

static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4525 4526
static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4527 4528 4529 4530
{
	return -EOPNOTSUPP;
}

4531 4532
static int io_recv(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4533 4534 4535 4536 4537 4538 4539 4540 4541
{
	return -EOPNOTSUPP;
}

static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4542 4543
static int io_accept(struct io_kiocb *req, bool force_nonblock,
		     struct io_comp_state *cs)
4544 4545 4546
{
	return -EOPNOTSUPP;
}
4547

4548 4549 4550 4551 4552
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4553 4554
static int io_connect(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4555
{
4556
	return -EOPNOTSUPP;
4557
}
4558
#endif /* CONFIG_NET */
4559

4560 4561 4562 4563 4564
struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};
4565

4566 4567 4568
static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
			   __poll_t mask, task_work_func_t func)
{
4569
	int ret;
4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

	trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);

	list_del_init(&poll->wait.entry);

	req->result = mask;
	init_task_work(&req->task_work, func);
4581 4582
	percpu_ref_get(&req->ctx->refs);

4583
	/*
4584 4585 4586 4587
	 * If this fails, then the task is exiting. When a task exits, the
	 * work gets canceled, so just cancel this request as well instead
	 * of executing it. We can't safely execute it anyway, as we may not
	 * have the needed state needed for it anyway.
4588
	 */
4589
	ret = io_req_task_work_add(req, &req->task_work);
4590
	if (unlikely(ret)) {
4591 4592
		struct task_struct *tsk;

4593
		WRITE_ONCE(poll->canceled, true);
4594
		tsk = io_wq_get_task(req->ctx->io_wq);
4595 4596
		task_work_add(tsk, &req->task_work, 0);
		wake_up_process(tsk);
4597
	}
4598 4599 4600
	return 1;
}

4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620
static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
	__acquires(&req->ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->result && !READ_ONCE(poll->canceled)) {
		struct poll_table_struct pt = { ._key = poll->events };

		req->result = vfs_poll(req->file, &pt) & poll->events;
	}

	spin_lock_irq(&ctx->completion_lock);
	if (!req->result && !READ_ONCE(poll->canceled)) {
		add_wait_queue(poll->head, &poll->wait);
		return true;
	}

	return false;
}

4621
static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
4622
{
4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638
	/* pure poll stashes this in ->io, poll driven retry elsewhere */
	if (req->opcode == IORING_OP_POLL_ADD)
		return (struct io_poll_iocb *) req->io;
	return req->apoll->double_poll;
}

static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
{
	if (req->opcode == IORING_OP_POLL_ADD)
		return &req->poll;
	return &req->apoll->poll;
}

static void io_poll_remove_double(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = io_poll_get_double(req);
4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657

	lockdep_assert_held(&req->ctx->completion_lock);

	if (poll && poll->head) {
		struct wait_queue_head *head = poll->head;

		spin_lock(&head->lock);
		list_del_init(&poll->wait.entry);
		if (poll->wait.private)
			refcount_dec(&req->refs);
		poll->head = NULL;
		spin_unlock(&head->lock);
	}
}

static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

4658
	io_poll_remove_double(req);
4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675
	req->poll.done = true;
	io_cqring_fill_event(req, error ? error : mangle_poll(mask));
	io_commit_cqring(ctx);
}

static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (io_poll_rewait(req, &req->poll)) {
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}

	hash_del(&req->hash_node);
	io_poll_complete(req, req->result, 0);
	req->flags |= REQ_F_COMP_LOCKED;
4676
	*nxt = io_put_req_find_next(req);
4677 4678 4679 4680 4681 4682 4683 4684
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
}

static void io_poll_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4685
	struct io_ring_ctx *ctx = req->ctx;
4686 4687 4688
	struct io_kiocb *nxt = NULL;

	io_poll_task_handler(req, &nxt);
4689 4690
	if (nxt)
		__io_req_task_submit(nxt);
4691
	percpu_ref_put(&ctx->refs);
4692 4693 4694 4695 4696 4697
}

static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
			       int sync, void *key)
{
	struct io_kiocb *req = wait->private;
4698
	struct io_poll_iocb *poll = io_poll_get_single(req);
4699 4700 4701 4702 4703 4704
	__poll_t mask = key_to_poll(key);

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

4705
	if (poll && poll->head) {
4706 4707
		bool done;

4708 4709
		spin_lock(&poll->head->lock);
		done = list_empty(&poll->wait.entry);
4710
		if (!done)
4711
			list_del_init(&poll->wait.entry);
4712 4713
		/* make sure double remove sees this as being gone */
		wait->private = NULL;
4714
		spin_unlock(&poll->head->lock);
4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733
		if (!done)
			__io_async_wake(req, poll, mask, io_poll_task_func);
	}
	refcount_dec(&req->refs);
	return 1;
}

static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
			      wait_queue_func_t wake_func)
{
	poll->head = NULL;
	poll->done = false;
	poll->canceled = false;
	poll->events = events;
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, wake_func);
}

static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
4734 4735
			    struct wait_queue_head *head,
			    struct io_poll_iocb **poll_ptr)
4736 4737 4738 4739 4740 4741 4742 4743 4744 4745
{
	struct io_kiocb *req = pt->req;

	/*
	 * If poll->head is already set, it's because the file being polled
	 * uses multiple waitqueues for poll handling (eg one for read, one
	 * for write). Setup a separate io_poll_iocb if this happens.
	 */
	if (unlikely(poll->head)) {
		/* already have a 2nd entry, fail a third attempt */
4746
		if (*poll_ptr) {
4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757
			pt->error = -EINVAL;
			return;
		}
		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
		if (!poll) {
			pt->error = -ENOMEM;
			return;
		}
		io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);
		refcount_inc(&req->refs);
		poll->wait.private = req;
4758
		*poll_ptr = poll;
4759 4760 4761 4762
	}

	pt->error = 0;
	poll->head = head;
4763 4764 4765 4766 4767

	if (poll->events & EPOLLEXCLUSIVE)
		add_wait_queue_exclusive(head, &poll->wait);
	else
		add_wait_queue(head, &poll->wait);
4768 4769 4770 4771 4772 4773
}

static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
4774
	struct async_poll *apoll = pt->req->apoll;
4775

4776
	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
4777 4778
}

4779 4780 4781 4782 4783 4784 4785 4786
static void io_async_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct async_poll *apoll = req->apoll;
	struct io_ring_ctx *ctx = req->ctx;

	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);

4787
	if (io_poll_rewait(req, &apoll->poll)) {
4788
		spin_unlock_irq(&ctx->completion_lock);
4789
		percpu_ref_put(&ctx->refs);
4790
		return;
4791 4792
	}

4793
	/* If req is still hashed, it cannot have been canceled. Don't check. */
4794
	if (hash_hashed(&req->hash_node))
4795
		hash_del(&req->hash_node);
4796

4797
	io_poll_remove_double(req);
4798 4799
	spin_unlock_irq(&ctx->completion_lock);

4800 4801 4802 4803
	if (!READ_ONCE(apoll->poll.canceled))
		__io_req_task_submit(req);
	else
		__io_req_task_cancel(req, -ECANCELED);
4804

4805
	percpu_ref_put(&ctx->refs);
4806
	kfree(apoll->double_poll);
4807
	kfree(apoll);
4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839
}

static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->apoll->poll;

	trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
					key_to_poll(key));

	return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
}

static void io_poll_req_insert(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct hlist_head *list;

	list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	hlist_add_head(&req->hash_node, list);
}

static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
				      struct io_poll_iocb *poll,
				      struct io_poll_table *ipt, __poll_t mask,
				      wait_queue_func_t wake_func)
	__acquires(&ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;
	bool cancel = false;

4840
	io_init_poll_iocb(poll, mask, wake_func);
4841
	poll->file = req->file;
4842
	poll->wait.private = req;
4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880

	ipt->pt._key = mask;
	ipt->req = req;
	ipt->error = -EINVAL;

	mask = vfs_poll(req->file, &ipt->pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
		if (unlikely(list_empty(&poll->wait.entry))) {
			if (ipt->error)
				cancel = true;
			ipt->error = 0;
			mask = 0;
		}
		if (mask || ipt->error)
			list_del_init(&poll->wait.entry);
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
			io_poll_req_insert(req);
		spin_unlock(&poll->head->lock);
	}

	return mask;
}

static bool io_arm_poll_handler(struct io_kiocb *req)
{
	const struct io_op_def *def = &io_op_defs[req->opcode];
	struct io_ring_ctx *ctx = req->ctx;
	struct async_poll *apoll;
	struct io_poll_table ipt;
	__poll_t mask, ret;

	if (!req->file || !file_can_poll(req->file))
		return false;
4881
	if (req->flags & REQ_F_POLLED)
4882 4883 4884 4885 4886 4887 4888
		return false;
	if (!def->pollin && !def->pollout)
		return false;

	apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
	if (unlikely(!apoll))
		return false;
4889
	apoll->double_poll = NULL;
4890 4891

	req->flags |= REQ_F_POLLED;
P
Pavel Begunkov 已提交
4892
	io_get_req_task(req);
4893 4894 4895
	req->apoll = apoll;
	INIT_HLIST_NODE(&req->hash_node);

4896
	mask = 0;
4897
	if (def->pollin)
4898
		mask |= POLLIN | POLLRDNORM;
4899 4900
	if (def->pollout)
		mask |= POLLOUT | POLLWRNORM;
4901 4902 4903 4904 4905 4906

	/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
	if ((req->opcode == IORING_OP_RECVMSG) &&
	    (req->sr_msg.msg_flags & MSG_ERRQUEUE))
		mask &= ~POLLIN;

4907 4908 4909 4910 4911 4912
	mask |= POLLERR | POLLPRI;

	ipt.pt._qproc = io_async_queue_proc;

	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
					io_async_wake);
4913
	if (ret || ipt.error) {
4914
		io_poll_remove_double(req);
4915
		spin_unlock_irq(&ctx->completion_lock);
4916
		kfree(apoll->double_poll);
4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927
		kfree(apoll);
		return false;
	}
	spin_unlock_irq(&ctx->completion_lock);
	trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
					apoll->poll.events);
	return true;
}

static bool __io_poll_remove_one(struct io_kiocb *req,
				 struct io_poll_iocb *poll)
4928
{
4929
	bool do_complete = false;
4930 4931 4932

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
4933 4934
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
4935
		do_complete = true;
4936 4937
	}
	spin_unlock(&poll->head->lock);
4938
	hash_del(&req->hash_node);
4939 4940 4941 4942 4943 4944 4945
	return do_complete;
}

static bool io_poll_remove_one(struct io_kiocb *req)
{
	bool do_complete;

4946 4947
	io_poll_remove_double(req);

4948 4949 4950
	if (req->opcode == IORING_OP_POLL_ADD) {
		do_complete = __io_poll_remove_one(req, &req->poll);
	} else {
4951 4952
		struct async_poll *apoll = req->apoll;

4953
		/* non-poll requests have submit ref still */
4954 4955
		do_complete = __io_poll_remove_one(req, &apoll->poll);
		if (do_complete) {
4956
			io_put_req(req);
4957
			kfree(apoll->double_poll);
4958 4959
			kfree(apoll);
		}
4960 4961
	}

4962 4963 4964 4965
	if (do_complete) {
		io_cqring_fill_event(req, -ECANCELED);
		io_commit_cqring(req->ctx);
		req->flags |= REQ_F_COMP_LOCKED;
4966
		req_set_fail_links(req);
4967 4968 4969 4970
		io_put_req(req);
	}

	return do_complete;
4971 4972 4973 4974
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
4975
	struct hlist_node *tmp;
4976
	struct io_kiocb *req;
4977
	int posted = 0, i;
4978 4979

	spin_lock_irq(&ctx->completion_lock);
4980 4981 4982 4983 4984
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node)
4985
			posted += io_poll_remove_one(req);
4986 4987
	}
	spin_unlock_irq(&ctx->completion_lock);
4988

4989 4990
	if (posted)
		io_cqring_ev_posted(ctx);
4991 4992
}

4993 4994
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
4995
	struct hlist_head *list;
4996 4997
	struct io_kiocb *req;

4998 4999
	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	hlist_for_each_entry(req, list, hash_node) {
5000 5001 5002
		if (sqe_addr != req->user_data)
			continue;
		if (io_poll_remove_one(req))
5003
			return 0;
5004
		return -EALREADY;
5005 5006 5007 5008 5009
	}

	return -ENOENT;
}

5010 5011
static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	req->poll.addr = READ_ONCE(sqe->addr);
	return 0;
}

5023 5024 5025 5026
/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
5027
static int io_poll_remove(struct io_kiocb *req)
5028 5029
{
	struct io_ring_ctx *ctx = req->ctx;
5030
	u64 addr;
5031
	int ret;
5032

5033
	addr = req->poll.addr;
5034
	spin_lock_irq(&ctx->completion_lock);
5035
	ret = io_poll_cancel(ctx, addr);
5036 5037
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
5038 5039
	if (ret < 0)
		req_set_fail_links(req);
5040
	io_req_complete(req, ret);
5041 5042 5043 5044 5045 5046
	return 0;
}

static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
5047 5048
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->poll;
5049

5050
	return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
5051 5052 5053 5054 5055 5056 5057
}

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

5058
	__io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->io);
5059 5060
}

5061
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5062 5063
{
	struct io_poll_iocb *poll = &req->poll;
5064
	u32 events;
5065 5066 5067 5068 5069

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
5070 5071
	if (!poll->file)
		return -EBADF;
5072

5073 5074 5075 5076
	events = READ_ONCE(sqe->poll32_events);
#ifdef __BIG_ENDIAN
	events = swahw32(events);
#endif
5077 5078
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
		       (events & EPOLLEXCLUSIVE);
5079

P
Pavel Begunkov 已提交
5080
	io_get_req_task(req);
5081 5082 5083
	return 0;
}

5084
static int io_poll_add(struct io_kiocb *req)
5085 5086 5087 5088 5089 5090
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	__poll_t mask;

5091
	INIT_HLIST_NODE(&req->hash_node);
5092
	ipt.pt._qproc = io_poll_queue_proc;
5093

5094 5095
	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
					io_poll_wake);
5096

J
Jens Axboe 已提交
5097
	if (mask) { /* no async, we'd stolen it */
5098
		ipt.error = 0;
5099
		io_poll_complete(req, mask, 0);
5100 5101 5102
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
5103 5104
	if (mask) {
		io_cqring_ev_posted(ctx);
5105
		io_put_req(req);
5106
	}
J
Jens Axboe 已提交
5107
	return ipt.error;
5108 5109
}

J
Jens Axboe 已提交
5110 5111
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
5112 5113 5114 5115
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
5116 5117 5118
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
5119 5120 5121
	atomic_set(&req->ctx->cq_timeouts,
		atomic_read(&req->ctx->cq_timeouts) + 1);

5122
	/*
5123 5124
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
5125
	 */
P
Pavel Begunkov 已提交
5126 5127
	if (!list_empty(&req->timeout.list))
		list_del_init(&req->timeout.list);
J
Jens Axboe 已提交
5128

5129
	io_cqring_fill_event(req, -ETIME);
J
Jens Axboe 已提交
5130 5131 5132 5133
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
5134
	req_set_fail_links(req);
J
Jens Axboe 已提交
5135 5136 5137 5138
	io_put_req(req);
	return HRTIMER_NORESTART;
}

5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155
static int __io_timeout_cancel(struct io_kiocb *req)
{
	int ret;

	list_del_init(&req->timeout.list);

	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
	if (ret == -1)
		return -EALREADY;

	req_set_fail_links(req);
	req->flags |= REQ_F_COMP_LOCKED;
	io_cqring_fill_event(req, -ECANCELED);
	io_put_req(req);
	return 0;
}

5156 5157 5158 5159 5160
static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
{
	struct io_kiocb *req;
	int ret = -ENOENT;

P
Pavel Begunkov 已提交
5161
	list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
5162 5163 5164 5165 5166 5167 5168 5169 5170
		if (user_data == req->user_data) {
			ret = 0;
			break;
		}
	}

	if (ret == -ENOENT)
		return ret;

5171
	return __io_timeout_cancel(req);
5172 5173
}

5174 5175
static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
5176 5177 5178
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
5179 5180 5181
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index || sqe->len)
5182 5183 5184 5185 5186 5187 5188 5189 5190 5191
		return -EINVAL;

	req->timeout.addr = READ_ONCE(sqe->addr);
	req->timeout.flags = READ_ONCE(sqe->timeout_flags);
	if (req->timeout.flags)
		return -EINVAL;

	return 0;
}

5192 5193 5194
/*
 * Remove or update an existing timeout command
 */
5195
static int io_timeout_remove(struct io_kiocb *req)
5196 5197
{
	struct io_ring_ctx *ctx = req->ctx;
5198
	int ret;
5199 5200

	spin_lock_irq(&ctx->completion_lock);
5201
	ret = io_timeout_cancel(ctx, req->timeout.addr);
5202

5203
	io_cqring_fill_event(req, ret);
5204 5205
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
5206
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
5207 5208
	if (ret < 0)
		req_set_fail_links(req);
5209
	io_put_req(req);
5210
	return 0;
J
Jens Axboe 已提交
5211 5212
}

5213
static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5214
			   bool is_timeout_link)
J
Jens Axboe 已提交
5215
{
5216
	struct io_timeout_data *data;
5217
	unsigned flags;
5218
	u32 off = READ_ONCE(sqe->off);
J
Jens Axboe 已提交
5219

5220
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
5221
		return -EINVAL;
5222
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
5223
		return -EINVAL;
5224
	if (off && is_timeout_link)
5225
		return -EINVAL;
5226 5227
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
5228
		return -EINVAL;
5229

5230
	req->timeout.off = off;
5231

5232
	if (!req->io && io_alloc_async_ctx(req))
5233 5234 5235
		return -ENOMEM;

	data = &req->io->timeout;
5236 5237 5238
	data->req = req;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
5239 5240
		return -EFAULT;

5241
	if (flags & IORING_TIMEOUT_ABS)
5242
		data->mode = HRTIMER_MODE_ABS;
5243
	else
5244
		data->mode = HRTIMER_MODE_REL;
5245

5246 5247 5248 5249
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}

5250
static int io_timeout(struct io_kiocb *req)
5251 5252
{
	struct io_ring_ctx *ctx = req->ctx;
5253
	struct io_timeout_data *data = &req->io->timeout;
5254
	struct list_head *entry;
5255
	u32 tail, off = req->timeout.off;
5256

5257
	spin_lock_irq(&ctx->completion_lock);
5258

J
Jens Axboe 已提交
5259 5260
	/*
	 * sqe->off holds how many events that need to occur for this
5261 5262
	 * timeout event to be satisfied. If it isn't set, then this is
	 * a pure timeout request, sequence isn't used.
J
Jens Axboe 已提交
5263
	 */
5264
	if (io_is_timeout_noseq(req)) {
5265 5266 5267
		entry = ctx->timeout_list.prev;
		goto add;
	}
J
Jens Axboe 已提交
5268

5269 5270
	tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
	req->timeout.target_seq = tail + off;
J
Jens Axboe 已提交
5271 5272 5273 5274 5275 5276

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	list_for_each_prev(entry, &ctx->timeout_list) {
P
Pavel Begunkov 已提交
5277 5278
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
						  timeout.list);
J
Jens Axboe 已提交
5279

5280
		if (io_is_timeout_noseq(nxt))
5281
			continue;
5282 5283
		/* nxt.seq is behind @tail, otherwise would've been completed */
		if (off >= nxt->timeout.target_seq - tail)
J
Jens Axboe 已提交
5284 5285
			break;
	}
5286
add:
P
Pavel Begunkov 已提交
5287
	list_add(&req->timeout.list, entry);
5288 5289
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
J
Jens Axboe 已提交
5290 5291 5292 5293
	spin_unlock_irq(&ctx->completion_lock);
	return 0;
}

5294 5295 5296 5297 5298 5299 5300
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

5301
static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
5302 5303 5304 5305
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;

5306
	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

5319 5320 5321
	return ret;
}

5322 5323
static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
5324
				     int success_ret)
5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
	if (ret != -ENOENT) {
		spin_lock_irqsave(&ctx->completion_lock, flags);
		goto done;
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr);
done:
5341 5342
	if (!ret)
		ret = success_ret;
5343 5344 5345 5346 5347
	io_cqring_fill_event(req, ret);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

J
Jens Axboe 已提交
5348 5349
	if (ret < 0)
		req_set_fail_links(req);
5350
	io_put_req(req);
5351 5352
}

5353 5354
static int io_async_cancel_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
5355
{
5356
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5357
		return -EINVAL;
5358 5359 5360
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
5361 5362
		return -EINVAL;

5363 5364 5365 5366
	req->cancel.addr = READ_ONCE(sqe->addr);
	return 0;
}

5367
static int io_async_cancel(struct io_kiocb *req)
5368 5369 5370
{
	struct io_ring_ctx *ctx = req->ctx;

5371
	io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
J
Jens Axboe 已提交
5372 5373 5374
	return 0;
}

5375 5376 5377
static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
5378 5379 5380
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->rw_flags)
5381 5382 5383 5384 5385 5386 5387 5388 5389 5390
		return -EINVAL;

	req->files_update.offset = READ_ONCE(sqe->off);
	req->files_update.nr_args = READ_ONCE(sqe->len);
	if (!req->files_update.nr_args)
		return -EINVAL;
	req->files_update.arg = READ_ONCE(sqe->addr);
	return 0;
}

5391 5392
static int io_files_update(struct io_kiocb *req, bool force_nonblock,
			   struct io_comp_state *cs)
5393 5394
{
	struct io_ring_ctx *ctx = req->ctx;
5395 5396
	struct io_uring_files_update up;
	int ret;
5397

5398
	if (force_nonblock)
5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
5410
	__io_req_complete(req, ret, 0, cs);
J
Jens Axboe 已提交
5411 5412 5413
	return 0;
}

5414 5415
static int io_req_defer_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
5416
{
J
Jens Axboe 已提交
5417
	ssize_t ret = 0;
5418

5419 5420 5421
	if (!sqe)
		return 0;

5422 5423
	if (io_alloc_async_ctx(req))
		return -EAGAIN;
5424 5425 5426
	ret = io_prep_work_files(req);
	if (unlikely(ret))
		return ret;
5427

5428
	switch (req->opcode) {
J
Jens Axboe 已提交
5429 5430
	case IORING_OP_NOP:
		break;
5431 5432
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
5433
	case IORING_OP_READ:
5434
		ret = io_read_prep(req, sqe, true);
5435 5436 5437
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
5438
	case IORING_OP_WRITE:
5439
		ret = io_write_prep(req, sqe, true);
5440
		break;
5441
	case IORING_OP_POLL_ADD:
5442
		ret = io_poll_add_prep(req, sqe);
5443 5444
		break;
	case IORING_OP_POLL_REMOVE:
5445
		ret = io_poll_remove_prep(req, sqe);
5446
		break;
5447
	case IORING_OP_FSYNC:
5448
		ret = io_prep_fsync(req, sqe);
5449 5450
		break;
	case IORING_OP_SYNC_FILE_RANGE:
5451
		ret = io_prep_sfr(req, sqe);
5452
		break;
5453
	case IORING_OP_SENDMSG:
5454
	case IORING_OP_SEND:
5455
		ret = io_sendmsg_prep(req, sqe);
5456 5457
		break;
	case IORING_OP_RECVMSG:
5458
	case IORING_OP_RECV:
5459
		ret = io_recvmsg_prep(req, sqe);
5460
		break;
5461
	case IORING_OP_CONNECT:
5462
		ret = io_connect_prep(req, sqe);
5463
		break;
5464
	case IORING_OP_TIMEOUT:
5465
		ret = io_timeout_prep(req, sqe, false);
5466
		break;
5467
	case IORING_OP_TIMEOUT_REMOVE:
5468
		ret = io_timeout_remove_prep(req, sqe);
5469
		break;
5470
	case IORING_OP_ASYNC_CANCEL:
5471
		ret = io_async_cancel_prep(req, sqe);
5472
		break;
5473
	case IORING_OP_LINK_TIMEOUT:
5474
		ret = io_timeout_prep(req, sqe, true);
5475
		break;
5476
	case IORING_OP_ACCEPT:
5477
		ret = io_accept_prep(req, sqe);
5478
		break;
5479 5480 5481
	case IORING_OP_FALLOCATE:
		ret = io_fallocate_prep(req, sqe);
		break;
5482 5483 5484
	case IORING_OP_OPENAT:
		ret = io_openat_prep(req, sqe);
		break;
5485 5486 5487
	case IORING_OP_CLOSE:
		ret = io_close_prep(req, sqe);
		break;
5488 5489 5490
	case IORING_OP_FILES_UPDATE:
		ret = io_files_update_prep(req, sqe);
		break;
5491 5492 5493
	case IORING_OP_STATX:
		ret = io_statx_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5494 5495 5496
	case IORING_OP_FADVISE:
		ret = io_fadvise_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5497 5498 5499
	case IORING_OP_MADVISE:
		ret = io_madvise_prep(req, sqe);
		break;
5500 5501 5502
	case IORING_OP_OPENAT2:
		ret = io_openat2_prep(req, sqe);
		break;
5503 5504 5505
	case IORING_OP_EPOLL_CTL:
		ret = io_epoll_ctl_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5506 5507 5508
	case IORING_OP_SPLICE:
		ret = io_splice_prep(req, sqe);
		break;
5509 5510 5511
	case IORING_OP_PROVIDE_BUFFERS:
		ret = io_provide_buffers_prep(req, sqe);
		break;
5512 5513 5514
	case IORING_OP_REMOVE_BUFFERS:
		ret = io_remove_buffers_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5515 5516 5517
	case IORING_OP_TEE:
		ret = io_tee_prep(req, sqe);
		break;
5518
	default:
J
Jens Axboe 已提交
5519 5520 5521
		printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
				req->opcode);
		ret = -EINVAL;
5522
		break;
5523 5524
	}

5525
	return ret;
5526 5527
}

5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541
static u32 io_get_sequence(struct io_kiocb *req)
{
	struct io_kiocb *pos;
	struct io_ring_ctx *ctx = req->ctx;
	u32 total_submitted, nr_reqs = 1;

	if (req->flags & REQ_F_LINK_HEAD)
		list_for_each_entry(pos, &req->link_list, link_list)
			nr_reqs++;

	total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
	return total_submitted - nr_reqs;
}

5542
static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5543
{
5544
	struct io_ring_ctx *ctx = req->ctx;
5545
	struct io_defer_entry *de;
5546
	int ret;
5547
	u32 seq;
5548

B
Bob Liu 已提交
5549
	/* Still need defer if there is pending req in defer list. */
5550 5551 5552 5553 5554 5555 5556
	if (likely(list_empty_careful(&ctx->defer_list) &&
		!(req->flags & REQ_F_IO_DRAIN)))
		return 0;

	seq = io_get_sequence(req);
	/* Still a chance to pass the sequence check */
	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
5557 5558
		return 0;

5559 5560
	if (!req->io) {
		ret = io_req_defer_prep(req, sqe);
5561
		if (ret)
5562 5563
			return ret;
	}
5564
	io_prep_async_link(req);
5565 5566 5567
	de = kmalloc(sizeof(*de), GFP_KERNEL);
	if (!de)
		return -ENOMEM;
5568

5569
	spin_lock_irq(&ctx->completion_lock);
5570
	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
5571
		spin_unlock_irq(&ctx->completion_lock);
5572
		kfree(de);
5573 5574
		io_queue_async_work(req);
		return -EIOCBQUEUED;
5575 5576
	}

5577
	trace_io_uring_defer(ctx, req, req->user_data);
5578
	de->req = req;
5579
	de->seq = seq;
5580
	list_add_tail(&de->list, &ctx->defer_list);
5581 5582 5583 5584
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

5585
static void __io_clean_op(struct io_kiocb *req)
P
Pavel Begunkov 已提交
5586 5587 5588
{
	struct io_async_ctx *io = req->io;

5589 5590 5591 5592 5593
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
5594
			kfree((void *)(unsigned long)req->rw.addr);
5595 5596 5597
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_RECV:
5598
			kfree(req->sr_msg.kbuf);
5599 5600 5601
			break;
		}
		req->flags &= ~REQ_F_BUFFER_SELECTED;
P
Pavel Begunkov 已提交
5602 5603
	}

5604 5605 5606 5607 5608 5609 5610 5611
	if (req->flags & REQ_F_NEED_CLEANUP) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
		case IORING_OP_WRITE:
5612 5613
			if (io->rw.free_iovec)
				kfree(io->rw.free_iovec);
5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_SENDMSG:
			if (io->msg.iov != io->msg.fast_iov)
				kfree(io->msg.iov);
			break;
		case IORING_OP_SPLICE:
		case IORING_OP_TEE:
			io_put_file(req, req->splice.file_in,
				    (req->splice.flags & SPLICE_F_FD_IN_FIXED));
			break;
		}
		req->flags &= ~REQ_F_NEED_CLEANUP;
P
Pavel Begunkov 已提交
5627
	}
5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639

	if (req->flags & REQ_F_INFLIGHT) {
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
		req->flags &= ~REQ_F_INFLIGHT;
	}
P
Pavel Begunkov 已提交
5640 5641
}

5642
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5643
			bool force_nonblock, struct io_comp_state *cs)
J
Jens Axboe 已提交
5644
{
5645
	struct io_ring_ctx *ctx = req->ctx;
5646
	int ret;
J
Jens Axboe 已提交
5647

5648
	switch (req->opcode) {
J
Jens Axboe 已提交
5649
	case IORING_OP_NOP:
5650
		ret = io_nop(req, cs);
J
Jens Axboe 已提交
5651 5652
		break;
	case IORING_OP_READV:
5653
	case IORING_OP_READ_FIXED:
5654
	case IORING_OP_READ:
5655 5656 5657 5658 5659
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5660
		ret = io_read(req, force_nonblock, cs);
5661
		break;
5662
	case IORING_OP_WRITEV:
5663
	case IORING_OP_WRITE_FIXED:
5664
	case IORING_OP_WRITE:
5665 5666 5667 5668 5669
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5670
		ret = io_write(req, force_nonblock, cs);
J
Jens Axboe 已提交
5671
		break;
C
Christoph Hellwig 已提交
5672
	case IORING_OP_FSYNC:
5673 5674 5675 5676 5677
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
5678
		ret = io_fsync(req, force_nonblock);
C
Christoph Hellwig 已提交
5679
		break;
5680
	case IORING_OP_POLL_ADD:
5681 5682 5683 5684 5685
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
5686
		ret = io_poll_add(req);
5687 5688
		break;
	case IORING_OP_POLL_REMOVE:
5689 5690 5691 5692 5693
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
5694
		ret = io_poll_remove(req);
5695
		break;
5696
	case IORING_OP_SYNC_FILE_RANGE:
5697 5698 5699 5700 5701
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
5702
		ret = io_sync_file_range(req, force_nonblock);
5703
		break;
J
Jens Axboe 已提交
5704
	case IORING_OP_SENDMSG:
5705
	case IORING_OP_SEND:
5706 5707 5708 5709 5710
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
5711
		if (req->opcode == IORING_OP_SENDMSG)
5712
			ret = io_sendmsg(req, force_nonblock, cs);
5713
		else
5714
			ret = io_send(req, force_nonblock, cs);
J
Jens Axboe 已提交
5715
		break;
J
Jens Axboe 已提交
5716
	case IORING_OP_RECVMSG:
5717
	case IORING_OP_RECV:
5718 5719 5720 5721 5722
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
5723
		if (req->opcode == IORING_OP_RECVMSG)
5724
			ret = io_recvmsg(req, force_nonblock, cs);
5725
		else
5726
			ret = io_recv(req, force_nonblock, cs);
J
Jens Axboe 已提交
5727
		break;
J
Jens Axboe 已提交
5728
	case IORING_OP_TIMEOUT:
5729 5730 5731 5732 5733
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
5734
		ret = io_timeout(req);
J
Jens Axboe 已提交
5735
		break;
5736
	case IORING_OP_TIMEOUT_REMOVE:
5737 5738 5739 5740 5741
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
5742
		ret = io_timeout_remove(req);
5743
		break;
5744
	case IORING_OP_ACCEPT:
5745 5746 5747 5748 5749
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
5750
		ret = io_accept(req, force_nonblock, cs);
5751
		break;
5752
	case IORING_OP_CONNECT:
5753 5754 5755 5756 5757
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
5758
		ret = io_connect(req, force_nonblock, cs);
5759
		break;
5760
	case IORING_OP_ASYNC_CANCEL:
5761 5762 5763 5764 5765
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
5766
		ret = io_async_cancel(req);
5767
		break;
5768 5769 5770 5771 5772 5773
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
5774
		ret = io_fallocate(req, force_nonblock);
5775
		break;
5776 5777 5778 5779 5780 5781
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
5782
		ret = io_openat(req, force_nonblock);
5783
		break;
5784 5785 5786 5787 5788 5789
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
5790
		ret = io_close(req, force_nonblock, cs);
5791
		break;
5792 5793 5794 5795 5796 5797
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
5798
		ret = io_files_update(req, force_nonblock, cs);
5799
		break;
5800 5801 5802 5803 5804 5805
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
5806
		ret = io_statx(req, force_nonblock);
5807
		break;
J
Jens Axboe 已提交
5808 5809 5810 5811 5812 5813
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
5814
		ret = io_fadvise(req, force_nonblock);
J
Jens Axboe 已提交
5815
		break;
J
Jens Axboe 已提交
5816 5817 5818 5819 5820 5821
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
5822
		ret = io_madvise(req, force_nonblock);
J
Jens Axboe 已提交
5823
		break;
5824 5825 5826 5827 5828 5829
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
5830
		ret = io_openat2(req, force_nonblock);
5831
		break;
5832 5833 5834 5835 5836 5837
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
5838
		ret = io_epoll_ctl(req, force_nonblock, cs);
5839
		break;
P
Pavel Begunkov 已提交
5840 5841 5842 5843 5844 5845
	case IORING_OP_SPLICE:
		if (sqe) {
			ret = io_splice_prep(req, sqe);
			if (ret < 0)
				break;
		}
5846
		ret = io_splice(req, force_nonblock);
P
Pavel Begunkov 已提交
5847
		break;
5848 5849 5850 5851 5852 5853
	case IORING_OP_PROVIDE_BUFFERS:
		if (sqe) {
			ret = io_provide_buffers_prep(req, sqe);
			if (ret)
				break;
		}
5854
		ret = io_provide_buffers(req, force_nonblock, cs);
5855
		break;
5856 5857 5858 5859 5860 5861
	case IORING_OP_REMOVE_BUFFERS:
		if (sqe) {
			ret = io_remove_buffers_prep(req, sqe);
			if (ret)
				break;
		}
5862
		ret = io_remove_buffers(req, force_nonblock, cs);
5863
		break;
P
Pavel Begunkov 已提交
5864 5865 5866 5867 5868 5869 5870 5871
	case IORING_OP_TEE:
		if (sqe) {
			ret = io_tee_prep(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_tee(req, force_nonblock);
		break;
J
Jens Axboe 已提交
5872 5873 5874 5875 5876
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
5877 5878 5879
	if (ret)
		return ret;

5880 5881
	/* If the op doesn't have a file, we're not polling for it */
	if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
5882 5883 5884 5885 5886 5887
		const bool in_async = io_wq_current_is_worker();

		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

J
Jens Axboe 已提交
5888
		io_iopoll_req_issued(req);
5889 5890 5891

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
5892 5893 5894
	}

	return 0;
J
Jens Axboe 已提交
5895 5896
}

5897
static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work)
J
Jens Axboe 已提交
5898 5899
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5900
	struct io_kiocb *timeout;
5901
	int ret = 0;
J
Jens Axboe 已提交
5902

5903 5904 5905
	timeout = io_prep_linked_timeout(req);
	if (timeout)
		io_queue_linked_timeout(timeout);
5906

5907 5908 5909
	/* if NO_CANCEL is set, we must still run the work */
	if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
				IO_WQ_WORK_CANCEL) {
5910
		ret = -ECANCELED;
5911
	}
5912

5913 5914
	if (!ret) {
		do {
5915
			ret = io_issue_sqe(req, NULL, false, NULL);
5916 5917 5918 5919 5920 5921 5922 5923 5924 5925
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
5926

5927
	if (ret) {
J
Jens Axboe 已提交
5928
		req_set_fail_links(req);
5929
		io_req_complete(req, ret);
5930
	}
J
Jens Axboe 已提交
5931

5932
	return io_steal_work(req);
J
Jens Axboe 已提交
5933 5934
}

5935 5936 5937 5938 5939
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

5940
	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
5941
	return table->files[index & IORING_FILE_TABLE_MASK];
5942 5943
}

5944 5945
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
			int fd, struct file **out_file, bool fixed)
J
Jens Axboe 已提交
5946
{
5947
	struct io_ring_ctx *ctx = req->ctx;
5948
	struct file *file;
J
Jens Axboe 已提交
5949

5950
	if (fixed) {
5951
		if (unlikely(!ctx->file_data ||
J
Jens Axboe 已提交
5952 5953
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
5954
		fd = array_index_nospec(fd, ctx->nr_user_files);
5955
		file = io_file_from_index(ctx, fd);
5956 5957 5958 5959
		if (file) {
			req->fixed_file_refs = ctx->file_data->cur_refs;
			percpu_ref_get(req->fixed_file_refs);
		}
J
Jens Axboe 已提交
5960
	} else {
5961
		trace_io_uring_file_get(ctx, fd);
5962
		file = __io_file_get(state, fd);
J
Jens Axboe 已提交
5963 5964
	}

5965 5966 5967 5968 5969
	if (file || io_op_defs[req->opcode].needs_file_no_error) {
		*out_file = file;
		return 0;
	}
	return -EBADF;
J
Jens Axboe 已提交
5970 5971
}

5972
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
5973
			   int fd)
5974 5975 5976
{
	bool fixed;

5977
	fixed = (req->flags & REQ_F_FIXED_FILE) != 0;
5978
	if (unlikely(!fixed && io_async_submit(req->ctx)))
5979 5980 5981 5982 5983
		return -EBADF;

	return io_file_get(state, req, fd, &req->file, fixed);
}

5984
static int io_grab_files(struct io_kiocb *req)
5985 5986
{
	int ret = -EBADF;
5987
	struct io_ring_ctx *ctx = req->ctx;
5988

5989 5990
	io_req_init_async(req);

5991
	if (req->work.files || (req->flags & REQ_F_NO_FILE_TABLE))
5992
		return 0;
P
Pavel Begunkov 已提交
5993
	if (!ctx->ring_file)
5994 5995
		return -EBADF;

5996 5997 5998 5999 6000 6001 6002 6003
	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
P
Pavel Begunkov 已提交
6004
	if (fcheck(ctx->ring_fd) == ctx->ring_file) {
6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

6016 6017 6018 6019 6020 6021 6022
static inline int io_prep_work_files(struct io_kiocb *req)
{
	if (!io_op_defs[req->opcode].file_table)
		return 0;
	return io_grab_files(req);
}

6023
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
J
Jens Axboe 已提交
6024
{
6025 6026 6027
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
6028 6029 6030 6031 6032 6033 6034 6035 6036 6037
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
6038 6039 6040
	if (!list_empty(&req->link_list)) {
		prev = list_entry(req->link_list.prev, struct io_kiocb,
				  link_list);
6041
		if (refcount_inc_not_zero(&prev->refs)) {
6042
			list_del_init(&req->link_list);
6043 6044
			prev->flags &= ~REQ_F_LINK_TIMEOUT;
		} else
6045
			prev = NULL;
6046 6047 6048 6049 6050
	}

	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
J
Jens Axboe 已提交
6051
		req_set_fail_links(prev);
6052
		io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
6053
		io_put_req(prev);
6054
	} else {
6055
		io_req_complete(req, -ETIME);
6056 6057 6058 6059
	}
	return HRTIMER_NORESTART;
}

6060
static void __io_queue_linked_timeout(struct io_kiocb *req)
6061
{
6062 6063 6064 6065
	/*
	 * If the list is now empty, then our linked request finished before
	 * we got a chance to setup the timer
	 */
6066
	if (!list_empty(&req->link_list)) {
6067
		struct io_timeout_data *data = &req->io->timeout;
6068

6069 6070 6071
		data->timer.function = io_link_timeout_fn;
		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
				data->mode);
6072
	}
6073 6074 6075 6076 6077 6078 6079 6080
}

static void io_queue_linked_timeout(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	__io_queue_linked_timeout(req);
6081
	spin_unlock_irq(&ctx->completion_lock);
6082 6083

	/* drop submission reference */
6084 6085
	io_put_req(req);
}
6086

6087
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
6088 6089 6090
{
	struct io_kiocb *nxt;

6091
	if (!(req->flags & REQ_F_LINK_HEAD))
6092
		return NULL;
6093
	if (req->flags & REQ_F_LINK_TIMEOUT)
6094
		return NULL;
6095

6096 6097
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
					link_list);
6098
	if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
6099
		return NULL;
6100

6101 6102
	req->flags |= REQ_F_LINK_TIMEOUT;
	return nxt;
6103 6104
}

6105 6106
static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			   struct io_comp_state *cs)
J
Jens Axboe 已提交
6107
{
6108
	struct io_kiocb *linked_timeout;
6109
	struct io_kiocb *nxt;
6110
	const struct cred *old_creds = NULL;
6111
	int ret;
J
Jens Axboe 已提交
6112

6113 6114 6115
again:
	linked_timeout = io_prep_linked_timeout(req);

6116 6117
	if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
	    req->work.creds != current_cred()) {
6118 6119 6120 6121 6122 6123 6124 6125
		if (old_creds)
			revert_creds(old_creds);
		if (old_creds == req->work.creds)
			old_creds = NULL; /* restored original creds */
		else
			old_creds = override_creds(req->work.creds);
	}

6126
	ret = io_issue_sqe(req, sqe, true, cs);
6127 6128 6129 6130 6131

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
6132
	if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6133
		if (!io_arm_poll_handler(req)) {
6134
punt:
6135 6136
			ret = io_prep_work_files(req);
			if (unlikely(ret))
6137
				goto err;
6138 6139 6140 6141 6142
			/*
			 * Queued up for async execution, worker will release
			 * submit reference when the iocb is actually submitted.
			 */
			io_queue_async_work(req);
J
Jens Axboe 已提交
6143
		}
6144

6145 6146
		if (linked_timeout)
			io_queue_linked_timeout(linked_timeout);
6147
		goto exit;
J
Jens Axboe 已提交
6148
	}
6149

6150
	if (unlikely(ret)) {
6151
err:
6152 6153
		/* un-prep timeout, so it'll be killed as any other linked */
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
6154
		req_set_fail_links(req);
6155
		io_put_req(req);
6156
		io_req_complete(req, ret);
6157
		goto exit;
J
Jens Axboe 已提交
6158
	}
6159 6160 6161 6162 6163 6164

	/* drop submission reference */
	nxt = io_put_req_find_next(req);
	if (linked_timeout)
		io_queue_linked_timeout(linked_timeout);

6165 6166
	if (nxt) {
		req = nxt;
6167 6168 6169

		if (req->flags & REQ_F_FORCE_ASYNC)
			goto punt;
6170 6171
		goto again;
	}
6172
exit:
6173 6174
	if (old_creds)
		revert_creds(old_creds);
J
Jens Axboe 已提交
6175 6176
}

6177 6178
static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 struct io_comp_state *cs)
6179 6180 6181
{
	int ret;

6182
	ret = io_req_defer(req, sqe);
6183 6184
	if (ret) {
		if (ret != -EIOCBQUEUED) {
6185
fail_req:
J
Jens Axboe 已提交
6186
			req_set_fail_links(req);
6187 6188
			io_put_req(req);
			io_req_complete(req, ret);
6189
		}
6190
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
6191 6192
		if (!req->io) {
			ret = io_req_defer_prep(req, sqe);
6193
			if (unlikely(ret))
6194 6195 6196
				goto fail_req;
		}

J
Jens Axboe 已提交
6197 6198 6199 6200
		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
6201
		io_req_init_async(req);
J
Jens Axboe 已提交
6202 6203 6204
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
6205
		__io_queue_sqe(req, sqe, cs);
J
Jens Axboe 已提交
6206
	}
6207 6208
}

6209 6210
static inline void io_queue_link_head(struct io_kiocb *req,
				      struct io_comp_state *cs)
6211
{
6212
	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
6213 6214
		io_put_req(req);
		io_req_complete(req, -ECANCELED);
6215
	} else
6216
		io_queue_sqe(req, NULL, cs);
6217 6218
}

6219
static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
6220
			 struct io_kiocb **link, struct io_comp_state *cs)
J
Jens Axboe 已提交
6221
{
6222
	struct io_ring_ctx *ctx = req->ctx;
6223
	int ret;
J
Jens Axboe 已提交
6224 6225 6226 6227 6228 6229 6230 6231 6232

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
P
Pavel Begunkov 已提交
6233
		struct io_kiocb *head = *link;
J
Jens Axboe 已提交
6234

6235 6236 6237 6238 6239 6240 6241
		/*
		 * Taking sequential execution of a link, draining both sides
		 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
		 * requests in the link. So, it drains the head and the
		 * next after the link request. The last one is done via
		 * drain_next flag to persist the effect across calls.
		 */
6242
		if (req->flags & REQ_F_IO_DRAIN) {
6243 6244 6245
			head->flags |= REQ_F_IO_DRAIN;
			ctx->drain_next = 1;
		}
6246
		ret = io_req_defer_prep(req, sqe);
6247
		if (unlikely(ret)) {
J
Jens Axboe 已提交
6248
			/* fail even hard links since we don't submit */
P
Pavel Begunkov 已提交
6249
			head->flags |= REQ_F_FAIL_LINK;
6250
			return ret;
6251
		}
P
Pavel Begunkov 已提交
6252
		trace_io_uring_link(ctx, req, head);
6253
		io_get_req_task(req);
P
Pavel Begunkov 已提交
6254
		list_add_tail(&req->link_list, &head->link_list);
6255 6256

		/* last request of a link, enqueue the link */
6257
		if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
6258
			io_queue_link_head(head, cs);
6259 6260
			*link = NULL;
		}
J
Jens Axboe 已提交
6261
	} else {
6262 6263
		if (unlikely(ctx->drain_next)) {
			req->flags |= REQ_F_IO_DRAIN;
6264
			ctx->drain_next = 0;
6265
		}
6266
		if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
6267
			req->flags |= REQ_F_LINK_HEAD;
6268
			INIT_LIST_HEAD(&req->link_list);
6269

6270
			ret = io_req_defer_prep(req, sqe);
6271
			if (unlikely(ret))
6272 6273 6274
				req->flags |= REQ_F_FAIL_LINK;
			*link = req;
		} else {
6275
			io_queue_sqe(req, sqe, cs);
6276
		}
J
Jens Axboe 已提交
6277
	}
6278

6279
	return 0;
J
Jens Axboe 已提交
6280 6281
}

6282 6283 6284 6285 6286
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
6287 6288
	if (!list_empty(&state->comp.list))
		io_submit_flush_completions(&state->comp);
6289
	blk_finish_plug(&state->plug);
P
Pavel Begunkov 已提交
6290
	io_state_file_put(state);
J
Jens Axboe 已提交
6291
	if (state->free_reqs)
6292
		kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
6293 6294 6295 6296 6297 6298
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
6299
				  struct io_ring_ctx *ctx, unsigned int max_ios)
6300 6301
{
	blk_start_plug(&state->plug);
6302 6303 6304
#ifdef CONFIG_BLOCK
	state->plug.nowait = true;
#endif
6305 6306 6307
	state->comp.nr = 0;
	INIT_LIST_HEAD(&state->comp.list);
	state->comp.ctx = ctx;
J
Jens Axboe 已提交
6308
	state->free_reqs = 0;
6309 6310 6311 6312
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
6313 6314
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
6315
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
6316

6317 6318 6319 6320 6321 6322
	/*
	 * Ensure any loads from the SQEs are done at this point,
	 * since once we write the new head, the application could
	 * write new data to them.
	 */
	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
6323 6324 6325
}

/*
6326
 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
J
Jens Axboe 已提交
6327 6328 6329 6330 6331 6332
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
6333
static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
6334
{
6335
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
6336 6337 6338 6339 6340 6341 6342 6343 6344 6345
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
6346
	head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
6347 6348
	if (likely(head < ctx->sq_entries))
		return &ctx->sq_sqes[head];
J
Jens Axboe 已提交
6349 6350

	/* drop invalid entries */
6351
	ctx->cached_sq_dropped++;
6352
	WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
6353 6354 6355 6356 6357 6358
	return NULL;
}

static inline void io_consume_sqe(struct io_ring_ctx *ctx)
{
	ctx->cached_sq_head++;
J
Jens Axboe 已提交
6359 6360
}

6361 6362 6363 6364 6365 6366
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK|	\
				IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
				IOSQE_BUFFER_SELECT)

static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
		       const struct io_uring_sqe *sqe,
6367
		       struct io_submit_state *state)
6368
{
6369
	unsigned int sqe_flags;
6370
	int id;
6371

6372 6373 6374 6375 6376 6377 6378 6379
	req->opcode = READ_ONCE(sqe->opcode);
	req->user_data = READ_ONCE(sqe->user_data);
	req->io = NULL;
	req->file = NULL;
	req->ctx = ctx;
	req->flags = 0;
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
P
Pavel Begunkov 已提交
6380
	req->task = current;
6381
	req->result = 0;
6382 6383 6384 6385

	if (unlikely(req->opcode >= IORING_OP_LAST))
		return -EINVAL;

6386 6387
	if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
		return -EFAULT;
6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399

	sqe_flags = READ_ONCE(sqe->flags);
	/* enforce forwards compatibility on users */
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
		return -EINVAL;

	if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
	    !io_op_defs[req->opcode].buffer_select)
		return -EOPNOTSUPP;

	id = READ_ONCE(sqe->personality);
	if (id) {
6400
		io_req_init_async(req);
6401 6402 6403 6404 6405 6406 6407
		req->work.creds = idr_find(&ctx->personality_idr, id);
		if (unlikely(!req->work.creds))
			return -EINVAL;
		get_cred(req->work.creds);
	}

	/* same numerical values with corresponding REQ_F_*, safe to copy */
6408
	req->flags |= sqe_flags;
6409

6410 6411 6412 6413
	if (!io_op_defs[req->opcode].needs_file)
		return 0;

	return io_req_set_file(state, req, READ_ONCE(sqe->fd));
6414 6415
}

6416
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
6417
			  struct file *ring_file, int ring_fd)
J
Jens Axboe 已提交
6418
{
6419
	struct io_submit_state state;
J
Jens Axboe 已提交
6420 6421
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
J
Jens Axboe 已提交
6422

6423
	/* if we have a backlog and couldn't flush it all, return BUSY */
6424 6425 6426 6427 6428
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}
J
Jens Axboe 已提交
6429

6430 6431
	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
6432

6433 6434
	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;
J
Jens Axboe 已提交
6435

6436
	io_submit_state_start(&state, ctx, nr);
J
Jens Axboe 已提交
6437

P
Pavel Begunkov 已提交
6438 6439 6440
	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

J
Jens Axboe 已提交
6441
	for (i = 0; i < nr; i++) {
6442
		const struct io_uring_sqe *sqe;
6443
		struct io_kiocb *req;
6444
		int err;
6445

6446 6447 6448 6449 6450
		sqe = io_get_sqe(ctx);
		if (unlikely(!sqe)) {
			io_consume_sqe(ctx);
			break;
		}
6451
		req = io_alloc_req(ctx, &state);
6452 6453 6454
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
6455
			break;
6456
		}
6457

6458
		err = io_init_req(ctx, req, sqe, &state);
6459
		io_consume_sqe(ctx);
6460 6461 6462
		/* will complete beyond this point, count as submitted */
		submitted++;

6463
		if (unlikely(err)) {
6464
fail_req:
6465 6466
			io_put_req(req);
			io_req_complete(req, err);
6467 6468
			break;
		}
6469

6470
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6471
						true, io_async_submit(ctx));
6472
		err = io_submit_sqe(req, sqe, &link, &state.comp);
6473 6474
		if (err)
			goto fail_req;
J
Jens Axboe 已提交
6475 6476
	}

6477 6478 6479 6480 6481
	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
J
Jens Axboe 已提交
6482
	if (link)
6483
		io_queue_link_head(link, &state.comp);
6484
	io_submit_state_end(&state);
J
Jens Axboe 已提交
6485

6486 6487 6488
	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

J
Jens Axboe 已提交
6489 6490 6491
	return submitted;
}

6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506
static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
{
	/* Tell userspace we may need a wakeup call */
	spin_lock_irq(&ctx->completion_lock);
	ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
	spin_unlock_irq(&ctx->completion_lock);
}

static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
	spin_unlock_irq(&ctx->completion_lock);
}

J
Jens Axboe 已提交
6507 6508 6509
static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
6510
	const struct cred *old_cred;
J
Jens Axboe 已提交
6511 6512
	DEFINE_WAIT(wait);
	unsigned long timeout;
6513
	int ret = 0;
J
Jens Axboe 已提交
6514

6515
	complete(&ctx->sq_thread_comp);
6516

6517
	old_cred = override_creds(ctx->creds);
J
Jens Axboe 已提交
6518

6519
	timeout = jiffies + ctx->sq_thread_idle;
6520
	while (!kthread_should_park()) {
6521
		unsigned int to_submit;
J
Jens Axboe 已提交
6522

6523
		if (!list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
6524 6525
			unsigned nr_events = 0;

6526
			mutex_lock(&ctx->uring_lock);
6527
			if (!list_empty(&ctx->iopoll_list) && !need_resched())
6528
				io_do_iopoll(ctx, &nr_events, 0);
6529
			else
J
Jens Axboe 已提交
6530
				timeout = jiffies + ctx->sq_thread_idle;
6531
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
6532 6533
		}

6534
		to_submit = io_sqring_entries(ctx);
6535 6536 6537 6538 6539

		/*
		 * If submit got -EBUSY, flag us as needing the application
		 * to enter the kernel to reap and flush events.
		 */
6540
		if (!to_submit || ret == -EBUSY || need_resched()) {
6541 6542 6543 6544 6545 6546
			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
6547
			io_sq_thread_drop_mm();
6548

J
Jens Axboe 已提交
6549 6550 6551
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
6552 6553 6554
			 * to sleep. The exception is if we got EBUSY doing
			 * more IO, we should wait for the application to
			 * reap events and wake us up.
J
Jens Axboe 已提交
6555
			 */
6556
			if (!list_empty(&ctx->iopoll_list) || need_resched() ||
6557 6558
			    (!time_after(jiffies, timeout) && ret != -EBUSY &&
			    !percpu_ref_is_dying(&ctx->refs))) {
6559
				io_run_task_work();
6560
				cond_resched();
J
Jens Axboe 已提交
6561 6562 6563 6564 6565 6566
				continue;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);

6567 6568
			/*
			 * While doing polled IO, before going to sleep, we need
6569 6570 6571 6572
			 * to check if there are new reqs added to iopoll_list,
			 * it is because reqs may have been punted to io worker
			 * and will be added to iopoll_list later, hence check
			 * the iopoll_list again.
6573 6574
			 */
			if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6575
			    !list_empty_careful(&ctx->iopoll_list)) {
6576 6577 6578 6579
				finish_wait(&ctx->sqo_wait, &wait);
				continue;
			}

6580
			io_ring_set_wakeup_flag(ctx);
J
Jens Axboe 已提交
6581

6582
			to_submit = io_sqring_entries(ctx);
6583
			if (!to_submit || ret == -EBUSY) {
6584
				if (kthread_should_park()) {
J
Jens Axboe 已提交
6585 6586 6587
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
6588
				if (io_run_task_work()) {
6589
					finish_wait(&ctx->sqo_wait, &wait);
6590
					io_ring_clear_wakeup_flag(ctx);
6591 6592
					continue;
				}
J
Jens Axboe 已提交
6593 6594 6595 6596 6597
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

6598
				io_ring_clear_wakeup_flag(ctx);
6599
				ret = 0;
J
Jens Axboe 已提交
6600 6601 6602 6603
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

6604
			io_ring_clear_wakeup_flag(ctx);
J
Jens Axboe 已提交
6605 6606
		}

6607
		mutex_lock(&ctx->uring_lock);
6608 6609
		if (likely(!percpu_ref_is_dying(&ctx->refs)))
			ret = io_submit_sqes(ctx, to_submit, NULL, -1);
6610
		mutex_unlock(&ctx->uring_lock);
6611
		timeout = jiffies + ctx->sq_thread_idle;
J
Jens Axboe 已提交
6612 6613
	}

6614
	io_run_task_work();
6615

6616
	io_sq_thread_drop_mm();
6617
	revert_creds(old_cred);
6618

6619
	kthread_parkme();
6620

J
Jens Axboe 已提交
6621 6622 6623
	return 0;
}

6624 6625 6626 6627 6628 6629 6630
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

6631
static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
6632 6633 6634 6635
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
6636
	 * Wake up if we have enough events, or if a timeout occurred since we
6637 6638 6639
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
6640
	return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
6641 6642 6643 6644 6645 6646 6647 6648 6649
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

6650 6651
	/* use noflush == true, as we can't safely rely on locking context */
	if (!io_should_wake(iowq, true))
6652 6653 6654 6655 6656
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
6657 6658 6659 6660 6661 6662 6663
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
6664 6665 6666 6667 6668 6669 6670 6671 6672
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
6673
	struct io_rings *rings = ctx->rings;
6674
	int ret = 0;
J
Jens Axboe 已提交
6675

6676 6677 6678
	do {
		if (io_cqring_events(ctx, false) >= min_events)
			return 0;
6679
		if (!io_run_task_work())
6680 6681
			break;
	} while (1);
J
Jens Axboe 已提交
6682 6683

	if (sig) {
6684 6685 6686
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
6687
						      sigsz);
6688 6689
		else
#endif
6690
			ret = set_user_sigmask(sig, sigsz);
6691

J
Jens Axboe 已提交
6692 6693 6694 6695
		if (ret)
			return ret;
	}

6696
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
6697
	trace_io_uring_cqring_wait(ctx, min_events);
6698 6699 6700
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
6701
		/* make sure we run task_work before checking for signals */
6702 6703
		if (io_run_task_work())
			continue;
6704
		if (signal_pending(current)) {
6705 6706 6707 6708 6709 6710 6711 6712
			if (current->jobctl & JOBCTL_TASK_WORK) {
				spin_lock_irq(&current->sighand->siglock);
				current->jobctl &= ~JOBCTL_TASK_WORK;
				recalc_sigpending();
				spin_unlock_irq(&current->sighand->siglock);
				continue;
			}
			ret = -EINTR;
6713 6714
			break;
		}
6715 6716 6717
		if (io_should_wake(&iowq, false))
			break;
		schedule();
6718 6719 6720
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

6721
	restore_saved_sigmask_unless(ret == -EINTR);
J
Jens Axboe 已提交
6722

6723
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
6724 6725
}

J
Jens Axboe 已提交
6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

6739 6740 6741 6742 6743 6744 6745
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
6746 6747 6748
#endif
}

6749 6750 6751 6752 6753 6754 6755 6756
static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}

J
Jens Axboe 已提交
6757 6758
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
6759
	struct fixed_file_data *data = ctx->file_data;
6760
	struct fixed_file_ref_node *ref_node = NULL;
6761 6762
	unsigned nr_tables, i;

6763
	if (!data)
J
Jens Axboe 已提交
6764 6765
		return -ENXIO;

6766
	spin_lock(&data->lock);
6767 6768 6769
	if (!list_empty(&data->ref_list))
		ref_node = list_first_entry(&data->ref_list,
				struct fixed_file_ref_node, node);
6770
	spin_unlock(&data->lock);
6771 6772 6773 6774 6775 6776
	if (ref_node)
		percpu_ref_kill(&ref_node->refs);

	percpu_ref_kill(&data->refs);

	/* wait for all refs nodes to complete */
6777
	flush_delayed_work(&ctx->file_put_work);
6778
	wait_for_completion(&data->done);
6779

J
Jens Axboe 已提交
6780
	__io_sqe_files_unregister(ctx);
6781 6782
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
6783 6784
		kfree(data->table[i].files);
	kfree(data->table);
6785 6786
	percpu_ref_exit(&data->refs);
	kfree(data);
6787
	ctx->file_data = NULL;
J
Jens Axboe 已提交
6788 6789 6790 6791
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
6792 6793 6794
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
6795
		wait_for_completion(&ctx->sq_thread_comp);
6796 6797 6798 6799 6800
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
6801
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
6802 6803 6804 6805 6806
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
6807 6808
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
6809 6810
	io_sq_thread_stop(ctx);

6811 6812 6813
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827
	}
}

#if defined(CONFIG_UNIX)
/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
6828
	int i, nr_files;
J
Jens Axboe 已提交
6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

6842
	nr_files = 0;
J
Jens Axboe 已提交
6843 6844
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
6845 6846 6847
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
6848
			continue;
6849
		fpl->fp[nr_files] = get_file(file);
6850 6851
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
6852 6853
	}

6854 6855 6856 6857
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
6858
		skb->destructor = unix_destruct_scm;
6859 6860
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
6861

6862 6863 6864 6865 6866 6867
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
6898 6899 6900 6901
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

6914 6915 6916 6917 6918 6919
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
6920
		struct fixed_file_table *table = &ctx->file_data->table[i];
6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
6935
		struct fixed_file_table *table = &ctx->file_data->table[i];
6936 6937 6938 6939 6940
		kfree(table->files);
	}
	return 1;
}

6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003
static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
	fput(file);
#endif
}

struct io_file_put {
7004
	struct list_head list;
7005 7006 7007
	struct file *file;
};

7008
static void __io_file_put_work(struct fixed_file_ref_node *ref_node)
7009
{
7010 7011
	struct fixed_file_data *file_data = ref_node->file_data;
	struct io_ring_ctx *ctx = file_data->ctx;
7012
	struct io_file_put *pfile, *tmp;
7013 7014

	list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) {
7015
		list_del(&pfile->list);
7016 7017
		io_ring_file_put(ctx, pfile->file);
		kfree(pfile);
7018
	}
7019

7020 7021 7022
	spin_lock(&file_data->lock);
	list_del(&ref_node->node);
	spin_unlock(&file_data->lock);
7023 7024 7025 7026

	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
	percpu_ref_put(&file_data->refs);
7027
}
7028

7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046
static void io_file_put_work(struct work_struct *work)
{
	struct io_ring_ctx *ctx;
	struct llist_node *node;

	ctx = container_of(work, struct io_ring_ctx, file_put_work.work);
	node = llist_del_all(&ctx->file_put_llist);

	while (node) {
		struct fixed_file_ref_node *ref_node;
		struct llist_node *next = node->next;

		ref_node = llist_entry(node, struct fixed_file_ref_node, llist);
		__io_file_put_work(ref_node);
		node = next;
	}
}

7047
static void io_file_data_ref_zero(struct percpu_ref *ref)
7048
{
7049
	struct fixed_file_ref_node *ref_node;
7050 7051 7052
	struct io_ring_ctx *ctx;
	bool first_add;
	int delay = HZ;
7053

7054
	ref_node = container_of(ref, struct fixed_file_ref_node, refs);
7055
	ctx = ref_node->file_data->ctx;
7056

7057 7058
	if (percpu_ref_is_dying(&ctx->file_data->refs))
		delay = 0;
7059

7060 7061 7062 7063 7064
	first_add = llist_add(&ref_node->llist, &ctx->file_put_llist);
	if (!delay)
		mod_delayed_work(system_wq, &ctx->file_put_work, 0);
	else if (first_add)
		queue_delayed_work(system_wq, &ctx->file_put_work, delay);
7065
}
7066

7067 7068
static struct fixed_file_ref_node *alloc_fixed_file_ref_node(
			struct io_ring_ctx *ctx)
7069
{
7070
	struct fixed_file_ref_node *ref_node;
7071

7072 7073 7074
	ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
	if (!ref_node)
		return ERR_PTR(-ENOMEM);
7075

7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090
	if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero,
			    0, GFP_KERNEL)) {
		kfree(ref_node);
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&ref_node->node);
	INIT_LIST_HEAD(&ref_node->file_list);
	ref_node->file_data = ctx->file_data;
	return ref_node;
}

static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node)
{
	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
7091 7092
}

J
Jens Axboe 已提交
7093 7094 7095 7096
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
7097
	unsigned nr_tables;
7098
	struct file *file;
J
Jens Axboe 已提交
7099 7100
	int fd, ret = 0;
	unsigned i;
7101
	struct fixed_file_ref_node *ref_node;
J
Jens Axboe 已提交
7102

7103
	if (ctx->file_data)
J
Jens Axboe 已提交
7104 7105 7106 7107 7108 7109
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

7110 7111 7112 7113 7114
	ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
	if (!ctx->file_data)
		return -ENOMEM;
	ctx->file_data->ctx = ctx;
	init_completion(&ctx->file_data->done);
7115
	INIT_LIST_HEAD(&ctx->file_data->ref_list);
7116
	spin_lock_init(&ctx->file_data->lock);
7117

7118
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
7119 7120
	ctx->file_data->table = kcalloc(nr_tables,
					sizeof(struct fixed_file_table),
7121
					GFP_KERNEL);
7122 7123 7124
	if (!ctx->file_data->table) {
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7125
		return -ENOMEM;
7126 7127
	}

7128
	if (percpu_ref_init(&ctx->file_data->refs, io_file_ref_kill,
7129 7130 7131 7132
				PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7133
		return -ENOMEM;
7134
	}
J
Jens Axboe 已提交
7135

7136
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
7137 7138 7139 7140
		percpu_ref_exit(&ctx->file_data->refs);
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
7141 7142 7143
		return -ENOMEM;
	}

7144
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
7145 7146 7147
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
7148 7149 7150
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
7151 7152 7153 7154 7155
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
7156

7157
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7158
		index = i & IORING_FILE_TABLE_MASK;
7159
		file = fget(fd);
J
Jens Axboe 已提交
7160 7161

		ret = -EBADF;
7162
		if (!file)
J
Jens Axboe 已提交
7163
			break;
7164

J
Jens Axboe 已提交
7165 7166 7167 7168 7169 7170 7171
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
7172 7173
		if (file->f_op == &io_uring_fops) {
			fput(file);
J
Jens Axboe 已提交
7174 7175 7176
			break;
		}
		ret = 0;
7177
		table->files[index] = file;
J
Jens Axboe 已提交
7178 7179 7180
	}

	if (ret) {
7181 7182 7183 7184 7185 7186
		for (i = 0; i < ctx->nr_user_files; i++) {
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
7187
			kfree(ctx->file_data->table[i].files);
J
Jens Axboe 已提交
7188

7189
		percpu_ref_exit(&ctx->file_data->refs);
7190 7191 7192
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7193 7194 7195 7196 7197
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
7198
	if (ret) {
J
Jens Axboe 已提交
7199
		io_sqe_files_unregister(ctx);
7200 7201
		return ret;
	}
J
Jens Axboe 已提交
7202

7203 7204 7205 7206 7207 7208 7209
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node)) {
		io_sqe_files_unregister(ctx);
		return PTR_ERR(ref_node);
	}

	ctx->file_data->cur_refs = &ref_node->refs;
7210
	spin_lock(&ctx->file_data->lock);
7211
	list_add(&ref_node->node, &ctx->file_data->ref_list);
7212
	spin_unlock(&ctx->file_data->lock);
7213
	percpu_ref_get(&ctx->file_data->refs);
J
Jens Axboe 已提交
7214 7215 7216
	return ret;
}

7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259
static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

7260
static int io_queue_file_removal(struct fixed_file_data *data,
7261
				 struct file *file)
7262
{
7263
	struct io_file_put *pfile;
7264 7265
	struct percpu_ref *refs = data->cur_refs;
	struct fixed_file_ref_node *ref_node;
7266 7267

	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
7268 7269
	if (!pfile)
		return -ENOMEM;
7270

7271
	ref_node = container_of(refs, struct fixed_file_ref_node, refs);
7272
	pfile->file = file;
7273 7274
	list_add(&pfile->list, &ref_node->file_list);

7275
	return 0;
7276 7277 7278 7279 7280 7281 7282
}

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *up,
				 unsigned nr_args)
{
	struct fixed_file_data *data = ctx->file_data;
7283
	struct fixed_file_ref_node *ref_node;
7284
	struct file *file;
7285 7286 7287
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;
7288
	bool needs_switch = false;
7289

7290
	if (check_add_overflow(up->offset, nr_args, &done))
7291 7292 7293 7294
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

7295 7296 7297 7298
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node))
		return PTR_ERR(ref_node);

7299
	done = 0;
7300
	fds = u64_to_user_ptr(up->fds);
7301
	while (nr_args) {
7302 7303 7304
		struct fixed_file_table *table;
		unsigned index;

7305 7306 7307 7308 7309
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
7310 7311
		i = array_index_nospec(up->offset, ctx->nr_user_files);
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7312 7313
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
7314
			file = io_file_from_index(ctx, index);
7315 7316 7317
			err = io_queue_file_removal(data, file);
			if (err)
				break;
7318
			table->files[index] = NULL;
7319
			needs_switch = true;
7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339
		}
		if (fd != -1) {
			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
7340
			table->files[index] = file;
7341
			err = io_sqe_file_register(ctx, file, i);
7342 7343
			if (err) {
				fput(file);
7344
				break;
7345
			}
7346 7347 7348
		}
		nr_args--;
		done++;
7349 7350 7351
		up->offset++;
	}

7352 7353
	if (needs_switch) {
		percpu_ref_kill(data->cur_refs);
7354
		spin_lock(&data->lock);
7355 7356
		list_add(&ref_node->node, &data->ref_list);
		data->cur_refs = &ref_node->refs;
7357
		spin_unlock(&data->lock);
7358 7359 7360
		percpu_ref_get(&ctx->file_data->refs);
	} else
		destroy_fixed_file_ref_node(ref_node);
7361 7362 7363

	return done ? done : err;
}
7364

7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380
static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}
7381

7382
static void io_free_work(struct io_wq_work *work)
7383 7384 7385
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

7386
	/* Consider that io_steal_work() relies on this ref */
7387 7388 7389
	io_put_req(req);
}

7390 7391 7392 7393 7394 7395 7396 7397 7398 7399
static int io_init_wq_offload(struct io_ring_ctx *ctx,
			      struct io_uring_params *p)
{
	struct io_wq_data data;
	struct fd f;
	struct io_ring_ctx *ctx_attach;
	unsigned int concurrency;
	int ret = 0;

	data.user = ctx->user;
7400
	data.free_work = io_free_work;
7401
	data.do_work = io_wq_submit_work;
7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436

	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
		/* Do QD, or 4 * CPUS, whatever is smallest */
		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

		ctx->io_wq = io_wq_create(concurrency, &data);
		if (IS_ERR(ctx->io_wq)) {
			ret = PTR_ERR(ctx->io_wq);
			ctx->io_wq = NULL;
		}
		return ret;
	}

	f = fdget(p->wq_fd);
	if (!f.file)
		return -EBADF;

	if (f.file->f_op != &io_uring_fops) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx_attach = f.file->private_data;
	/* @io_wq is protected by holding the fd */
	if (!io_wq_get(ctx_attach->io_wq, &data)) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx->io_wq = ctx_attach->io_wq;
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
7437 7438
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
7439 7440 7441 7442 7443 7444
{
	int ret;

	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

J
Jens Axboe 已提交
7445
	if (ctx->flags & IORING_SETUP_SQPOLL) {
7446 7447 7448 7449
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

7450 7451 7452 7453
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
7454
		if (p->flags & IORING_SETUP_SQ_AFF) {
7455
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
7456

7457
			ret = -EINVAL;
7458 7459
			if (cpu >= nr_cpu_ids)
				goto err;
7460
			if (!cpu_online(cpu))
7461 7462
				goto err;

J
Jens Axboe 已提交
7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

7482 7483
	ret = io_init_wq_offload(ctx, p);
	if (ret)
J
Jens Axboe 已提交
7484 7485 7486 7487
		goto err;

	return 0;
err:
7488
	io_finish_async(ctx);
7489 7490 7491 7492
	if (ctx->sqo_mm) {
		mmdrop(ctx->sqo_mm);
		ctx->sqo_mm = NULL;
	}
J
Jens Axboe 已提交
7493 7494 7495
	return ret;
}

7496 7497
static inline void __io_unaccount_mem(struct user_struct *user,
				      unsigned long nr_pages)
J
Jens Axboe 已提交
7498 7499 7500 7501
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

7502 7503
static inline int __io_account_mem(struct user_struct *user,
				   unsigned long nr_pages)
J
Jens Axboe 已提交
7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

7521 7522
static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			     enum io_mem_account acct)
7523
{
7524
	if (ctx->limit_mem)
7525
		__io_unaccount_mem(ctx->user, nr_pages);
7526

7527 7528 7529 7530 7531 7532
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm -= nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7533 7534
}

7535 7536
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			  enum io_mem_account acct)
7537
{
7538 7539 7540 7541 7542 7543 7544 7545
	int ret;

	if (ctx->limit_mem) {
		ret = __io_account_mem(ctx->user, nr_pages);
		if (ret)
			return ret;
	}

7546 7547 7548 7549 7550 7551
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm += nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7552 7553 7554 7555

	return 0;
}

J
Jens Axboe 已提交
7556 7557
static void io_mem_free(void *ptr)
{
7558 7559 7560 7561
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
7562

7563
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

7592 7593 7594
	if (sq_offset)
		*sq_offset = off;

7595 7596 7597 7598 7599 7600 7601 7602 7603 7604
	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	return off;
}

J
Jens Axboe 已提交
7605 7606
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
7607
	size_t pages;
J
Jens Axboe 已提交
7608

7609 7610 7611 7612
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
7613

7614
	return pages;
J
Jens Axboe 已提交
7615 7616
}

7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
7628
			unpin_user_page(imu->bvec[j].bv_page);
7629

7630
		io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
7631
		kvfree(imu->bvec);
7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

7655
		dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
7693
			goto err;
7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

7713
		ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
7714 7715
		if (ret)
			goto err;
7716 7717 7718

		ret = 0;
		if (!pages || nr_pages > got_pages) {
7719 7720
			kvfree(vmas);
			kvfree(pages);
7721
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
7722
						GFP_KERNEL);
7723
			vmas = kvmalloc_array(nr_pages,
7724 7725 7726 7727
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
7728
				io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7729 7730 7731 7732 7733
				goto err;
			}
			got_pages = nr_pages;
		}

7734
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
7735 7736 7737
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
7738
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7739 7740 7741 7742
			goto err;
		}

		ret = 0;
7743
		mmap_read_lock(current->mm);
7744
		pret = pin_user_pages(ubuf, nr_pages,
7745 7746
				      FOLL_WRITE | FOLL_LONGTERM,
				      pages, vmas);
7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
7761
		mmap_read_unlock(current->mm);
7762 7763 7764 7765 7766
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
7767
			if (pret > 0)
7768
				unpin_user_pages(pages, pret);
7769
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7770
			kvfree(imu->bvec);
7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
7793 7794
	kvfree(pages);
	kvfree(vmas);
7795 7796
	return 0;
err:
7797 7798
	kvfree(pages);
	kvfree(vmas);
7799 7800 7801 7802
	io_sqe_buffer_unregister(ctx);
	return ret;
}

7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

7835 7836 7837 7838 7839
static int __io_destroy_buffers(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	struct io_buffer *buf = p;

7840
	__io_remove_buffers(ctx, buf, id, -1U);
7841 7842 7843 7844 7845 7846 7847 7848 7849
	return 0;
}

static void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
	idr_destroy(&ctx->io_buffer_idr);
}

J
Jens Axboe 已提交
7850 7851
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
7852
	io_finish_async(ctx);
7853
	io_sqe_buffer_unregister(ctx);
7854
	if (ctx->sqo_mm) {
J
Jens Axboe 已提交
7855
		mmdrop(ctx->sqo_mm);
7856 7857
		ctx->sqo_mm = NULL;
	}
J
Jens Axboe 已提交
7858

J
Jens Axboe 已提交
7859
	io_sqe_files_unregister(ctx);
7860
	io_eventfd_unregister(ctx);
7861
	io_destroy_buffers(ctx);
J
Jens Axboe 已提交
7862
	idr_destroy(&ctx->personality_idr);
J
Jens Axboe 已提交
7863

J
Jens Axboe 已提交
7864
#if defined(CONFIG_UNIX)
7865 7866
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
7867
		sock_release(ctx->ring_sock);
7868
	}
J
Jens Axboe 已提交
7869 7870
#endif

7871
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
7872 7873 7874 7875
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	free_uid(ctx->user);
7876
	put_cred(ctx->creds);
7877
	kfree(ctx->cancel_hash);
7878
	kmem_cache_free(req_cachep, ctx->fallback_req);
J
Jens Axboe 已提交
7879 7880 7881 7882 7883 7884 7885 7886 7887
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
7888 7889 7890 7891
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
7892
	smp_rmb();
7893 7894
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
7895
		mask |= EPOLLOUT | EPOLLWRNORM;
7896
	if (io_cqring_events(ctx, false))
J
Jens Axboe 已提交
7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919
static int io_remove_personalities(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	const struct cred *cred;

	cred = idr_remove(&ctx->personality_idr, id);
	if (cred)
		put_cred(cred);
	return 0;
}

7920 7921
static void io_ring_exit_work(struct work_struct *work)
{
7922 7923
	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
					       exit_work);
7924

7925 7926 7927 7928 7929 7930
	/*
	 * If we're doing polled IO and end up having requests being
	 * submitted async (out-of-line), then completions can come in while
	 * we're waiting for refs to drop. We need to reap these manually,
	 * as nobody else will be looking for them.
	 */
7931
	do {
7932 7933
		if (ctx->rings)
			io_cqring_overflow_flush(ctx, true);
7934 7935
		io_iopoll_try_reap_events(ctx);
	} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
7936 7937 7938
	io_ring_ctx_free(ctx);
}

J
Jens Axboe 已提交
7939 7940 7941 7942 7943 7944
static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
7945
	io_kill_timeouts(ctx);
7946
	io_poll_remove_all(ctx);
7947 7948 7949 7950

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

7951 7952 7953
	/* if we failed setting up the ctx, we might not have any rings */
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);
7954
	io_iopoll_try_reap_events(ctx);
7955
	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
7956 7957 7958 7959 7960 7961

	/*
	 * Do this upfront, so we won't have a grace period where the ring
	 * is closed but resources aren't reaped yet. This can cause
	 * spurious failure in setting up a new ring.
	 */
7962 7963
	io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
			 ACCT_LOCKED);
7964

7965
	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
7966 7967 7968 7969 7970 7971 7972
	/*
	 * Use system_unbound_wq to avoid spawning tons of event kworkers
	 * if we're exiting a ton of rings at the same time. It just adds
	 * noise and overhead, there's no discernable change in runtime
	 * over using system_wq.
	 */
	queue_work(system_unbound_wq, &ctx->exit_work);
J
Jens Axboe 已提交
7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

7984 7985 7986 7987 7988 7989 7990
static bool io_wq_files_match(struct io_wq_work *work, void *data)
{
	struct files_struct *files = data;

	return work->files == files;
}

7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055
/*
 * Returns true if 'preq' is the link parent of 'req'
 */
static bool io_match_link(struct io_kiocb *preq, struct io_kiocb *req)
{
	struct io_kiocb *link;

	if (!(preq->flags & REQ_F_LINK_HEAD))
		return false;

	list_for_each_entry(link, &preq->link_list, link_list) {
		if (link == req)
			return true;
	}

	return false;
}

/*
 * We're looking to cancel 'req' because it's holding on to our files, but
 * 'req' could be a link to another request. See if it is, and cancel that
 * parent request if so.
 */
static bool io_poll_remove_link(struct io_ring_ctx *ctx, struct io_kiocb *req)
{
	struct hlist_node *tmp;
	struct io_kiocb *preq;
	bool found = false;
	int i;

	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(preq, tmp, list, hash_node) {
			found = io_match_link(preq, req);
			if (found) {
				io_poll_remove_one(preq);
				break;
			}
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	return found;
}

static bool io_timeout_remove_link(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	struct io_kiocb *preq;
	bool found = false;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry(preq, &ctx->timeout_list, timeout.list) {
		found = io_match_link(preq, req);
		if (found) {
			__io_timeout_cancel(preq);
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	return found;
}

8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082
static bool io_cancel_link_cb(struct io_wq_work *work, void *data)
{
	return io_match_link(container_of(work, struct io_kiocb, work), data);
}

static void io_attempt_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
{
	enum io_wq_cancel cret;

	/* cancel this particular work, if it's running */
	cret = io_wq_cancel_work(ctx->io_wq, &req->work);
	if (cret != IO_WQ_CANCEL_NOTFOUND)
		return;

	/* find links that hold this pending, cancel those */
	cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_link_cb, req, true);
	if (cret != IO_WQ_CANCEL_NOTFOUND)
		return;

	/* if we have a poll link holding this pending, cancel that */
	if (io_poll_remove_link(ctx, req))
		return;

	/* final option, timeout link is holding this req pending */
	io_timeout_remove_link(ctx, req);
}

8083 8084 8085
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
8086 8087 8088 8089 8090 8091
	if (list_empty_careful(&ctx->inflight_list))
		return;

	/* cancel all at once, should be faster than doing it one by one*/
	io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);

8092
	while (!list_empty_careful(&ctx->inflight_list)) {
8093 8094
		struct io_kiocb *cancel_req = NULL, *req;
		DEFINE_WAIT(wait);
8095 8096 8097

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
8098 8099 8100 8101 8102 8103 8104
			if (req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
8105
		}
8106
		if (cancel_req)
8107
			prepare_to_wait(&ctx->inflight_wait, &wait,
8108
						TASK_UNINTERRUPTIBLE);
8109 8110
		spin_unlock_irq(&ctx->inflight_lock);

8111 8112
		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
8113
			break;
8114 8115 8116
		/* cancel this request, or head link requests */
		io_attempt_cancel(ctx, cancel_req);
		io_put_req(cancel_req);
8117
		schedule();
8118
		finish_wait(&ctx->inflight_wait, &wait);
8119 8120 8121
	}
}

8122
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
8123
{
8124 8125
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct task_struct *task = data;
8126

8127
	return req->task == task;
8128 8129
}

8130 8131 8132 8133 8134
static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
8135 8136 8137 8138

	/*
	 * If the task is going away, cancel work it may have pending
	 */
8139 8140
	if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
		io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
8141

8142 8143 8144
	return 0;
}

8145 8146
static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
J
Jens Axboe 已提交
8147 8148
{
	struct io_ring_ctx *ctx = file->private_data;
8149
	loff_t offset = pgoff << PAGE_SHIFT;
J
Jens Axboe 已提交
8150 8151 8152 8153 8154
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
8155 8156
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
8157 8158 8159 8160 8161
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
8162
		return ERR_PTR(-EINVAL);
J
Jens Axboe 已提交
8163 8164 8165
	}

	page = virt_to_head_page(ptr);
8166
	if (sz > page_size(page))
8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182
		return ERR_PTR(-EINVAL);

	return ptr;
}

#ifdef CONFIG_MMU

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);
J
Jens Axboe 已提交
8183 8184 8185 8186 8187

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214
#else /* !CONFIG_MMU */

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
}

static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
{
	return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
}

static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len,
	unsigned long pgoff, unsigned long flags)
{
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, pgoff, len);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	return (unsigned long) ptr;
}

#endif /* !CONFIG_MMU */

J
Jens Axboe 已提交
8215 8216 8217 8218 8219 8220 8221 8222 8223
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

8224
	io_run_task_work();
8225

J
Jens Axboe 已提交
8226
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
8242 8243 8244 8245 8246
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
8247
	ret = 0;
J
Jens Axboe 已提交
8248
	if (ctx->flags & IORING_SETUP_SQPOLL) {
8249 8250
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false);
J
Jens Axboe 已提交
8251 8252 8253
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
8254
	} else if (to_submit) {
J
Jens Axboe 已提交
8255
		mutex_lock(&ctx->uring_lock);
8256
		submitted = io_submit_sqes(ctx, to_submit, f.file, fd);
J
Jens Axboe 已提交
8257
		mutex_unlock(&ctx->uring_lock);
8258 8259 8260

		if (submitted != to_submit)
			goto out;
J
Jens Axboe 已提交
8261 8262 8263 8264
	}
	if (flags & IORING_ENTER_GETEVENTS) {
		min_complete = min(min_complete, ctx->cq_entries);

8265 8266 8267 8268 8269 8270 8271 8272
		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
8273
			ret = io_iopoll_check(ctx, min_complete);
J
Jens Axboe 已提交
8274 8275 8276
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
8277 8278
	}

8279
out:
8280
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
8281 8282 8283 8284 8285
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

8286
#ifdef CONFIG_PROC_FS
8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347
static int io_uring_show_cred(int id, void *p, void *data)
{
	const struct cred *cred = p;
	struct seq_file *m = data;
	struct user_namespace *uns = seq_user_ns(m);
	struct group_info *gi;
	kernel_cap_t cap;
	unsigned __capi;
	int g;

	seq_printf(m, "%5d\n", id);
	seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
	seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
	seq_puts(m, "\n\tGroups:\t");
	gi = cred->group_info;
	for (g = 0; g < gi->ngroups; g++) {
		seq_put_decimal_ull(m, g ? " " : "",
					from_kgid_munged(uns, gi->gid[g]));
	}
	seq_puts(m, "\n\tCapEff:\t");
	cap = cred->cap_effective;
	CAP_FOR_EACH_U32(__capi)
		seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
	seq_putc(m, '\n');
	return 0;
}

static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	int i;

	mutex_lock(&ctx->uring_lock);
	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (!idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358
	seq_printf(m, "PollList:\n");
	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list = &ctx->cancel_hash[i];
		struct io_kiocb *req;

		hlist_for_each_entry(req, list, hash_node)
			seq_printf(m, "  op=%d, task_works=%d\n", req->opcode,
					req->task->task_works != NULL);
	}
	spin_unlock_irq(&ctx->completion_lock);
8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370
	mutex_unlock(&ctx->uring_lock);
}

static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
{
	struct io_ring_ctx *ctx = f->private_data;

	if (percpu_ref_tryget(&ctx->refs)) {
		__io_uring_show_fdinfo(ctx, m);
		percpu_ref_put(&ctx->refs);
	}
}
8371
#endif
8372

J
Jens Axboe 已提交
8373 8374
static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
8375
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
8376
	.mmap		= io_uring_mmap,
8377 8378 8379 8380
#ifndef CONFIG_MMU
	.get_unmapped_area = io_uring_nommu_get_unmapped_area,
	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
#endif
J
Jens Axboe 已提交
8381 8382
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
8383
#ifdef CONFIG_PROC_FS
8384
	.show_fdinfo	= io_uring_show_fdinfo,
8385
#endif
J
Jens Axboe 已提交
8386 8387 8388 8389 8390
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
8391 8392
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
8393

8394 8395 8396 8397
	/* make sure these are sane, as we already accounted them */
	ctx->sq_entries = p->sq_entries;
	ctx->cq_entries = p->cq_entries;

8398 8399 8400 8401 8402 8403
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
8404 8405
		return -ENOMEM;

8406 8407 8408 8409 8410 8411 8412 8413
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
J
Jens Axboe 已提交
8414 8415

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
8416 8417 8418
	if (size == SIZE_MAX) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8419
		return -EOVERFLOW;
8420
	}
J
Jens Axboe 已提交
8421 8422

	ctx->sq_sqes = io_mem_alloc(size);
8423 8424 8425
	if (!ctx->sq_sqes) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8426
		return -ENOMEM;
8427
	}
J
Jens Axboe 已提交
8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

8475 8476
static int io_uring_create(unsigned entries, struct io_uring_params *p,
			   struct io_uring_params __user *params)
J
Jens Axboe 已提交
8477 8478 8479
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
8480
	bool limit_mem;
J
Jens Axboe 已提交
8481 8482
	int ret;

8483
	if (!entries)
J
Jens Axboe 已提交
8484
		return -EINVAL;
8485 8486 8487 8488 8489
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}
J
Jens Axboe 已提交
8490 8491 8492 8493 8494

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
8495 8496 8497
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
8498 8499
	 */
	p->sq_entries = roundup_pow_of_two(entries);
8500 8501 8502 8503 8504 8505
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
8506
		if (p->cq_entries < p->sq_entries)
8507
			return -EINVAL;
8508 8509 8510 8511 8512
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
8513 8514 8515 8516
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
8517 8518

	user = get_uid(current_user());
8519
	limit_mem = !capable(CAP_IPC_LOCK);
J
Jens Axboe 已提交
8520

8521
	if (limit_mem) {
8522
		ret = __io_account_mem(user,
J
Jens Axboe 已提交
8523 8524 8525 8526 8527 8528 8529 8530 8531
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
8532
		if (limit_mem)
8533
			__io_unaccount_mem(user, ring_pages(p->sq_entries,
J
Jens Axboe 已提交
8534 8535 8536 8537 8538 8539
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->user = user;
8540
	ctx->creds = get_current_cred();
J
Jens Axboe 已提交
8541

8542 8543 8544 8545 8546 8547 8548 8549 8550 8551
	/*
	 * Account memory _before_ installing the file descriptor. Once
	 * the descriptor is installed, it can get closed at any time. Also
	 * do this before hitting the general error path, as ring freeing
	 * will un-account as well.
	 */
	io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
		       ACCT_LOCKED);
	ctx->limit_mem = limit_mem;

J
Jens Axboe 已提交
8552 8553 8554 8555
	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
8556
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
8557 8558 8559 8560
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
8561 8562 8563 8564 8565 8566 8567
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
8568 8569

	memset(&p->cq_off, 0, sizeof(p->cq_off));
8570 8571 8572 8573 8574 8575
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
8576
	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
8577

8578 8579
	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
8580 8581
			IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
			IORING_FEAT_POLL_32BITS;
8582 8583 8584 8585 8586

	if (copy_to_user(params, p, sizeof(*p))) {
		ret = -EFAULT;
		goto err;
	}
8587

8588 8589 8590 8591 8592 8593 8594 8595
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

8596
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
J
Jens Axboe 已提交
8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
8620
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8621
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
8622
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
J
Jens Axboe 已提交
8623 8624
		return -EINVAL;

8625
	return  io_uring_create(entries, &p, params);
J
Jens Axboe 已提交
8626 8627 8628 8629 8630 8631 8632 8633
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672
static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}

8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711
static int io_register_personality(struct io_ring_ctx *ctx)
{
	const struct cred *creds = get_current_cred();
	int id;

	id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
				USHRT_MAX, GFP_KERNEL);
	if (id < 0)
		put_cred(creds);
	return id;
}

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}

static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}

8712 8713
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
8714 8715
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
8716 8717 8718
{
	int ret;

8719 8720 8721 8722 8723 8724 8725 8726
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

8727
	if (io_register_op_must_quiesce(opcode)) {
8728
		percpu_ref_kill(&ctx->refs);
8729

8730 8731 8732 8733 8734 8735 8736 8737 8738
		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
8739
		ret = wait_for_completion_interruptible(&ctx->ref_comp);
8740
		mutex_lock(&ctx->uring_lock);
8741 8742 8743 8744 8745
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
8746
	}
8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
8758 8759 8760 8761 8762 8763 8764 8765 8766
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
8767 8768 8769
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
8770
	case IORING_REGISTER_EVENTFD:
8771
	case IORING_REGISTER_EVENTFD_ASYNC:
8772 8773 8774 8775
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
8776 8777 8778 8779 8780 8781
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
8782 8783 8784 8785 8786 8787 8788
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
8789 8790 8791 8792 8793 8794
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
8807 8808 8809 8810 8811
	default:
		ret = -EINVAL;
		break;
	}

8812
	if (io_register_op_must_quiesce(opcode)) {
8813 8814
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
8815
out:
8816
		reinit_completion(&ctx->ref_comp);
8817
	}
8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
8841 8842
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
8843 8844 8845 8846 8847
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
8848 8849
static int __init io_uring_init(void)
{
8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
P
Pavel Begunkov 已提交
8865
	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
8866 8867 8868 8869 8870
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
8871 8872
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
8873 8874 8875 8876 8877 8878 8879 8880
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
P
Pavel Begunkov 已提交
8881
	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
8882 8883 8884
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
P
Pavel Begunkov 已提交
8885
	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);
8886

8887
	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
8888
	BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
J
Jens Axboe 已提交
8889 8890 8891 8892
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);