io_uring.c 213.0 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
47
#include <net/compat.h>
J
Jens Axboe 已提交
48 49
#include <linux/refcount.h>
#include <linux/uio.h>
50
#include <linux/bits.h>
J
Jens Axboe 已提交
51 52 53 54 55 56 57 58 59

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
60
#include <linux/kthread.h>
J
Jens Axboe 已提交
61
#include <linux/blkdev.h>
62
#include <linux/bvec.h>
J
Jens Axboe 已提交
63 64 65
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
66
#include <net/scm.h>
J
Jens Axboe 已提交
67 68 69 70
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
71 72
#include <linux/sizes.h>
#include <linux/hugetlb.h>
73
#include <linux/highmem.h>
74 75
#include <linux/namei.h>
#include <linux/fsnotify.h>
J
Jens Axboe 已提交
76
#include <linux/fadvise.h>
77
#include <linux/eventpoll.h>
78
#include <linux/fs_struct.h>
P
Pavel Begunkov 已提交
79
#include <linux/splice.h>
80
#include <linux/task_work.h>
81
#include <linux/pagemap.h>
J
Jens Axboe 已提交
82

83 84 85
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
86 87 88
#include <uapi/linux/io_uring.h>

#include "internal.h"
89
#include "io-wq.h"
J
Jens Axboe 已提交
90

91
#define IORING_MAX_ENTRIES	32768
92
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
93 94 95 96 97 98 99 100

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
101 102 103 104 105 106

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
107
/*
108 109
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
110 111 112 113
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
114
struct io_rings {
S
Stefan Bühler 已提交
115 116 117 118
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
119 120 121
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
122
	 */
123
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
124
	/*
125
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
126 127
	 * ring_entries - 1)
	 */
128 129 130
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
131 132 133 134 135 136 137 138 139 140 141 142
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
143
	u32			sq_dropped;
S
Stefan Bühler 已提交
144
	/*
145
	 * Runtime SQ flags
S
Stefan Bühler 已提交
146 147 148 149 150 151 152
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
153
	u32			sq_flags;
154 155 156 157 158 159 160
	/*
	 * Runtime CQ flags
	 *
	 * Written by the application, shouldn't be modified by the
	 * kernel.
	 */
	u32                     cq_flags;
S
Stefan Bühler 已提交
161 162 163
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
L
LimingWu 已提交
164
	 * there are not more requests pending than there is space in
S
Stefan Bühler 已提交
165 166 167 168 169 170 171 172 173
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
174
	u32			cq_overflow;
S
Stefan Bühler 已提交
175 176 177 178 179 180 181
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
182
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
183 184
};

185 186 187 188 189 190 191
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

192 193
struct fixed_file_table {
	struct file		**files;
194 195
};

196 197 198 199 200
struct fixed_file_ref_node {
	struct percpu_ref		refs;
	struct list_head		node;
	struct list_head		file_list;
	struct fixed_file_data		*file_data;
201
	struct llist_node		llist;
202 203
};

204 205 206 207
struct fixed_file_data {
	struct fixed_file_table		*table;
	struct io_ring_ctx		*ctx;

208
	struct percpu_ref		*cur_refs;
209 210
	struct percpu_ref		refs;
	struct completion		done;
211 212
	struct list_head		ref_list;
	spinlock_t			lock;
213 214
};

215 216 217 218 219 220 221
struct io_buffer {
	struct list_head list;
	__u64 addr;
	__s32 len;
	__u16 bid;
};

J
Jens Axboe 已提交
222 223 224 225 226 227 228
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
229
		unsigned int		compat: 1;
230
		unsigned int		limit_mem: 1;
231 232 233
		unsigned int		cq_overflow_flushed: 1;
		unsigned int		drain_next: 1;
		unsigned int		eventfd_async: 1;
J
Jens Axboe 已提交
234

235 236 237 238 239 240 241 242 243 244 245 246
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
247 248 249
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
250
		unsigned		sq_thread_idle;
251
		unsigned		cached_sq_dropped;
252
		atomic_t		cached_cq_overflow;
253
		unsigned long		sq_check_overflow;
254 255

		struct list_head	defer_list;
J
Jens Axboe 已提交
256
		struct list_head	timeout_list;
257
		struct list_head	cq_overflow_list;
258 259

		wait_queue_head_t	inflight_wait;
260
		struct io_uring_sqe	*sq_sqes;
J
Jens Axboe 已提交
261 262
	} ____cacheline_aligned_in_smp;

263 264
	struct io_rings	*rings;

J
Jens Axboe 已提交
265
	/* IO offload */
266
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
267
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
268
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
269
	wait_queue_head_t	sqo_wait;
270

J
Jens Axboe 已提交
271 272 273 274 275
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
276
	struct fixed_file_data	*file_data;
J
Jens Axboe 已提交
277
	unsigned		nr_user_files;
P
Pavel Begunkov 已提交
278 279
	int 			ring_fd;
	struct file 		*ring_file;
J
Jens Axboe 已提交
280

281 282 283 284
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
285 286
	struct user_struct	*user;

287
	const struct cred	*creds;
288

289 290
	struct completion	ref_comp;
	struct completion	sq_thread_comp;
291

292 293 294
	/* if all else fails... */
	struct io_kiocb		*fallback_req;

295 296 297 298
#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif

299 300
	struct idr		io_buffer_idr;

301 302
	struct idr		personality_idr;

303 304 305 306 307
	struct {
		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		unsigned		cq_mask;
		atomic_t		cq_timeouts;
308
		unsigned long		cq_check_overflow;
309 310 311 312
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
		struct eventfd_ctx	*cq_ev_fd;
	} ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
313 314 315 316 317 318 319 320

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
321

J
Jens Axboe 已提交
322
		/*
323
		 * ->iopoll_list is protected by the ctx->uring_lock for
J
Jens Axboe 已提交
324 325 326 327
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
328
		struct list_head	iopoll_list;
329 330
		struct hlist_head	*cancel_hash;
		unsigned		cancel_hash_bits;
331
		bool			poll_multi_file;
332

333 334
		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
335
	} ____cacheline_aligned_in_smp;
336

337 338 339
	struct delayed_work		file_put_work;
	struct llist_head		file_put_llist;

340
	struct work_struct		exit_work;
J
Jens Axboe 已提交
341 342
};

J
Jens Axboe 已提交
343 344 345 346
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
347 348
struct io_poll_iocb {
	struct file			*file;
349 350 351 352
	union {
		struct wait_queue_head	*head;
		u64			addr;
	};
353
	__poll_t			events;
J
Jens Axboe 已提交
354
	bool				done;
355
	bool				canceled;
356
	struct wait_queue_entry		wait;
357 358
};

359 360 361 362 363 364
struct io_close {
	struct file			*file;
	struct file			*put_file;
	int				fd;
};

365 366 367 368 369 370 371
struct io_timeout_data {
	struct io_kiocb			*req;
	struct hrtimer			timer;
	struct timespec64		ts;
	enum hrtimer_mode		mode;
};

372 373 374 375 376
struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
377
	unsigned long			nofile;
378 379 380 381 382 383 384
};

struct io_sync {
	struct file			*file;
	loff_t				len;
	loff_t				off;
	int				flags;
385
	int				mode;
386 387
};

388 389 390 391 392
struct io_cancel {
	struct file			*file;
	u64				addr;
};

393 394 395 396
struct io_timeout {
	struct file			*file;
	u64				addr;
	int				flags;
397 398
	u32				off;
	u32				target_seq;
P
Pavel Begunkov 已提交
399
	struct list_head		list;
400 401
};

402 403 404 405 406 407 408
struct io_rw {
	/* NOTE: kiocb has the file as the first member, so don't do it here */
	struct kiocb			kiocb;
	u64				addr;
	u64				len;
};

409 410 411 412 413 414
struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
};

415 416
struct io_sr_msg {
	struct file			*file;
417
	union {
418
		struct user_msghdr __user *umsg;
419 420
		void __user		*buf;
	};
421
	int				msg_flags;
422
	int				bgid;
423
	size_t				len;
424
	struct io_buffer		*kbuf;
425 426
};

427 428 429 430
struct io_open {
	struct file			*file;
	int				dfd;
	struct filename			*filename;
431
	struct open_how			how;
432
	unsigned long			nofile;
433 434
};

435 436 437 438 439 440 441
struct io_files_update {
	struct file			*file;
	u64				arg;
	u32				nr_args;
	u32				offset;
};

J
Jens Axboe 已提交
442 443 444 445 446 447 448
struct io_fadvise {
	struct file			*file;
	u64				offset;
	u32				len;
	u32				advice;
};

J
Jens Axboe 已提交
449 450 451 452 453 454 455
struct io_madvise {
	struct file			*file;
	u64				addr;
	u32				len;
	u32				advice;
};

456 457 458 459 460 461
struct io_epoll {
	struct file			*file;
	int				epfd;
	int				op;
	int				fd;
	struct epoll_event		event;
462 463
};

P
Pavel Begunkov 已提交
464 465 466 467 468 469 470 471 472
struct io_splice {
	struct file			*file_out;
	struct file			*file_in;
	loff_t				off_out;
	loff_t				off_in;
	u64				len;
	unsigned int			flags;
};

473 474 475 476 477 478 479 480 481
struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__s32				len;
	__u32				bgid;
	__u16				nbufs;
	__u16				bid;
};

482 483 484 485 486
struct io_statx {
	struct file			*file;
	int				dfd;
	unsigned int			mask;
	unsigned int			flags;
B
Bijan Mottahedeh 已提交
487
	const char __user		*filename;
488 489 490
	struct statx __user		*buffer;
};

491 492 493
struct io_completion {
	struct file			*file;
	struct list_head		list;
494
	int				cflags;
495 496
};

497 498 499 500
struct io_async_connect {
	struct sockaddr_storage		address;
};

501 502 503 504 505
struct io_async_msghdr {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	struct sockaddr __user		*uaddr;
	struct msghdr			msg;
506
	struct sockaddr_storage		addr;
507 508
};

509 510
struct io_async_rw {
	struct iovec			fast_iov[UIO_FASTIOV];
511 512
	const struct iovec		*free_iovec;
	struct iov_iter			iter;
513
	size_t				bytes_done;
514
	struct wait_page_queue		wpq;
515 516
};

517
struct io_async_ctx {
518 519
	union {
		struct io_async_rw	rw;
520
		struct io_async_msghdr	msg;
521
		struct io_async_connect	connect;
522
		struct io_timeout_data	timeout;
523
	};
524 525
};

526 527 528 529 530 531
enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
532
	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
533

534
	REQ_F_LINK_HEAD_BIT,
535 536 537 538 539 540 541
	REQ_F_FAIL_LINK_BIT,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_ISREG_BIT,
	REQ_F_COMP_LOCKED_BIT,
P
Pavel Begunkov 已提交
542
	REQ_F_NEED_CLEANUP_BIT,
543
	REQ_F_POLLED_BIT,
544
	REQ_F_BUFFER_SELECTED_BIT,
545
	REQ_F_NO_FILE_TABLE_BIT,
546
	REQ_F_WORK_INITIALIZED_BIT,
P
Pavel Begunkov 已提交
547
	REQ_F_TASK_PINNED_BIT,
548 549 550

	/* not a real bit, just to check we're not overflowing the space */
	__REQ_F_LAST_BIT,
551 552 553 554 555 556 557 558 559 560 561 562 563
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),
564 565
	/* IOSQE_BUFFER_SELECT */
	REQ_F_BUFFER_SELECT	= BIT(REQ_F_BUFFER_SELECT_BIT),
566

567 568
	/* head of a link */
	REQ_F_LINK_HEAD		= BIT(REQ_F_LINK_HEAD_BIT),
569 570 571 572 573 574 575 576 577 578 579 580 581 582
	/* fail rest of links */
	REQ_F_FAIL_LINK		= BIT(REQ_F_FAIL_LINK_BIT),
	/* on inflight list */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* has linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* completion under lock */
	REQ_F_COMP_LOCKED	= BIT(REQ_F_COMP_LOCKED_BIT),
P
Pavel Begunkov 已提交
583 584
	/* needs cleanup */
	REQ_F_NEED_CLEANUP	= BIT(REQ_F_NEED_CLEANUP_BIT),
585 586
	/* already went through poll handler */
	REQ_F_POLLED		= BIT(REQ_F_POLLED_BIT),
587 588
	/* buffer already selected */
	REQ_F_BUFFER_SELECTED	= BIT(REQ_F_BUFFER_SELECTED_BIT),
589 590
	/* doesn't need file table for this request */
	REQ_F_NO_FILE_TABLE	= BIT(REQ_F_NO_FILE_TABLE_BIT),
591 592
	/* io_wq_work is initialized */
	REQ_F_WORK_INITIALIZED	= BIT(REQ_F_WORK_INITIALIZED_BIT),
P
Pavel Begunkov 已提交
593 594
	/* req->task is refcounted */
	REQ_F_TASK_PINNED	= BIT(REQ_F_TASK_PINNED_BIT),
595 596 597 598
};

struct async_poll {
	struct io_poll_iocb	poll;
599
	struct io_poll_iocb	*double_poll;
600 601
};

J
Jens Axboe 已提交
602 603 604 605 606 607
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
608
struct io_kiocb {
609
	union {
J
Jens Axboe 已提交
610
		struct file		*file;
611
		struct io_rw		rw;
612
		struct io_poll_iocb	poll;
613 614
		struct io_accept	accept;
		struct io_sync		sync;
615
		struct io_cancel	cancel;
616
		struct io_timeout	timeout;
617
		struct io_connect	connect;
618
		struct io_sr_msg	sr_msg;
619
		struct io_open		open;
620
		struct io_close		close;
621
		struct io_files_update	files_update;
J
Jens Axboe 已提交
622
		struct io_fadvise	fadvise;
J
Jens Axboe 已提交
623
		struct io_madvise	madvise;
624
		struct io_epoll		epoll;
P
Pavel Begunkov 已提交
625
		struct io_splice	splice;
626
		struct io_provide_buf	pbuf;
627
		struct io_statx		statx;
628 629
		/* use only after cleaning per-op data, see io_clean_op() */
		struct io_completion	compl;
630
	};
J
Jens Axboe 已提交
631

632
	struct io_async_ctx		*io;
633
	u8				opcode;
634 635
	/* polled IO has completed */
	u8				iopoll_completed;
J
Jens Axboe 已提交
636

637
	u16				buf_index;
638
	u32				result;
639

P
Pavel Begunkov 已提交
640 641 642 643 644
	struct io_ring_ctx		*ctx;
	unsigned int			flags;
	refcount_t			refs;
	struct task_struct		*task;
	u64				user_data;
645

P
Pavel Begunkov 已提交
646
	struct list_head		link_list;
647

648 649 650 651
	/*
	 * 1. used with ctx->iopoll_list with reads/writes
	 * 2. to track reqs with ->files (see io_op_def::file_table)
	 */
P
Pavel Begunkov 已提交
652 653 654 655 656 657 658 659
	struct list_head		inflight_entry;

	struct percpu_ref		*fixed_file_refs;
	struct callback_head		task_work;
	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
	struct hlist_node		hash_node;
	struct async_poll		*apoll;
	struct io_wq_work		work;
J
Jens Axboe 已提交
660
};
661

662 663 664
struct io_defer_entry {
	struct list_head	list;
	struct io_kiocb		*req;
665
	u32			seq;
J
Jens Axboe 已提交
666 667
};

J
Jens Axboe 已提交
668
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
669

670 671 672 673 674 675
struct io_comp_state {
	unsigned int		nr;
	struct list_head	list;
	struct io_ring_ctx	*ctx;
};

676 677 678
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
679 680 681 682
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
683
	unsigned int		free_reqs;
J
Jens Axboe 已提交
684

685 686 687 688 689
	/*
	 * Batch completion logic
	 */
	struct io_comp_state	comp;

690 691 692 693 694 695 696 697 698
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		ios_left;
};

699 700 701 702 703 704 705
struct io_op_def {
	/* needs req->io allocated for deferral/async */
	unsigned		async_ctx : 1;
	/* needs current->mm setup, does mm access */
	unsigned		needs_mm : 1;
	/* needs req->file assigned */
	unsigned		needs_file : 1;
706 707
	/* don't fail if file grab fails */
	unsigned		needs_file_no_error : 1;
708 709 710 711
	/* hash wq insertion if file is a regular file */
	unsigned		hash_reg_file : 1;
	/* unbound wq insertion if file is a non-regular file */
	unsigned		unbound_nonreg_file : 1;
712 713
	/* opcode is not supported by this kernel */
	unsigned		not_supported : 1;
714 715
	/* needs file table */
	unsigned		file_table : 1;
716 717
	/* needs ->fs */
	unsigned		needs_fs : 1;
718 719 720
	/* set if opcode supports polled "wait" */
	unsigned		pollin : 1;
	unsigned		pollout : 1;
721 722
	/* op supports buffer selection */
	unsigned		buffer_select : 1;
723
	unsigned		needs_fsize : 1;
724 725 726
};

static const struct io_op_def io_op_defs[] = {
727 728
	[IORING_OP_NOP] = {},
	[IORING_OP_READV] = {
729 730 731 732
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
733
		.pollin			= 1,
734
		.buffer_select		= 1,
735
	},
736
	[IORING_OP_WRITEV] = {
737 738 739 740 741
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
742
		.pollout		= 1,
743
		.needs_fsize		= 1,
744
	},
745
	[IORING_OP_FSYNC] = {
746 747
		.needs_file		= 1,
	},
748
	[IORING_OP_READ_FIXED] = {
749 750
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
751
		.pollin			= 1,
752
	},
753
	[IORING_OP_WRITE_FIXED] = {
754 755 756
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
757
		.pollout		= 1,
758
		.needs_fsize		= 1,
759
	},
760
	[IORING_OP_POLL_ADD] = {
761 762 763
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
764 765
	[IORING_OP_POLL_REMOVE] = {},
	[IORING_OP_SYNC_FILE_RANGE] = {
766 767
		.needs_file		= 1,
	},
768
	[IORING_OP_SENDMSG] = {
769 770 771 772
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
773
		.needs_fs		= 1,
774
		.pollout		= 1,
775
	},
776
	[IORING_OP_RECVMSG] = {
777 778 779 780
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
781
		.needs_fs		= 1,
782
		.pollin			= 1,
783
		.buffer_select		= 1,
784
	},
785
	[IORING_OP_TIMEOUT] = {
786 787 788
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
789 790
	[IORING_OP_TIMEOUT_REMOVE] = {},
	[IORING_OP_ACCEPT] = {
791 792 793
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
794
		.file_table		= 1,
795
		.pollin			= 1,
796
	},
797 798
	[IORING_OP_ASYNC_CANCEL] = {},
	[IORING_OP_LINK_TIMEOUT] = {
799 800 801
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
802
	[IORING_OP_CONNECT] = {
803 804 805 806
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
807
		.pollout		= 1,
808
	},
809
	[IORING_OP_FALLOCATE] = {
810
		.needs_file		= 1,
811
		.needs_fsize		= 1,
812
	},
813
	[IORING_OP_OPENAT] = {
814
		.file_table		= 1,
815
		.needs_fs		= 1,
816
	},
817
	[IORING_OP_CLOSE] = {
818 819
		.needs_file		= 1,
		.needs_file_no_error	= 1,
820
		.file_table		= 1,
821
	},
822
	[IORING_OP_FILES_UPDATE] = {
823
		.needs_mm		= 1,
824
		.file_table		= 1,
825
	},
826
	[IORING_OP_STATX] = {
827
		.needs_mm		= 1,
828
		.needs_fs		= 1,
829
		.file_table		= 1,
830
	},
831
	[IORING_OP_READ] = {
832 833 834
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
835
		.pollin			= 1,
836
		.buffer_select		= 1,
837
	},
838
	[IORING_OP_WRITE] = {
839 840 841
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
842
		.pollout		= 1,
843
		.needs_fsize		= 1,
844
	},
845
	[IORING_OP_FADVISE] = {
J
Jens Axboe 已提交
846 847
		.needs_file		= 1,
	},
848
	[IORING_OP_MADVISE] = {
J
Jens Axboe 已提交
849 850
		.needs_mm		= 1,
	},
851
	[IORING_OP_SEND] = {
852 853 854
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
855
		.pollout		= 1,
856
	},
857
	[IORING_OP_RECV] = {
858 859 860
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
861
		.pollin			= 1,
862
		.buffer_select		= 1,
863
	},
864
	[IORING_OP_OPENAT2] = {
865
		.file_table		= 1,
866
		.needs_fs		= 1,
867
	},
868 869 870 871
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.file_table		= 1,
	},
P
Pavel Begunkov 已提交
872 873 874 875
	[IORING_OP_SPLICE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
876 877
	},
	[IORING_OP_PROVIDE_BUFFERS] = {},
878
	[IORING_OP_REMOVE_BUFFERS] = {},
P
Pavel Begunkov 已提交
879 880 881 882 883
	[IORING_OP_TEE] = {
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
884 885
};

886 887 888 889 890
enum io_mem_account {
	ACCT_LOCKED,
	ACCT_PINNED,
};

891 892
static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
			     struct io_comp_state *cs);
893
static void io_cqring_fill_event(struct io_kiocb *req, long res);
894
static void io_put_req(struct io_kiocb *req);
895
static void io_double_put_req(struct io_kiocb *req);
896
static void __io_double_put_req(struct io_kiocb *req);
897
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
898
static void __io_queue_linked_timeout(struct io_kiocb *req);
899
static void io_queue_linked_timeout(struct io_kiocb *req);
900 901 902
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *ip,
				 unsigned nr_args);
903
static int io_prep_work_files(struct io_kiocb *req);
904
static void __io_clean_op(struct io_kiocb *req);
905 906 907
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
		       int fd, struct file **out_file, bool fixed);
static void __io_queue_sqe(struct io_kiocb *req,
908 909
			   const struct io_uring_sqe *sqe,
			   struct io_comp_state *cs);
910
static void io_file_put_work(struct work_struct *work);
911

912 913 914
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock);
915 916
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
			     const struct iovec *fast_iov,
917
			     struct iov_iter *iter, bool force);
918

J
Jens Axboe 已提交
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

P
Pavel Begunkov 已提交
936 937 938 939 940 941 942 943
static void io_get_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		return;
	get_task_struct(req->task);
	req->flags |= REQ_F_TASK_PINNED;
}

944 945
static inline void io_clean_op(struct io_kiocb *req)
{
946 947
	if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED |
			  REQ_F_INFLIGHT))
948 949 950
		__io_clean_op(req);
}

P
Pavel Begunkov 已提交
951 952 953 954 955 956 957
/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
static void __io_put_req_task(struct io_kiocb *req)
{
	if (req->flags & REQ_F_TASK_PINNED)
		put_task_struct(req->task);
}

958
static void io_sq_thread_drop_mm(void)
959 960 961 962 963 964 965 966 967 968 969 970
{
	struct mm_struct *mm = current->mm;

	if (mm) {
		kthread_unuse_mm(mm);
		mmput(mm);
	}
}

static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
{
	if (!current->mm) {
971 972
		if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) ||
			     !mmget_not_zero(ctx->sqo_mm)))
973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
			return -EFAULT;
		kthread_use_mm(ctx->sqo_mm);
	}

	return 0;
}

static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	if (!io_op_defs[req->opcode].needs_mm)
		return 0;
	return __io_sq_thread_acquire_mm(ctx);
}

static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}
993

994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
/*
 * Note: must call io_req_init_async() for the first time you
 * touch any members of io_wq_work.
 */
static inline void io_req_init_async(struct io_kiocb *req)
{
	if (req->flags & REQ_F_WORK_INITIALIZED)
		return;

	memset(&req->work, 0, sizeof(req->work));
	req->flags |= REQ_F_WORK_INITIALIZED;
}

1007 1008 1009 1010 1011
static inline bool io_async_submit(struct io_ring_ctx *ctx)
{
	return ctx->flags & IORING_SETUP_SQPOLL;
}

J
Jens Axboe 已提交
1012 1013 1014 1015
static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

1016
	complete(&ctx->ref_comp);
J
Jens Axboe 已提交
1017 1018
}

1019 1020 1021 1022 1023
static inline bool io_is_timeout_noseq(struct io_kiocb *req)
{
	return !req->timeout.off;
}

J
Jens Axboe 已提交
1024 1025 1026
static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;
1027
	int hash_bits;
J
Jens Axboe 已提交
1028 1029 1030 1031 1032

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

1033 1034 1035 1036
	ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
	if (!ctx->fallback_req)
		goto err;

1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
	/*
	 * Use 5 bits less than the max cq entries, that should give us around
	 * 32 entries per hash list if totally full and uniformly spread.
	 */
	hash_bits = ilog2(p->cq_entries);
	hash_bits -= 5;
	if (hash_bits <= 0)
		hash_bits = 1;
	ctx->cancel_hash_bits = hash_bits;
	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
					GFP_KERNEL);
	if (!ctx->cancel_hash)
		goto err;
	__hash_init(ctx->cancel_hash, 1U << hash_bits);

1052
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1053 1054
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
		goto err;
J
Jens Axboe 已提交
1055 1056

	ctx->flags = p->flags;
1057
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
1058
	init_waitqueue_head(&ctx->cq_wait);
1059
	INIT_LIST_HEAD(&ctx->cq_overflow_list);
1060 1061
	init_completion(&ctx->ref_comp);
	init_completion(&ctx->sq_thread_comp);
1062
	idr_init(&ctx->io_buffer_idr);
1063
	idr_init(&ctx->personality_idr);
J
Jens Axboe 已提交
1064 1065 1066
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
1067
	INIT_LIST_HEAD(&ctx->iopoll_list);
1068
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
1069
	INIT_LIST_HEAD(&ctx->timeout_list);
1070 1071 1072
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
1073 1074
	INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work);
	init_llist_head(&ctx->file_put_llist);
J
Jens Axboe 已提交
1075
	return ctx;
1076
err:
1077 1078
	if (ctx->fallback_req)
		kmem_cache_free(req_cachep, ctx->fallback_req);
1079
	kfree(ctx->cancel_hash);
1080 1081
	kfree(ctx);
	return NULL;
J
Jens Axboe 已提交
1082 1083
}

1084
static bool req_need_defer(struct io_kiocb *req, u32 seq)
1085
{
1086 1087
	if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
		struct io_ring_ctx *ctx = req->ctx;
1088

1089
		return seq != ctx->cached_cq_tail
1090
				+ atomic_read(&ctx->cached_cq_overflow);
1091
	}
1092

B
Bob Liu 已提交
1093
	return false;
1094 1095 1096
}

static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
1097
{
1098
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1099

1100 1101
	/* order cqe stores with ring update */
	smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
1102

1103 1104 1105
	if (wq_has_sleeper(&ctx->cq_wait)) {
		wake_up_interruptible(&ctx->cq_wait);
		kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
J
Jens Axboe 已提交
1106 1107 1108
	}
}

1109 1110 1111 1112 1113
/*
 * Returns true if we need to defer file table putting. This can only happen
 * from the error path with REQ_F_COMP_LOCKED set.
 */
static bool io_req_clean_work(struct io_kiocb *req)
1114
{
1115
	if (!(req->flags & REQ_F_WORK_INITIALIZED))
1116 1117 1118
		return false;

	req->flags &= ~REQ_F_WORK_INITIALIZED;
1119

1120 1121 1122 1123 1124 1125 1126 1127
	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
1128 1129 1130
	if (req->work.fs) {
		struct fs_struct *fs = req->work.fs;

1131 1132 1133
		if (req->flags & REQ_F_COMP_LOCKED)
			return true;

1134 1135 1136 1137 1138 1139
		spin_lock(&req->work.fs->lock);
		if (--fs->users)
			fs = NULL;
		spin_unlock(&req->work.fs->lock);
		if (fs)
			free_fs_struct(fs);
1140
		req->work.fs = NULL;
1141
	}
1142 1143

	return false;
1144 1145
}

1146
static void io_prep_async_work(struct io_kiocb *req)
1147
{
1148
	const struct io_op_def *def = &io_op_defs[req->opcode];
1149

1150 1151
	io_req_init_async(req);

1152
	if (req->flags & REQ_F_ISREG) {
1153
		if (def->hash_reg_file || (req->ctx->flags & IORING_SETUP_IOPOLL))
1154
			io_wq_hash_work(&req->work, file_inode(req->file));
1155 1156
	} else {
		if (def->unbound_nonreg_file)
1157
			req->work.flags |= IO_WQ_WORK_UNBOUND;
1158
	}
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	if (!req->work.mm && def->needs_mm) {
		mmgrab(current->mm);
		req->work.mm = current->mm;
	}
	if (!req->work.creds)
		req->work.creds = get_current_cred();
	if (!req->work.fs && def->needs_fs) {
		spin_lock(&current->fs->lock);
		if (!current->fs->in_exec) {
			req->work.fs = current->fs;
			req->work.fs->users++;
		} else {
			req->work.flags |= IO_WQ_WORK_CANCEL;
		}
		spin_unlock(&current->fs->lock);
	}
1175 1176 1177 1178
	if (def->needs_fsize)
		req->work.fsize = rlimit(RLIMIT_FSIZE);
	else
		req->work.fsize = RLIM_INFINITY;
1179
}
1180

1181
static void io_prep_async_link(struct io_kiocb *req)
1182
{
1183
	struct io_kiocb *cur;
1184

1185 1186 1187 1188
	io_prep_async_work(req);
	if (req->flags & REQ_F_LINK_HEAD)
		list_for_each_entry(cur, &req->link_list, link_list)
			io_prep_async_work(cur);
1189 1190
}

1191
static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req)
1192
{
1193
	struct io_ring_ctx *ctx = req->ctx;
1194
	struct io_kiocb *link = io_prep_linked_timeout(req);
1195

1196 1197 1198
	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
					&req->work, req->flags);
	io_wq_enqueue(ctx->io_wq, &req->work);
1199
	return link;
1200 1201
}

1202 1203
static void io_queue_async_work(struct io_kiocb *req)
{
1204 1205
	struct io_kiocb *link;

1206 1207
	/* init ->work of the whole link before punting */
	io_prep_async_link(req);
1208 1209 1210 1211
	link = __io_queue_async_work(req);

	if (link)
		io_queue_linked_timeout(link);
1212 1213
}

J
Jens Axboe 已提交
1214 1215 1216 1217
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

1218
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
J
Jens Axboe 已提交
1219
	if (ret != -1) {
1220 1221
		atomic_set(&req->ctx->cq_timeouts,
			atomic_read(&req->ctx->cq_timeouts) + 1);
P
Pavel Begunkov 已提交
1222
		list_del_init(&req->timeout.list);
1223
		req->flags |= REQ_F_COMP_LOCKED;
1224
		io_cqring_fill_event(req, 0);
1225
		io_put_req(req);
J
Jens Axboe 已提交
1226 1227 1228 1229 1230 1231 1232 1233
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
P
Pavel Begunkov 已提交
1234
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list)
J
Jens Axboe 已提交
1235 1236 1237 1238
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

1239
static void __io_queue_deferred(struct io_ring_ctx *ctx)
1240
{
1241
	do {
1242 1243
		struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
						struct io_defer_entry, list);
1244
		struct io_kiocb *link;
1245

1246
		if (req_need_defer(de->req, de->seq))
1247
			break;
1248
		list_del_init(&de->list);
1249
		/* punt-init is done before queueing for defer */
1250 1251 1252 1253 1254 1255 1256
		link = __io_queue_async_work(de->req);
		if (link) {
			__io_queue_linked_timeout(link);
			/* drop submission reference */
			link->flags |= REQ_F_COMP_LOCKED;
			io_put_req(link);
		}
1257
		kfree(de);
1258 1259 1260
	} while (!list_empty(&ctx->defer_list));
}

1261
static void io_flush_timeouts(struct io_ring_ctx *ctx)
1262
{
1263 1264
	while (!list_empty(&ctx->timeout_list)) {
		struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
P
Pavel Begunkov 已提交
1265
						struct io_kiocb, timeout.list);
1266

1267
		if (io_is_timeout_noseq(req))
1268
			break;
1269 1270
		if (req->timeout.target_seq != ctx->cached_cq_tail
					- atomic_read(&ctx->cq_timeouts))
1271
			break;
1272

P
Pavel Begunkov 已提交
1273
		list_del_init(&req->timeout.list);
J
Jens Axboe 已提交
1274
		io_kill_timeout(req);
1275 1276
	}
}
J
Jens Axboe 已提交
1277

1278 1279 1280
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	io_flush_timeouts(ctx);
1281 1282
	__io_commit_cqring(ctx);

1283 1284
	if (unlikely(!list_empty(&ctx->defer_list)))
		__io_queue_deferred(ctx);
1285 1286
}

J
Jens Axboe 已提交
1287 1288
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
1289
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1290 1291 1292
	unsigned tail;

	tail = ctx->cached_cq_tail;
1293 1294 1295 1296 1297
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
1298
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
1299 1300 1301
		return NULL;

	ctx->cached_cq_tail++;
1302
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
1303 1304
}

1305 1306
static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
{
1307 1308
	if (!ctx->cq_ev_fd)
		return false;
1309 1310
	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
		return false;
1311 1312
	if (!ctx->eventfd_async)
		return true;
1313
	return io_wq_current_is_worker();
1314 1315
}

1316
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1317 1318 1319 1320 1321
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
1322
	if (io_should_trigger_evfd(ctx))
1323 1324 1325
		eventfd_signal(ctx->cq_ev_fd, 1);
}

1326 1327 1328 1329 1330 1331 1332 1333 1334
static void io_cqring_mark_overflow(struct io_ring_ctx *ctx)
{
	if (list_empty(&ctx->cq_overflow_list)) {
		clear_bit(0, &ctx->sq_check_overflow);
		clear_bit(0, &ctx->cq_check_overflow);
		ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
	}
}

1335 1336
/* Returns true if there are no backlogged entries after the flush */
static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1337 1338 1339 1340 1341 1342 1343 1344 1345
{
	struct io_rings *rings = ctx->rings;
	struct io_uring_cqe *cqe;
	struct io_kiocb *req;
	unsigned long flags;
	LIST_HEAD(list);

	if (!force) {
		if (list_empty_careful(&ctx->cq_overflow_list))
1346
			return true;
1347 1348
		if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
		    rings->cq_ring_entries))
1349
			return false;
1350 1351 1352 1353 1354 1355
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/* if force is set, the ring is going away. always drop after that */
	if (force)
1356
		ctx->cq_overflow_flushed = 1;
1357

1358
	cqe = NULL;
1359 1360 1361 1362 1363 1364
	while (!list_empty(&ctx->cq_overflow_list)) {
		cqe = io_get_cqring(ctx);
		if (!cqe && !force)
			break;

		req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1365 1366
						compl.list);
		list_move(&req->compl.list, &list);
1367 1368 1369
		if (cqe) {
			WRITE_ONCE(cqe->user_data, req->user_data);
			WRITE_ONCE(cqe->res, req->result);
1370
			WRITE_ONCE(cqe->flags, req->compl.cflags);
1371 1372 1373 1374 1375 1376 1377
		} else {
			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
		}
	}

	io_commit_cqring(ctx);
1378 1379
	io_cqring_mark_overflow(ctx);

1380 1381 1382 1383
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	while (!list_empty(&list)) {
1384 1385
		req = list_first_entry(&list, struct io_kiocb, compl.list);
		list_del(&req->compl.list);
1386
		io_put_req(req);
1387
	}
1388 1389

	return cqe != NULL;
1390 1391
}

1392
static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1393
{
1394
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1395 1396
	struct io_uring_cqe *cqe;

1397
	trace_io_uring_complete(ctx, req->user_data, res);
J
Jens Axboe 已提交
1398

J
Jens Axboe 已提交
1399 1400 1401 1402 1403 1404
	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
1405
	if (likely(cqe)) {
1406
		WRITE_ONCE(cqe->user_data, req->user_data);
J
Jens Axboe 已提交
1407
		WRITE_ONCE(cqe->res, res);
1408
		WRITE_ONCE(cqe->flags, cflags);
1409
	} else if (ctx->cq_overflow_flushed) {
1410 1411
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
1412
	} else {
1413 1414 1415
		if (list_empty(&ctx->cq_overflow_list)) {
			set_bit(0, &ctx->sq_check_overflow);
			set_bit(0, &ctx->cq_check_overflow);
1416
			ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
1417
		}
1418
		io_clean_op(req);
1419
		req->result = res;
1420
		req->compl.cflags = cflags;
1421 1422
		refcount_inc(&req->refs);
		list_add_tail(&req->compl.list, &ctx->cq_overflow_list);
J
Jens Axboe 已提交
1423 1424 1425
	}
}

1426 1427 1428 1429 1430
static void io_cqring_fill_event(struct io_kiocb *req, long res)
{
	__io_cqring_fill_event(req, res, 0);
}

1431
static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
J
Jens Axboe 已提交
1432
{
1433
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1434 1435 1436
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
1437
	__io_cqring_fill_event(req, res, cflags);
J
Jens Axboe 已提交
1438 1439 1440
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
1441
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1442 1443
}

1444
static void io_submit_flush_completions(struct io_comp_state *cs)
1445
{
1446 1447 1448 1449 1450 1451
	struct io_ring_ctx *ctx = cs->ctx;

	spin_lock_irq(&ctx->completion_lock);
	while (!list_empty(&cs->list)) {
		struct io_kiocb *req;

1452 1453
		req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
		list_del(&req->compl.list);
1454
		__io_cqring_fill_event(req, req->result, req->compl.cflags);
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
		if (!(req->flags & REQ_F_LINK_HEAD)) {
			req->flags |= REQ_F_COMP_LOCKED;
			io_put_req(req);
		} else {
			spin_unlock_irq(&ctx->completion_lock);
			io_put_req(req);
			spin_lock_irq(&ctx->completion_lock);
		}
	}
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	cs->nr = 0;
}

static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
			      struct io_comp_state *cs)
{
	if (!cs) {
		io_cqring_add_event(req, res, cflags);
		io_put_req(req);
	} else {
1478
		io_clean_op(req);
1479
		req->result = res;
1480
		req->compl.cflags = cflags;
1481
		list_add_tail(&req->compl.list, &cs->list);
1482 1483 1484
		if (++cs->nr >= 32)
			io_submit_flush_completions(cs);
	}
1485 1486 1487
}

static void io_req_complete(struct io_kiocb *req, long res)
1488
{
1489
	__io_req_complete(req, res, 0, NULL);
1490 1491
}

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
static inline bool io_is_fallback_req(struct io_kiocb *req)
{
	return req == (struct io_kiocb *)
			((unsigned long) req->ctx->fallback_req & ~1UL);
}

static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	req = ctx->fallback_req;
1503
	if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req))
1504 1505 1506 1507 1508
		return req;

	return NULL;
}

1509 1510
static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
				     struct io_submit_state *state)
J
Jens Axboe 已提交
1511
{
1512
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
1513 1514
	struct io_kiocb *req;

1515
	if (!state->free_reqs) {
J
Jens Axboe 已提交
1516 1517 1518 1519
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
1520 1521 1522 1523 1524 1525 1526 1527 1528
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
1529
				goto fallback;
1530 1531
			ret = 1;
		}
J
Jens Axboe 已提交
1532
		state->free_reqs = ret - 1;
1533
		req = state->reqs[ret - 1];
J
Jens Axboe 已提交
1534 1535
	} else {
		state->free_reqs--;
1536
		req = state->reqs[state->free_reqs];
J
Jens Axboe 已提交
1537 1538
	}

J
Jens Axboe 已提交
1539
	return req;
1540
fallback:
1541
	return io_get_fallback_req(ctx);
J
Jens Axboe 已提交
1542 1543
}

1544 1545 1546 1547
static inline void io_put_file(struct io_kiocb *req, struct file *file,
			  bool fixed)
{
	if (fixed)
1548
		percpu_ref_put(req->fixed_file_refs);
1549 1550 1551 1552
	else
		fput(file);
}

1553
static bool io_dismantle_req(struct io_kiocb *req)
J
Jens Axboe 已提交
1554
{
1555
	io_clean_op(req);
1556

1557 1558
	if (req->io)
		kfree(req->io);
1559 1560
	if (req->file)
		io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1561

1562
	return io_req_clean_work(req);
1563 1564
}

1565
static void __io_free_req_finish(struct io_kiocb *req)
1566
{
1567
	struct io_ring_ctx *ctx = req->ctx;
1568

1569
	__io_put_req_task(req);
1570 1571 1572
	if (likely(!io_is_fallback_req(req)))
		kmem_cache_free(req_cachep, req);
	else
1573 1574
		clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req);
	percpu_ref_put(&ctx->refs);
1575 1576
}

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
static void io_req_task_file_table_put(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct fs_struct *fs = req->work.fs;

	spin_lock(&req->work.fs->lock);
	if (--fs->users)
		fs = NULL;
	spin_unlock(&req->work.fs->lock);
	if (fs)
		free_fs_struct(fs);
	req->work.fs = NULL;
	__io_free_req_finish(req);
}

static void __io_free_req(struct io_kiocb *req)
{
	if (!io_dismantle_req(req)) {
		__io_free_req_finish(req);
	} else {
		int ret;

		init_task_work(&req->task_work, io_req_task_file_table_put);
		ret = task_work_add(req->task, &req->task_work, TWA_RESUME);
		if (unlikely(ret)) {
			struct task_struct *tsk;

			tsk = io_wq_get_task(req->ctx->io_wq);
			task_work_add(tsk, &req->task_work, 0);
		}
	}
}

1610
static bool io_link_cancel_timeout(struct io_kiocb *req)
1611
{
1612
	struct io_ring_ctx *ctx = req->ctx;
1613 1614
	int ret;

1615
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1616
	if (ret != -1) {
1617
		io_cqring_fill_event(req, -ECANCELED);
1618
		io_commit_cqring(ctx);
1619
		req->flags &= ~REQ_F_LINK_HEAD;
1620
		io_put_req(req);
1621 1622 1623 1624
		return true;
	}

	return false;
1625 1626
}

1627
static bool __io_kill_linked_timeout(struct io_kiocb *req)
J
Jens Axboe 已提交
1628
{
1629
	struct io_kiocb *link;
1630
	bool wake_ev;
1631 1632

	if (list_empty(&req->link_list))
1633
		return false;
1634 1635
	link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	if (link->opcode != IORING_OP_LINK_TIMEOUT)
1636
		return false;
1637 1638

	list_del_init(&link->link_list);
1639
	link->flags |= REQ_F_COMP_LOCKED;
1640 1641
	wake_ev = io_link_cancel_timeout(link);
	req->flags &= ~REQ_F_LINK_TIMEOUT;
1642 1643 1644 1645
	return wake_ev;
}

static void io_kill_linked_timeout(struct io_kiocb *req)
J
Jens Axboe 已提交
1646
{
1647
	struct io_ring_ctx *ctx = req->ctx;
1648
	bool wake_ev;
J
Jens Axboe 已提交
1649

1650 1651 1652 1653 1654
	if (!(req->flags & REQ_F_COMP_LOCKED)) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->completion_lock, flags);
		wake_ev = __io_kill_linked_timeout(req);
1655
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
1656 1657 1658 1659
	} else {
		wake_ev = __io_kill_linked_timeout(req);
	}

1660 1661 1662 1663
	if (wake_ev)
		io_cqring_ev_posted(ctx);
}

1664
static struct io_kiocb *io_req_link_next(struct io_kiocb *req)
1665 1666
{
	struct io_kiocb *nxt;
1667

J
Jens Axboe 已提交
1668 1669 1670 1671 1672
	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
1673
	if (unlikely(list_empty(&req->link_list)))
1674
		return NULL;
1675

1676 1677 1678 1679
	nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	list_del_init(&req->link_list);
	if (!list_empty(&nxt->link_list))
		nxt->flags |= REQ_F_LINK_HEAD;
1680
	return nxt;
J
Jens Axboe 已提交
1681 1682 1683
}

/*
1684
 * Called if REQ_F_LINK_HEAD is set, and we fail the head request
J
Jens Axboe 已提交
1685
 */
1686
static void __io_fail_links(struct io_kiocb *req)
J
Jens Axboe 已提交
1687
{
1688
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1689 1690

	while (!list_empty(&req->link_list)) {
1691 1692
		struct io_kiocb *link = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
J
Jens Axboe 已提交
1693

1694
		list_del_init(&link->link_list);
1695
		trace_io_uring_fail_link(req, link);
1696

1697
		io_cqring_fill_event(link, -ECANCELED);
1698
		link->flags |= REQ_F_COMP_LOCKED;
1699
		__io_double_put_req(link);
1700
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
1701
	}
1702 1703 1704

	io_commit_cqring(ctx);
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1705 1706
}

1707
static void io_fail_links(struct io_kiocb *req)
J
Jens Axboe 已提交
1708
{
1709
	struct io_ring_ctx *ctx = req->ctx;
1710

1711
	if (!(req->flags & REQ_F_COMP_LOCKED)) {
1712 1713 1714
		unsigned long flags;

		spin_lock_irqsave(&ctx->completion_lock, flags);
1715
		__io_fail_links(req);
1716 1717
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
1718
		__io_fail_links(req);
J
Jens Axboe 已提交
1719 1720
	}

1721
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1722 1723
}

1724
static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
1725
{
P
Pavel Begunkov 已提交
1726
	req->flags &= ~REQ_F_LINK_HEAD;
1727 1728
	if (req->flags & REQ_F_LINK_TIMEOUT)
		io_kill_linked_timeout(req);
1729

J
Jens Axboe 已提交
1730 1731 1732 1733 1734 1735
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
1736 1737 1738 1739
	if (likely(!(req->flags & REQ_F_FAIL_LINK)))
		return io_req_link_next(req);
	io_fail_links(req);
	return NULL;
1740
}
J
Jens Axboe 已提交
1741

1742 1743 1744 1745 1746 1747 1748
static struct io_kiocb *io_req_find_next(struct io_kiocb *req)
{
	if (likely(!(req->flags & REQ_F_LINK_HEAD)))
		return NULL;
	return __io_req_find_next(req);
}

1749 1750
static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb,
				bool twa_signal_ok)
1751 1752 1753
{
	struct task_struct *tsk = req->task;
	struct io_ring_ctx *ctx = req->ctx;
1754
	int ret, notify;
1755

1756 1757 1758
	if (tsk->flags & PF_EXITING)
		return -ESRCH;

1759
	/*
1760 1761 1762 1763
	 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
	 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
	 * processing task_work. There's no reliable way to tell if TWA_RESUME
	 * will do the job.
1764
	 */
1765
	notify = 0;
1766
	if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok)
1767 1768 1769 1770 1771
		notify = TWA_SIGNAL;

	ret = task_work_add(tsk, cb, notify);
	if (!ret)
		wake_up_process(tsk);
1772

1773 1774 1775
	return ret;
}

1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
static void __io_req_task_cancel(struct io_kiocb *req, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	io_cqring_fill_event(req, error);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	req_set_fail_links(req);
	io_double_put_req(req);
}

static void io_req_task_cancel(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
1793
	struct io_ring_ctx *ctx = req->ctx;
1794 1795

	__io_req_task_cancel(req, -ECANCELED);
1796
	percpu_ref_put(&ctx->refs);
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814
}

static void __io_req_task_submit(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!__io_sq_thread_acquire_mm(ctx)) {
		mutex_lock(&ctx->uring_lock);
		__io_queue_sqe(req, NULL, NULL);
		mutex_unlock(&ctx->uring_lock);
	} else {
		__io_req_task_cancel(req, -EFAULT);
	}
}

static void io_req_task_submit(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
1815
	struct io_ring_ctx *ctx = req->ctx;
1816 1817

	__io_req_task_submit(req);
1818
	percpu_ref_put(&ctx->refs);
1819 1820 1821 1822 1823 1824 1825
}

static void io_req_task_queue(struct io_kiocb *req)
{
	int ret;

	init_task_work(&req->task_work, io_req_task_submit);
1826
	percpu_ref_get(&req->ctx->refs);
1827

1828
	ret = io_req_task_work_add(req, &req->task_work, true);
1829
	if (unlikely(ret)) {
1830 1831
		struct task_struct *tsk;

1832 1833
		init_task_work(&req->task_work, io_req_task_cancel);
		tsk = io_wq_get_task(req->ctx->io_wq);
1834 1835
		task_work_add(tsk, &req->task_work, 0);
		wake_up_process(tsk);
1836 1837 1838
	}
}

1839
static void io_queue_next(struct io_kiocb *req)
1840
{
1841
	struct io_kiocb *nxt = io_req_find_next(req);
1842 1843

	if (nxt)
1844
		io_req_task_queue(nxt);
1845 1846
}

1847
static void io_free_req(struct io_kiocb *req)
1848
{
1849 1850 1851
	io_queue_next(req);
	__io_free_req(req);
}
1852

1853 1854 1855
struct req_batch {
	void *reqs[IO_IOPOLL_BATCH];
	int to_free;
1856

1857 1858
	struct task_struct	*task;
	int			task_refs;
1859 1860
};

1861 1862 1863 1864 1865 1866 1867
static inline void io_init_req_batch(struct req_batch *rb)
{
	rb->to_free = 0;
	rb->task_refs = 0;
	rb->task = NULL;
}

1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880
static void __io_req_free_batch_flush(struct io_ring_ctx *ctx,
				      struct req_batch *rb)
{
	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
	percpu_ref_put_many(&ctx->refs, rb->to_free);
	rb->to_free = 0;
}

static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
				     struct req_batch *rb)
{
	if (rb->to_free)
		__io_req_free_batch_flush(ctx, rb);
1881 1882 1883 1884
	if (rb->task) {
		put_task_struct_many(rb->task, rb->task_refs);
		rb->task = NULL;
	}
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
}

static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req)
{
	if (unlikely(io_is_fallback_req(req))) {
		io_free_req(req);
		return;
	}
	if (req->flags & REQ_F_LINK_HEAD)
		io_queue_next(req);

1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
	if (req->flags & REQ_F_TASK_PINNED) {
		if (req->task != rb->task) {
			if (rb->task)
				put_task_struct_many(rb->task, rb->task_refs);
			rb->task = req->task;
			rb->task_refs = 0;
		}
		rb->task_refs++;
		req->flags &= ~REQ_F_TASK_PINNED;
	}

1907
	WARN_ON_ONCE(io_dismantle_req(req));
1908 1909 1910
	rb->reqs[rb->to_free++] = req;
	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		__io_req_free_batch_flush(req->ctx, rb);
1911 1912
}

1913 1914 1915 1916
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
1917
static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
1918
{
1919 1920
	struct io_kiocb *nxt = NULL;

1921
	if (refcount_dec_and_test(&req->refs)) {
1922
		nxt = io_req_find_next(req);
1923
		__io_free_req(req);
1924
	}
1925
	return nxt;
J
Jens Axboe 已提交
1926 1927
}

1928 1929 1930 1931
static void io_put_req(struct io_kiocb *req)
{
	if (refcount_dec_and_test(&req->refs))
		io_free_req(req);
J
Jens Axboe 已提交
1932 1933
}

1934
static struct io_wq_work *io_steal_work(struct io_kiocb *req)
1935
{
1936
	struct io_kiocb *nxt;
1937

1938
	/*
1939 1940 1941
	 * A ref is owned by io-wq in which context we're. So, if that's the
	 * last one, it's safe to steal next work. False negatives are Ok,
	 * it just will be re-punted async in io_put_work()
1942
	 */
1943 1944
	if (refcount_read(&req->refs) != 1)
		return NULL;
1945

1946
	nxt = io_req_find_next(req);
1947
	return nxt ? &nxt->work : NULL;
1948 1949
}

1950 1951 1952 1953 1954
/*
 * Must only be used if we don't need to care about links, usually from
 * within the completion handling itself.
 */
static void __io_double_put_req(struct io_kiocb *req)
1955 1956 1957 1958 1959 1960
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		__io_free_req(req);
}

1961 1962 1963 1964 1965 1966 1967
static void io_double_put_req(struct io_kiocb *req)
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		io_free_req(req);
}

1968
static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
1969
{
1970 1971
	struct io_rings *rings = ctx->rings;

1972 1973 1974 1975 1976 1977 1978 1979
	if (test_bit(0, &ctx->cq_check_overflow)) {
		/*
		 * noflush == true is from the waitqueue handler, just ensure
		 * we wake up the task, and the next invocation will flush the
		 * entries. We cannot safely to it from here.
		 */
		if (noflush && !list_empty(&ctx->cq_overflow_list))
			return -1U;
1980

1981 1982
		io_cqring_overflow_flush(ctx, false);
	}
1983

1984 1985
	/* See comment at the top of this file */
	smp_rmb();
1986
	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
1987 1988
}

1989 1990 1991 1992 1993 1994 1995 1996
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

1997
static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
1998
{
1999
	unsigned int cflags;
2000

2001 2002
	cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	cflags |= IORING_CQE_F_BUFFER;
2003
	req->flags &= ~REQ_F_BUFFER_SELECTED;
2004 2005
	kfree(kbuf);
	return cflags;
2006 2007
}

2008
static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2009
{
2010
	struct io_buffer *kbuf;
2011

2012
	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2013 2014 2015
	return io_put_kbuf(req, kbuf);
}

2016 2017
static inline bool io_run_task_work(void)
{
2018 2019 2020 2021 2022 2023
	/*
	 * Not safe to run on exiting task, and the task_work handling will
	 * not add work to such a task.
	 */
	if (unlikely(current->flags & PF_EXITING))
		return false;
2024 2025 2026 2027 2028 2029 2030
	if (current->task_works) {
		__set_current_state(TASK_RUNNING);
		task_work_run();
		return true;
	}

	return false;
2031 2032
}

2033 2034 2035 2036 2037
static void io_iopoll_queue(struct list_head *again)
{
	struct io_kiocb *req;

	do {
2038 2039
		req = list_first_entry(again, struct io_kiocb, inflight_entry);
		list_del(&req->inflight_entry);
2040
		__io_complete_rw(req, -EAGAIN, 0, NULL);
2041 2042 2043
	} while (!list_empty(again));
}

J
Jens Axboe 已提交
2044 2045 2046 2047 2048 2049
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
2050
	struct req_batch rb;
J
Jens Axboe 已提交
2051
	struct io_kiocb *req;
2052 2053 2054 2055
	LIST_HEAD(again);

	/* order with ->result store in io_complete_rw_iopoll() */
	smp_rmb();
J
Jens Axboe 已提交
2056

2057
	io_init_req_batch(&rb);
J
Jens Axboe 已提交
2058
	while (!list_empty(done)) {
2059 2060
		int cflags = 0;

2061
		req = list_first_entry(done, struct io_kiocb, inflight_entry);
2062
		if (READ_ONCE(req->result) == -EAGAIN) {
2063
			req->result = 0;
2064
			req->iopoll_completed = 0;
2065
			list_move_tail(&req->inflight_entry, &again);
2066 2067
			continue;
		}
2068
		list_del(&req->inflight_entry);
J
Jens Axboe 已提交
2069

2070
		if (req->flags & REQ_F_BUFFER_SELECTED)
2071
			cflags = io_put_rw_kbuf(req);
2072 2073

		__io_cqring_fill_event(req, req->result, cflags);
J
Jens Axboe 已提交
2074 2075
		(*nr_events)++;

2076
		if (refcount_dec_and_test(&req->refs))
2077
			io_req_free_batch(&rb, req);
J
Jens Axboe 已提交
2078 2079
	}

J
Jens Axboe 已提交
2080
	io_commit_cqring(ctx);
2081 2082
	if (ctx->flags & IORING_SETUP_SQPOLL)
		io_cqring_ev_posted(ctx);
2083
	io_req_free_batch_finish(ctx, &rb);
2084

2085 2086
	if (!list_empty(&again))
		io_iopoll_queue(&again);
2087 2088
}

J
Jens Axboe 已提交
2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
2104
	list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
2105
		struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2106 2107

		/*
2108 2109 2110
		 * Move completed and retryable entries to our local lists.
		 * If we find a request that requires polling, break out
		 * and complete those lists first, if we have entries there.
J
Jens Axboe 已提交
2111
		 */
2112
		if (READ_ONCE(req->iopoll_completed)) {
2113
			list_move_tail(&req->inflight_entry, &done);
J
Jens Axboe 已提交
2114 2115 2116 2117 2118 2119 2120 2121 2122
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

2123 2124
		/* iopoll may have completed current req */
		if (READ_ONCE(req->iopoll_completed))
2125
			list_move_tail(&req->inflight_entry, &done);
2126

J
Jens Axboe 已提交
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
2139
 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
J
Jens Axboe 已提交
2140 2141 2142 2143 2144 2145
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
2146
	while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
J
Jens Axboe 已提交
2147 2148 2149 2150 2151
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
2152
		if (*nr_events >= min)
J
Jens Axboe 已提交
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
2163
static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
2164 2165 2166 2167 2168
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
2169
	while (!list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
2170 2171
		unsigned int nr_events = 0;

2172
		io_do_iopoll(ctx, &nr_events, 0);
2173

2174 2175 2176
		/* let it sleep and repeat later if can't complete a request */
		if (nr_events == 0)
			break;
2177 2178 2179
		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
2180
		 * Also let task_work, etc. to progress by releasing the mutex
2181
		 */
2182 2183 2184 2185 2186
		if (need_resched()) {
			mutex_unlock(&ctx->uring_lock);
			cond_resched();
			mutex_lock(&ctx->uring_lock);
		}
J
Jens Axboe 已提交
2187 2188 2189 2190
	}
	mutex_unlock(&ctx->uring_lock);
}

2191
static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
J
Jens Axboe 已提交
2192
{
2193
	unsigned int nr_events = 0;
2194
	int iters = 0, ret = 0;
2195

2196 2197 2198 2199 2200 2201
	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
J
Jens Axboe 已提交
2202
	do {
2203 2204 2205 2206 2207
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
2208
		if (io_cqring_events(ctx, false))
2209 2210
			break;

2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
2223
			io_run_task_work();
2224 2225 2226
			mutex_lock(&ctx->uring_lock);
		}

2227
		ret = io_iopoll_getevents(ctx, &nr_events, min);
J
Jens Axboe 已提交
2228 2229 2230
		if (ret <= 0)
			break;
		ret = 0;
2231
	} while (min && !nr_events && !need_resched());
J
Jens Axboe 已提交
2232

2233
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
2234 2235 2236
	return ret;
}

2237
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
2238
{
2239 2240 2241 2242 2243 2244
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
2245

2246
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
2247
	}
2248
	file_end_write(req->file);
J
Jens Axboe 已提交
2249 2250
}

2251 2252
static void io_complete_rw_common(struct kiocb *kiocb, long res,
				  struct io_comp_state *cs)
J
Jens Axboe 已提交
2253
{
2254
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2255
	int cflags = 0;
J
Jens Axboe 已提交
2256

2257 2258
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
2259

J
Jens Axboe 已提交
2260 2261
	if (res != req->result)
		req_set_fail_links(req);
2262
	if (req->flags & REQ_F_BUFFER_SELECTED)
2263
		cflags = io_put_rw_kbuf(req);
2264
	__io_req_complete(req, res, cflags, cs);
2265 2266
}

2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296
#ifdef CONFIG_BLOCK
static bool io_resubmit_prep(struct io_kiocb *req, int error)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	ssize_t ret = -ECANCELED;
	struct iov_iter iter;
	int rw;

	if (error) {
		ret = error;
		goto end_req;
	}

	switch (req->opcode) {
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
		rw = READ;
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		rw = WRITE;
		break;
	default:
		printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
				req->opcode);
		goto end_req;
	}

2297 2298 2299 2300 2301 2302 2303 2304 2305
	if (!req->io) {
		ret = io_import_iovec(rw, req, &iovec, &iter, false);
		if (ret < 0)
			goto end_req;
		ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false);
		if (!ret)
			return true;
		kfree(iovec);
	} else {
2306
		return true;
2307
	}
2308 2309
end_req:
	req_set_fail_links(req);
2310
	io_req_complete(req, ret);
2311 2312 2313 2314 2315 2316 2317
	return false;
}
#endif

static bool io_rw_reissue(struct io_kiocb *req, long res)
{
#ifdef CONFIG_BLOCK
2318
	umode_t mode = file_inode(req->file)->i_mode;
2319 2320
	int ret;

2321 2322
	if (!S_ISBLK(mode) && !S_ISREG(mode))
		return false;
2323 2324 2325
	if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker())
		return false;

2326
	ret = io_sq_thread_acquire_mm(req->ctx, req);
2327

2328 2329 2330
	if (io_resubmit_prep(req, ret)) {
		refcount_inc(&req->refs);
		io_queue_async_work(req);
2331
		return true;
2332 2333
	}

2334 2335 2336 2337
#endif
	return false;
}

2338 2339 2340 2341 2342
static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
			     struct io_comp_state *cs)
{
	if (!io_rw_reissue(req, res))
		io_complete_rw_common(&req->rw.kiocb, res, cs);
2343 2344 2345 2346
}

static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
2347
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2348

2349
	__io_complete_rw(req, res, res2, NULL);
J
Jens Axboe 已提交
2350 2351
}

J
Jens Axboe 已提交
2352 2353
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
2354
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
2355

2356 2357
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
2358

2359
	if (res != -EAGAIN && res != req->result)
J
Jens Axboe 已提交
2360
		req_set_fail_links(req);
2361 2362 2363

	WRITE_ONCE(req->result, res);
	/* order with io_poll_complete() checking ->result */
2364 2365
	smp_wmb();
	WRITE_ONCE(req->iopoll_completed, 1);
J
Jens Axboe 已提交
2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
2383
	if (list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
2384 2385 2386 2387
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

2388
		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
2389
						inflight_entry);
2390
		if (list_req->file != req->file)
J
Jens Axboe 已提交
2391 2392 2393 2394 2395 2396 2397
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
2398
	if (READ_ONCE(req->iopoll_completed))
2399
		list_add(&req->inflight_entry, &ctx->iopoll_list);
J
Jens Axboe 已提交
2400
	else
2401
		list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
2402 2403 2404 2405

	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
	    wq_has_sleeper(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
J
Jens Axboe 已提交
2406 2407
}

P
Pavel Begunkov 已提交
2408
static void __io_state_file_put(struct io_submit_state *state)
2409
{
2410 2411
	if (state->has_refs)
		fput_many(state->file, state->has_refs);
P
Pavel Begunkov 已提交
2412 2413 2414 2415 2416 2417 2418
	state->file = NULL;
}

static inline void io_state_file_put(struct io_submit_state *state)
{
	if (state->file)
		__io_state_file_put(state);
2419 2420 2421 2422 2423 2424 2425
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
2426
static struct file *__io_file_get(struct io_submit_state *state, int fd)
2427 2428 2429 2430 2431 2432
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
2433
			state->has_refs--;
2434 2435 2436
			state->ios_left--;
			return state->file;
		}
P
Pavel Begunkov 已提交
2437
		__io_state_file_put(state);
2438 2439 2440 2441 2442 2443 2444
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->ios_left--;
2445
	state->has_refs = state->ios_left;
2446 2447 2448
	return state->file;
}

2449 2450 2451 2452 2453 2454 2455 2456 2457
static bool io_bdev_nowait(struct block_device *bdev)
{
#ifdef CONFIG_BLOCK
	return !bdev || queue_is_mq(bdev_get_queue(bdev));
#else
	return true;
#endif
}

J
Jens Axboe 已提交
2458 2459 2460 2461 2462
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
2463
static bool io_file_supports_async(struct file *file, int rw)
J
Jens Axboe 已提交
2464 2465 2466
{
	umode_t mode = file_inode(file)->i_mode;

2467 2468 2469 2470 2471 2472
	if (S_ISBLK(mode)) {
		if (io_bdev_nowait(file->f_inode->i_bdev))
			return true;
		return false;
	}
	if (S_ISCHR(mode) || S_ISSOCK(mode))
J
Jens Axboe 已提交
2473
		return true;
2474 2475 2476 2477 2478 2479
	if (S_ISREG(mode)) {
		if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
		    file->f_op != &io_uring_fops)
			return true;
		return false;
	}
J
Jens Axboe 已提交
2480

2481 2482 2483 2484
	/* any ->read/write should understand O_NONBLOCK */
	if (file->f_flags & O_NONBLOCK)
		return true;

2485 2486 2487 2488 2489 2490 2491
	if (!(file->f_mode & FMODE_NOWAIT))
		return false;

	if (rw == READ)
		return file->f_op->read_iter != NULL;

	return file->f_op->write_iter != NULL;
J
Jens Axboe 已提交
2492 2493
}

2494 2495
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		      bool force_nonblock)
J
Jens Axboe 已提交
2496
{
J
Jens Axboe 已提交
2497
	struct io_ring_ctx *ctx = req->ctx;
2498
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2499 2500
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
2501

2502 2503 2504
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

J
Jens Axboe 已提交
2505
	kiocb->ki_pos = READ_ONCE(sqe->off);
2506 2507 2508 2509
	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
		req->flags |= REQ_F_CUR_POS;
		kiocb->ki_pos = req->file->f_pos;
	}
J
Jens Axboe 已提交
2510
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
2511 2512 2513 2514
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
		return ret;
J
Jens Axboe 已提交
2515 2516 2517 2518 2519

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
2520
			return ret;
J
Jens Axboe 已提交
2521 2522 2523 2524 2525

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

2526
	/* don't allow async punt if RWF_NOWAIT was requested */
2527
	if (kiocb->ki_flags & IOCB_NOWAIT)
2528 2529
		req->flags |= REQ_F_NOWAIT;

2530 2531 2532
	if (kiocb->ki_flags & IOCB_DIRECT)
		io_get_req_task(req);

2533
	if (force_nonblock)
J
Jens Axboe 已提交
2534
		kiocb->ki_flags |= IOCB_NOWAIT;
2535

J
Jens Axboe 已提交
2536 2537 2538
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
2539
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
2540

J
Jens Axboe 已提交
2541 2542
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
2543
		req->iopoll_completed = 0;
2544
		io_get_req_task(req);
J
Jens Axboe 已提交
2545
	} else {
J
Jens Axboe 已提交
2546 2547
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
2548 2549
		kiocb->ki_complete = io_complete_rw;
	}
2550

2551 2552
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
2553
	req->buf_index = READ_ONCE(sqe->buf_index);
J
Jens Axboe 已提交
2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

2578 2579
static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
		       struct io_comp_state *cs)
2580
{
2581 2582
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

2583 2584 2585 2586 2587 2588 2589 2590
	/* add previously done IO, if any */
	if (req->io && req->io->rw.bytes_done > 0) {
		if (ret < 0)
			ret = req->io->rw.bytes_done;
		else
			ret += req->io->rw.bytes_done;
	}

2591 2592
	if (req->flags & REQ_F_CUR_POS)
		req->file->f_pos = kiocb->ki_pos;
P
Pavel Begunkov 已提交
2593
	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
2594
		__io_complete_rw(req, ret, 0, cs);
2595 2596 2597 2598
	else
		io_rw_done(kiocb, ret);
}

2599
static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
2600
			       struct iov_iter *iter)
2601
{
2602 2603
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
2604
	struct io_mapped_ubuf *imu;
2605
	u16 index, buf_index;
2606 2607 2608 2609 2610 2611 2612
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

2613
	buf_index = req->buf_index;
2614 2615 2616 2617 2618
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
2619
	buf_addr = req->rw.addr;
2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
2665
			iter->count -= bvec->bv_len + offset;
2666 2667 2668 2669
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

2670
	return len;
2671 2672
}

2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
{
	if (needs_lock)
		mutex_unlock(&ctx->uring_lock);
}

static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
{
	/*
	 * "Normal" inline submissions always hold the uring_lock, since we
	 * grab it from the system call. Same is true for the SQPOLL offload.
	 * The only exception is when we've detached the request and issue it
	 * from an async worker thread, grab the lock for that case.
	 */
	if (needs_lock)
		mutex_lock(&ctx->uring_lock);
}

static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
					  int bgid, struct io_buffer *kbuf,
					  bool needs_lock)
{
	struct io_buffer *head;

	if (req->flags & REQ_F_BUFFER_SELECTED)
		return kbuf;

	io_ring_submit_lock(req->ctx, needs_lock);

	lockdep_assert_held(&req->ctx->uring_lock);

	head = idr_find(&req->ctx->io_buffer_idr, bgid);
	if (head) {
		if (!list_empty(&head->list)) {
			kbuf = list_last_entry(&head->list, struct io_buffer,
							list);
			list_del(&kbuf->list);
		} else {
			kbuf = head;
			idr_remove(&req->ctx->io_buffer_idr, bgid);
		}
		if (*len > kbuf->len)
			*len = kbuf->len;
	} else {
		kbuf = ERR_PTR(-ENOBUFS);
	}

	io_ring_submit_unlock(req->ctx, needs_lock);

	return kbuf;
}

2725 2726 2727 2728
static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
					bool needs_lock)
{
	struct io_buffer *kbuf;
2729
	u16 bgid;
2730 2731

	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2732
	bgid = req->buf_index;
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791
	kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;
	req->rw.addr = (u64) (unsigned long) kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;
	return u64_to_user_ptr(kbuf->addr);
}

#ifdef CONFIG_COMPAT
static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
				bool needs_lock)
{
	struct compat_iovec __user *uiov;
	compat_ssize_t clen;
	void __user *buf;
	ssize_t len;

	uiov = u64_to_user_ptr(req->rw.addr);
	if (!access_ok(uiov, sizeof(*uiov)))
		return -EFAULT;
	if (__get_user(clen, &uiov->iov_len))
		return -EFAULT;
	if (clen < 0)
		return -EINVAL;

	len = clen;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = (compat_size_t) len;
	return 0;
}
#endif

static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				      bool needs_lock)
{
	struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
	void __user *buf;
	ssize_t len;

	if (copy_from_user(iov, uiov, sizeof(*uiov)))
		return -EFAULT;

	len = iov[0].iov_len;
	if (len < 0)
		return -EINVAL;
	buf = io_rw_buffer_select(req, &len, needs_lock);
	if (IS_ERR(buf))
		return PTR_ERR(buf);
	iov[0].iov_base = buf;
	iov[0].iov_len = len;
	return 0;
}

static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
				    bool needs_lock)
{
2792 2793 2794 2795 2796 2797
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		struct io_buffer *kbuf;

		kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
		iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		iov[0].iov_len = kbuf->len;
2798
		return 0;
2799
	}
2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812
	if (!req->rw.len)
		return 0;
	else if (req->rw.len > 1)
		return -EINVAL;

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		return io_compat_import(req, iov, needs_lock);
#endif

	return __io_iov_buffer_select(req, iov, needs_lock);
}

2813 2814 2815
static ssize_t __io_import_iovec(int rw, struct io_kiocb *req,
				 struct iovec **iovec, struct iov_iter *iter,
				 bool needs_lock)
J
Jens Axboe 已提交
2816
{
2817 2818
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
2819
	ssize_t ret;
2820 2821
	u8 opcode;

2822
	opcode = req->opcode;
2823
	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2824
		*iovec = NULL;
2825
		return io_import_fixed(req, rw, iter);
2826
	}
J
Jens Axboe 已提交
2827

2828
	/* buffer index only valid with fixed read/write, or buffer select  */
2829
	if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
2830 2831
		return -EINVAL;

2832
	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2833
		if (req->flags & REQ_F_BUFFER_SELECT) {
2834
			buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
2835
			if (IS_ERR(buf))
2836
				return PTR_ERR(buf);
2837
			req->rw.len = sqe_len;
2838 2839
		}

2840 2841
		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
2842
		return ret < 0 ? ret : sqe_len;
2843 2844
	}

2845 2846
	if (req->flags & REQ_F_BUFFER_SELECT) {
		ret = io_iov_buffer_select(req, *iovec, needs_lock);
2847 2848 2849 2850
		if (!ret) {
			ret = (*iovec)->iov_len;
			iov_iter_init(iter, rw, *iovec, 1, ret);
		}
2851 2852 2853 2854
		*iovec = NULL;
		return ret;
	}

J
Jens Axboe 已提交
2855
#ifdef CONFIG_COMPAT
2856
	if (req->ctx->compat)
J
Jens Axboe 已提交
2857 2858 2859 2860 2861 2862 2863
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

2864 2865 2866 2867 2868 2869 2870 2871 2872 2873
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter,
			       bool needs_lock)
{
	if (!req->io)
		return __io_import_iovec(rw, req, iovec, iter, needs_lock);
	*iovec = NULL;
	return iov_iter_count(&req->io->rw.iter);
}

2874 2875 2876 2877 2878
static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
{
	return kiocb->ki_filp->f_mode & FMODE_STREAM ? NULL : &kiocb->ki_pos;
}

2879
/*
2880 2881
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
2882
 */
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
2899
		struct iovec iovec;
2900 2901
		ssize_t nr;

2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

2912 2913
		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
2914
					      iovec.iov_len, io_kiocb_ppos(kiocb));
2915 2916
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
2917
					       iovec.iov_len, io_kiocb_ppos(kiocb));
2918 2919
		}

2920 2921 2922
		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

2937 2938
static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
			  const struct iovec *fast_iov, struct iov_iter *iter)
2939
{
2940 2941
	struct io_async_rw *rw = &req->io->rw;

2942 2943
	memcpy(&rw->iter, iter, sizeof(*iter));
	rw->free_iovec = NULL;
2944
	rw->bytes_done = 0;
2945 2946 2947
	/* can only be fixed buffers, no need to do anything */
	if (iter->type == ITER_BVEC)
		return;
2948
	if (!iovec) {
2949 2950 2951 2952 2953 2954 2955 2956 2957
		unsigned iov_off = 0;

		rw->iter.iov = rw->fast_iov;
		if (iter->iov != fast_iov) {
			iov_off = iter->iov - fast_iov;
			rw->iter.iov += iov_off;
		}
		if (rw->fast_iov != fast_iov)
			memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
2958
			       sizeof(struct iovec) * iter->nr_segs);
P
Pavel Begunkov 已提交
2959
	} else {
2960
		rw->free_iovec = iovec;
P
Pavel Begunkov 已提交
2961
		req->flags |= REQ_F_NEED_CLEANUP;
2962 2963 2964
	}
}

2965 2966 2967 2968 2969 2970
static inline int __io_alloc_async_ctx(struct io_kiocb *req)
{
	req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
	return req->io == NULL;
}

2971
static int io_alloc_async_ctx(struct io_kiocb *req)
2972
{
2973 2974
	if (!io_op_defs[req->opcode].async_ctx)
		return 0;
2975 2976

	return  __io_alloc_async_ctx(req);
2977 2978
}

2979 2980
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
			     const struct iovec *fast_iov,
2981
			     struct iov_iter *iter, bool force)
2982
{
2983
	if (!force && !io_op_defs[req->opcode].async_ctx)
2984
		return 0;
2985
	if (!req->io) {
2986
		if (__io_alloc_async_ctx(req))
2987
			return -ENOMEM;
2988

2989
		io_req_map_rw(req, iovec, fast_iov, iter);
2990
	}
2991
	return 0;
2992 2993
}

2994 2995 2996
static inline int io_rw_prep_async(struct io_kiocb *req, int rw,
				   bool force_nonblock)
{
2997
	struct io_async_rw *iorw = &req->io->rw;
2998
	struct iovec *iov;
2999 3000
	ssize_t ret;

3001 3002
	iorw->iter.iov = iov = iorw->fast_iov;
	ret = __io_import_iovec(rw, req, &iov, &iorw->iter, !force_nonblock);
3003 3004 3005
	if (unlikely(ret < 0))
		return ret;

3006
	iorw->iter.iov = iov;
3007
	io_req_map_rw(req, iorw->iter.iov, iorw->fast_iov, &iorw->iter);
3008 3009 3010
	return 0;
}

3011 3012
static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
3013 3014 3015
{
	ssize_t ret;

3016 3017 3018
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
3019

3020 3021
	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;
3022

3023 3024
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
3025
		return 0;
3026
	return io_rw_prep_async(req, READ, force_nonblock);
3027 3028
}

3029 3030 3031 3032 3033 3034 3035 3036 3037 3038
/*
 * This is our waitqueue callback handler, registered through lock_page_async()
 * when we initially tried to do the IO with the iocb armed our waitqueue.
 * This gets called when the page is unlocked, and we generally expect that to
 * happen when the page IO is completed and the page is now uptodate. This will
 * queue a task_work based retry of the operation, attempting to copy the data
 * again. If the latter fails because the page was NOT uptodate, then we will
 * do a thread based blocking retry of the operation. That's the unexpected
 * slow path.
 */
3039 3040 3041 3042 3043 3044 3045 3046 3047 3048
static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
			     int sync, void *arg)
{
	struct wait_page_queue *wpq;
	struct io_kiocb *req = wait->private;
	struct wait_page_key *key = arg;
	int ret;

	wpq = container_of(wait, struct wait_page_queue, wait);

3049 3050 3051
	if (!wake_page_match(wpq, key))
		return 0;

3052 3053
	list_del_init(&wait->entry);

3054
	init_task_work(&req->task_work, io_req_task_submit);
3055 3056
	percpu_ref_get(&req->ctx->refs);

3057 3058
	/* submit ref gets dropped, acquire a new one */
	refcount_inc(&req->refs);
3059
	ret = io_req_task_work_add(req, &req->task_work, true);
3060
	if (unlikely(ret)) {
3061 3062
		struct task_struct *tsk;

3063
		/* queue just for cancelation */
3064
		init_task_work(&req->task_work, io_req_task_cancel);
3065
		tsk = io_wq_get_task(req->ctx->io_wq);
3066
		task_work_add(tsk, &req->task_work, 0);
3067
		wake_up_process(tsk);
3068 3069 3070 3071
	}
	return 1;
}

3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083
/*
 * This controls whether a given IO request should be armed for async page
 * based retry. If we return false here, the request is handed to the async
 * worker threads for retry. If we're doing buffered reads on a regular file,
 * we prepare a private wait_page_queue entry and retry the operation. This
 * will either succeed because the page is now uptodate and unlocked, or it
 * will register a callback when the page is unlocked at IO completion. Through
 * that callback, io_uring uses task_work to setup a retry of the operation.
 * That retry will attempt the buffered read again. The retry will generally
 * succeed, or in rare cases where it fails, we then fall back to using the
 * async worker threads for a blocking retry.
 */
3084
static bool io_rw_should_retry(struct io_kiocb *req)
3085
{
3086
	struct wait_page_queue *wait = &req->io->rw.wpq;
3087
	struct kiocb *kiocb = &req->rw.kiocb;
3088

3089 3090 3091
	/* never retry for NOWAIT, we just complete with -EAGAIN */
	if (req->flags & REQ_F_NOWAIT)
		return false;
3092

3093
	/* Only for buffered IO */
3094
	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
3095
		return false;
3096

3097 3098 3099 3100 3101 3102
	/*
	 * just use poll if we can, and don't attempt if the fs doesn't
	 * support callback based unlocks
	 */
	if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
		return false;
3103

3104 3105 3106 3107 3108 3109
	wait->wait.func = io_async_buf_func;
	wait->wait.private = req;
	wait->wait.flags = 0;
	INIT_LIST_HEAD(&wait->wait.entry);
	kiocb->ki_flags |= IOCB_WAITQ;
	kiocb->ki_waitq = wait;
3110

3111 3112
	io_get_req_task(req);
	return true;
3113 3114 3115 3116 3117 3118
}

static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (req->file->f_op->read_iter)
		return call_read_iter(req->file, &req->rw.kiocb, iter);
3119 3120 3121 3122
	else if (req->file->f_op->read)
		return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
	else
		return -EINVAL;
3123 3124
}

3125 3126
static int io_read(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
J
Jens Axboe 已提交
3127 3128
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3129
	struct kiocb *kiocb = &req->rw.kiocb;
3130
	struct iov_iter __iter, *iter = &__iter;
3131
	ssize_t io_size, ret, ret2;
3132
	size_t iov_count;
3133
	bool no_async;
3134 3135 3136

	if (req->io)
		iter = &req->io->rw.iter;
J
Jens Axboe 已提交
3137

3138
	ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3139 3140
	if (ret < 0)
		return ret;
3141
	iov_count = iov_iter_count(iter);
3142 3143
	io_size = ret;
	req->result = io_size;
3144
	ret = 0;
J
Jens Axboe 已提交
3145

3146 3147
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
3148
		kiocb->ki_flags &= ~IOCB_NOWAIT;
3149

3150
	/* If the file doesn't support async, just async punt */
3151 3152
	no_async = force_nonblock && !io_file_supports_async(req->file, READ);
	if (no_async)
3153
		goto copy_iov;
J
Jens Axboe 已提交
3154

3155
	ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), iov_count);
3156 3157
	if (unlikely(ret))
		goto out_free;
J
Jens Axboe 已提交
3158

3159
	ret = io_iter_do_read(req, iter);
3160

3161 3162 3163 3164 3165 3166
	if (!ret) {
		goto done;
	} else if (ret == -EIOCBQUEUED) {
		ret = 0;
		goto out_free;
	} else if (ret == -EAGAIN) {
3167 3168
		/* IOPOLL retry should happen for io-wq threads */
		if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
3169
			goto done;
3170 3171 3172
		/* no retry on NONBLOCK marked file */
		if (req->file->f_flags & O_NONBLOCK)
			goto done;
3173 3174
		/* some cases will consume bytes even on error returns */
		iov_iter_revert(iter, iov_count - iov_iter_count(iter));
3175
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
3176 3177
		if (ret)
			goto out_free;
3178 3179
		return -EAGAIN;
	} else if (ret < 0) {
3180 3181
		/* make sure -ERESTARTSYS -> -EINTR is done */
		goto done;
3182 3183 3184
	}

	/* read it all, or we did blocking attempt. no retry. */
3185 3186
	if (!iov_iter_count(iter) || !force_nonblock ||
	    (req->file->f_flags & O_NONBLOCK))
3187 3188 3189 3190 3191 3192 3193 3194 3195
		goto done;

	io_size -= ret;
copy_iov:
	ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
	if (ret2) {
		ret = ret2;
		goto out_free;
	}
3196 3197
	if (no_async)
		return -EAGAIN;
3198 3199 3200 3201 3202 3203 3204 3205
	/* it's copied and will be cleaned with ->io */
	iovec = NULL;
	/* now use our persistent iterator, if we aren't already */
	iter = &req->io->rw.iter;
retry:
	req->io->rw.bytes_done += ret;
	/* if we can retry, do so with the callbacks armed */
	if (!io_rw_should_retry(req)) {
3206 3207
		kiocb->ki_flags &= ~IOCB_WAITQ;
		return -EAGAIN;
J
Jens Axboe 已提交
3208
	}
3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226

	/*
	 * Now retry read with the IOCB_WAITQ parts set in the iocb. If we
	 * get -EIOCBQUEUED, then we'll get a notification when the desired
	 * page gets unlocked. We can also get a partial read here, and if we
	 * do, then just retry at the new offset.
	 */
	ret = io_iter_do_read(req, iter);
	if (ret == -EIOCBQUEUED) {
		ret = 0;
		goto out_free;
	} else if (ret > 0 && ret < io_size) {
		/* we got some bytes, but not all. retry. */
		goto retry;
	}
done:
	kiocb_done(kiocb, ret, cs);
	ret = 0;
3227
out_free:
3228
	/* it's reportedly faster than delegating the null check to kfree() */
3229
	if (iovec)
3230
		kfree(iovec);
J
Jens Axboe 已提交
3231 3232 3233
	return ret;
}

3234 3235
static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 bool force_nonblock)
3236 3237 3238
{
	ssize_t ret;

3239 3240 3241
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
3242

3243 3244
	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
		return -EBADF;
3245

3246 3247
	/* either don't need iovec imported or already have it */
	if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
3248
		return 0;
3249
	return io_rw_prep_async(req, WRITE, force_nonblock);
3250 3251
}

3252 3253
static int io_write(struct io_kiocb *req, bool force_nonblock,
		    struct io_comp_state *cs)
J
Jens Axboe 已提交
3254 3255
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3256
	struct kiocb *kiocb = &req->rw.kiocb;
3257
	struct iov_iter __iter, *iter = &__iter;
3258
	size_t iov_count;
3259
	ssize_t ret, ret2, io_size;
J
Jens Axboe 已提交
3260

3261 3262 3263 3264
	if (req->io)
		iter = &req->io->rw.iter;

	ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3265 3266
	if (ret < 0)
		return ret;
3267
	iov_count = iov_iter_count(iter);
3268 3269
	io_size = ret;
	req->result = io_size;
J
Jens Axboe 已提交
3270

3271 3272
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
3273
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
3274

3275
	/* If the file doesn't support async, just async punt */
3276
	if (force_nonblock && !io_file_supports_async(req->file, WRITE))
3277
		goto copy_iov;
3278

3279 3280 3281
	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
3282
		goto copy_iov;
3283

3284
	ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), iov_count);
3285 3286
	if (unlikely(ret))
		goto out_free;
3287

3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301
	/*
	 * Open-code file_start_write here to grab freeze protection,
	 * which will be released by another thread in
	 * io_complete_rw().  Fool lockdep by telling it the lock got
	 * released so that it doesn't complain about the held lock when
	 * we return to userspace.
	 */
	if (req->flags & REQ_F_ISREG) {
		__sb_start_write(file_inode(req->file)->i_sb,
					SB_FREEZE_WRITE, true);
		__sb_writers_release(file_inode(req->file)->i_sb,
					SB_FREEZE_WRITE);
	}
	kiocb->ki_flags |= IOCB_WRITE;
3302

3303
	if (req->file->f_op->write_iter)
3304
		ret2 = call_write_iter(req->file, kiocb, iter);
3305
	else if (req->file->f_op->write)
3306
		ret2 = loop_rw_iter(WRITE, req->file, kiocb, iter);
3307 3308
	else
		ret2 = -EINVAL;
3309

3310 3311 3312 3313 3314 3315
	/*
	 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
	 * retry them without IOCB_NOWAIT.
	 */
	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
		ret2 = -EAGAIN;
3316 3317 3318
	/* no retry on NONBLOCK marked file */
	if (ret2 == -EAGAIN && (req->file->f_flags & O_NONBLOCK))
		goto done;
3319
	if (!force_nonblock || ret2 != -EAGAIN) {
3320 3321 3322
		/* IOPOLL retry should happen for io-wq threads */
		if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
			goto copy_iov;
3323
done:
3324 3325
		kiocb_done(kiocb, ret2, cs);
	} else {
3326
copy_iov:
3327 3328
		/* some cases will consume bytes even on error returns */
		iov_iter_revert(iter, iov_count - iov_iter_count(iter));
3329
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
3330 3331
		if (!ret)
			return -EAGAIN;
J
Jens Axboe 已提交
3332
	}
3333
out_free:
3334
	/* it's reportedly faster than delegating the null check to kfree() */
3335
	if (iovec)
3336
		kfree(iovec);
J
Jens Axboe 已提交
3337 3338 3339
	return ret;
}

P
Pavel Begunkov 已提交
3340 3341
static int __io_splice_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
P
Pavel Begunkov 已提交
3342 3343 3344 3345 3346 3347 3348
{
	struct io_splice* sp = &req->splice;
	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
	int ret;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3349 3350
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
P
Pavel Begunkov 已提交
3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364

	sp->file_in = NULL;
	sp->len = READ_ONCE(sqe->len);
	sp->flags = READ_ONCE(sqe->splice_flags);

	if (unlikely(sp->flags & ~valid_flags))
		return -EINVAL;

	ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
			  (sp->flags & SPLICE_F_FD_IN_FIXED));
	if (ret)
		return ret;
	req->flags |= REQ_F_NEED_CLEANUP;

3365 3366 3367 3368 3369 3370
	if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
		/*
		 * Splice operation will be punted aync, and here need to
		 * modify io_wq_work.flags, so initialize io_wq_work firstly.
		 */
		io_req_init_async(req);
P
Pavel Begunkov 已提交
3371
		req->work.flags |= IO_WQ_WORK_UNBOUND;
3372
	}
P
Pavel Begunkov 已提交
3373 3374 3375 3376

	return 0;
}

P
Pavel Begunkov 已提交
3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402
static int io_tee_prep(struct io_kiocb *req,
		       const struct io_uring_sqe *sqe)
{
	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
		return -EINVAL;
	return __io_splice_prep(req, sqe);
}

static int io_tee(struct io_kiocb *req, bool force_nonblock)
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	long ret = 0;

	if (force_nonblock)
		return -EAGAIN;
	if (sp->len)
		ret = do_tee(in, out, sp->len, flags);

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail_links(req);
3403
	io_req_complete(req, ret);
P
Pavel Begunkov 已提交
3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
	return 0;
}

static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_splice* sp = &req->splice;

	sp->off_in = READ_ONCE(sqe->splice_off_in);
	sp->off_out = READ_ONCE(sqe->off);
	return __io_splice_prep(req, sqe);
}

3416
static int io_splice(struct io_kiocb *req, bool force_nonblock)
P
Pavel Begunkov 已提交
3417 3418 3419 3420 3421 3422
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	loff_t *poff_in, *poff_out;
3423
	long ret = 0;
P
Pavel Begunkov 已提交
3424

3425 3426
	if (force_nonblock)
		return -EAGAIN;
P
Pavel Begunkov 已提交
3427 3428 3429

	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
3430

3431
	if (sp->len)
3432
		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
P
Pavel Begunkov 已提交
3433 3434 3435 3436 3437 3438

	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail_links(req);
3439
	io_req_complete(req, ret);
P
Pavel Begunkov 已提交
3440 3441 3442
	return 0;
}

J
Jens Axboe 已提交
3443 3444 3445
/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
3446
static int io_nop(struct io_kiocb *req, struct io_comp_state *cs)
J
Jens Axboe 已提交
3447 3448 3449
{
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
3450 3451 3452
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

3453
	__io_req_complete(req, 0, 0, cs);
J
Jens Axboe 已提交
3454 3455 3456
	return 0;
}

3457
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
C
Christoph Hellwig 已提交
3458
{
J
Jens Axboe 已提交
3459
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
3460

J
Jens Axboe 已提交
3461 3462
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
3463

J
Jens Axboe 已提交
3464
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
3465
		return -EINVAL;
3466
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
3467 3468
		return -EINVAL;

3469 3470 3471 3472 3473 3474
	req->sync.flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
C
Christoph Hellwig 已提交
3475 3476 3477
	return 0;
}

3478
static int io_fsync(struct io_kiocb *req, bool force_nonblock)
3479 3480 3481 3482
{
	loff_t end = req->sync.off + req->sync.len;
	int ret;

3483 3484 3485 3486
	/* fsync always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

3487
	ret = vfs_fsync_range(req->file, req->sync.off,
3488 3489 3490 3491
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
3492
	io_req_complete(req, ret);
C
Christoph Hellwig 已提交
3493 3494 3495
	return 0;
}

3496 3497 3498 3499 3500
static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;
3501 3502
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3503 3504 3505 3506 3507 3508 3509

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
	return 0;
}

3510
static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
3511
{
3512 3513
	int ret;

3514
	/* fallocate always requiring blocking context */
3515
	if (force_nonblock)
3516
		return -EAGAIN;
3517 3518 3519 3520
	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
				req->sync.len);
	if (ret < 0)
		req_set_fail_links(req);
3521
	io_req_complete(req, ret);
3522 3523 3524
	return 0;
}

3525
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3526
{
3527
	const char __user *fname;
3528
	int ret;
3529

3530
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3531
		return -EINVAL;
3532
	if (unlikely(sqe->ioprio || sqe->buf_index))
3533
		return -EINVAL;
3534
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
3535
		return -EBADF;
3536

3537 3538
	/* open.how should be already initialised */
	if (!(req->open.how.flags & O_PATH) && force_o_largefile())
3539
		req->open.how.flags |= O_LARGEFILE;
3540

3541 3542
	req->open.dfd = READ_ONCE(sqe->fd);
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3543
	req->open.filename = getname(fname);
3544 3545 3546 3547 3548
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}
3549
	req->open.nofile = rlimit(RLIMIT_NOFILE);
3550
	req->flags |= REQ_F_NEED_CLEANUP;
3551
	return 0;
3552 3553
}

3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565
static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	u64 flags, mode;

	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
	mode = READ_ONCE(sqe->len);
	flags = READ_ONCE(sqe->open_flags);
	req->open.how = build_open_how(flags, mode);
	return __io_openat_prep(req, sqe);
}

3566
static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
3567
{
3568 3569
	struct open_how __user *how;
	size_t len;
J
Jens Axboe 已提交
3570 3571
	int ret;

3572 3573
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
3574 3575 3576 3577
	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	len = READ_ONCE(sqe->len);
	if (len < OPEN_HOW_SIZE_VER0)
		return -EINVAL;
3578

3579 3580 3581 3582
	ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
					len);
	if (ret)
		return ret;
3583

3584
	return __io_openat_prep(req, sqe);
3585 3586
}

3587
static int io_openat2(struct io_kiocb *req, bool force_nonblock)
3588 3589 3590 3591 3592
{
	struct open_flags op;
	struct file *file;
	int ret;

3593
	if (force_nonblock)
3594 3595
		return -EAGAIN;

3596
	ret = build_open_flags(&req->open.how, &op);
3597 3598 3599
	if (ret)
		goto err;

3600
	ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613
	if (ret < 0)
		goto err;

	file = do_filp_open(req->open.dfd, req->open.filename, &op);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
	} else {
		fsnotify_open(file);
		fd_install(ret, file);
	}
err:
	putname(req->open.filename);
3614
	req->flags &= ~REQ_F_NEED_CLEANUP;
3615 3616
	if (ret < 0)
		req_set_fail_links(req);
3617
	io_req_complete(req, ret);
3618 3619 3620
	return 0;
}

3621
static int io_openat(struct io_kiocb *req, bool force_nonblock)
3622
{
3623
	return io_openat2(req, force_nonblock);
3624 3625
}

3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670
static int io_remove_buffers_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}

static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
			       int bgid, unsigned nbufs)
{
	unsigned i = 0;

	/* shouldn't happen */
	if (!nbufs)
		return 0;

	/* the head kbuf is the list itself */
	while (!list_empty(&buf->list)) {
		struct io_buffer *nxt;

		nxt = list_first_entry(&buf->list, struct io_buffer, list);
		list_del(&nxt->list);
		kfree(nxt);
		if (++i == nbufs)
			return i;
	}
	i++;
	kfree(buf);
	idr_remove(&ctx->io_buffer_idr, bgid);

	return i;
}

3671 3672
static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock,
			     struct io_comp_state *cs)
3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	ret = -ENOENT;
	head = idr_find(&ctx->io_buffer_idr, p->bgid);
	if (head)
		ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);

	io_ring_submit_lock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
3691
	__io_req_complete(req, ret, 0, cs);
3692 3693 3694
	return 0;
}

3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710
static int io_provide_buffers_prep(struct io_kiocb *req,
				   const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

3711
	if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}

static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
{
	struct io_buffer *buf;
	u64 addr = pbuf->addr;
	int i, bid = pbuf->bid;

	for (i = 0; i < pbuf->nbufs; i++) {
		buf = kmalloc(sizeof(*buf), GFP_KERNEL);
		if (!buf)
			break;

		buf->addr = addr;
		buf->len = pbuf->len;
		buf->bid = bid;
		addr += pbuf->len;
		bid++;
		if (!*head) {
			INIT_LIST_HEAD(&buf->list);
			*head = buf;
		} else {
			list_add_tail(&buf->list, &(*head)->list);
		}
	}

	return i ? i : -ENOMEM;
}

3749 3750
static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock,
			      struct io_comp_state *cs)
3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head, *list;
	int ret = 0;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	list = head = idr_find(&ctx->io_buffer_idr, p->bgid);

	ret = io_add_buffers(p, &head);
	if (ret < 0)
		goto out;

	if (!list) {
		ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
					GFP_KERNEL);
		if (ret < 0) {
3771
			__io_remove_buffers(ctx, head, p->bgid, -1U);
3772 3773 3774 3775 3776 3777 3778
			goto out;
		}
	}
out:
	io_ring_submit_unlock(ctx, !force_nonblock);
	if (ret < 0)
		req_set_fail_links(req);
3779
	__io_req_complete(req, ret, 0, cs);
3780
	return 0;
3781 3782
}

3783 3784 3785 3786 3787 3788
static int io_epoll_ctl_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_EPOLL)
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3789 3790
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809

	req->epoll.epfd = READ_ONCE(sqe->fd);
	req->epoll.op = READ_ONCE(sqe->len);
	req->epoll.fd = READ_ONCE(sqe->off);

	if (ep_op_has_event(req->epoll.op)) {
		struct epoll_event __user *ev;

		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
		if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
			return -EFAULT;
	}

	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3810 3811
static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock,
			struct io_comp_state *cs)
3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
{
#if defined(CONFIG_EPOLL)
	struct io_epoll *ie = &req->epoll;
	int ret;

	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;

	if (ret < 0)
		req_set_fail_links(req);
3823
	__io_req_complete(req, ret, 0, cs);
3824 3825 3826 3827 3828 3829
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3830 3831 3832 3833 3834
static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	if (sqe->ioprio || sqe->buf_index || sqe->off)
		return -EINVAL;
3835 3836
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846

	req->madvise.addr = READ_ONCE(sqe->addr);
	req->madvise.len = READ_ONCE(sqe->len);
	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3847
static int io_madvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
3859
	io_req_complete(req, ret);
J
Jens Axboe 已提交
3860 3861 3862 3863 3864 3865
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
3866 3867 3868 3869
static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr)
		return -EINVAL;
3870 3871
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
J
Jens Axboe 已提交
3872 3873 3874 3875 3876 3877 3878

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}

3879
static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
J
Jens Axboe 已提交
3880 3881 3882 3883
{
	struct io_fadvise *fa = &req->fadvise;
	int ret;

3884 3885 3886 3887 3888 3889 3890 3891 3892 3893
	if (force_nonblock) {
		switch (fa->advice) {
		case POSIX_FADV_NORMAL:
		case POSIX_FADV_RANDOM:
		case POSIX_FADV_SEQUENTIAL:
			break;
		default:
			return -EAGAIN;
		}
	}
J
Jens Axboe 已提交
3894 3895 3896 3897

	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
	if (ret < 0)
		req_set_fail_links(req);
3898
	io_req_complete(req, ret);
J
Jens Axboe 已提交
3899 3900 3901
	return 0;
}

3902 3903
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
3904 3905
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
3906 3907
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
3908
	if (req->flags & REQ_F_FIXED_FILE)
3909
		return -EBADF;
3910

3911 3912
	req->statx.dfd = READ_ONCE(sqe->fd);
	req->statx.mask = READ_ONCE(sqe->len);
B
Bijan Mottahedeh 已提交
3913
	req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
3914 3915
	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	req->statx.flags = READ_ONCE(sqe->statx_flags);
3916 3917 3918 3919

	return 0;
}

3920
static int io_statx(struct io_kiocb *req, bool force_nonblock)
3921
{
3922
	struct io_statx *ctx = &req->statx;
3923 3924
	int ret;

3925 3926 3927 3928
	if (force_nonblock) {
		/* only need file table for an actual valid fd */
		if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD)
			req->flags |= REQ_F_NO_FILE_TABLE;
3929
		return -EAGAIN;
3930
	}
3931

B
Bijan Mottahedeh 已提交
3932 3933
	ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
		       ctx->buffer);
3934 3935 3936

	if (ret < 0)
		req_set_fail_links(req);
3937
	io_req_complete(req, ret);
3938 3939 3940
	return 0;
}

3941 3942 3943 3944
static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	/*
	 * If we queue this for async, it must not be cancellable. That would
3945 3946
	 * leave the 'file' in an undeterminate state, and here need to modify
	 * io_wq_work.flags, so initialize io_wq_work firstly.
3947
	 */
3948
	io_req_init_async(req);
3949 3950
	req->work.flags |= IO_WQ_WORK_NO_CANCEL;

3951 3952
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
3953 3954 3955
	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
	    sqe->rw_flags || sqe->buf_index)
		return -EINVAL;
3956
	if (req->flags & REQ_F_FIXED_FILE)
3957
		return -EBADF;
3958 3959

	req->close.fd = READ_ONCE(sqe->fd);
3960 3961 3962
	if ((req->file && req->file->f_op == &io_uring_fops) ||
	    req->close.fd == req->ctx->ring_fd)
		return -EBADF;
3963

3964
	req->close.put_file = NULL;
3965 3966 3967
	return 0;
}

3968 3969
static int io_close(struct io_kiocb *req, bool force_nonblock,
		    struct io_comp_state *cs)
3970
{
3971
	struct io_close *close = &req->close;
3972 3973
	int ret;

3974 3975 3976 3977 3978 3979
	/* might be already done during nonblock submission */
	if (!close->put_file) {
		ret = __close_fd_get_file(close->fd, &close->put_file);
		if (ret < 0)
			return (ret == -ENOENT) ? -EBADF : ret;
	}
3980 3981

	/* if the file has a flush method, be safe and punt to async */
3982
	if (close->put_file->f_op->flush && force_nonblock) {
3983 3984
		/* was never set, but play safe */
		req->flags &= ~REQ_F_NOWAIT;
3985
		/* avoid grabbing files - we don't need the files */
3986
		req->flags |= REQ_F_NO_FILE_TABLE;
3987
		return -EAGAIN;
P
Pavel Begunkov 已提交
3988
	}
3989

3990 3991 3992 3993 3994 3995
	/* No ->flush() or already async, safely close from here */
	ret = filp_close(close->put_file, req->work.files);
	if (ret < 0)
		req_set_fail_links(req);
	fput(close->put_file);
	close->put_file = NULL;
3996
	__io_req_complete(req, ret, 0, cs);
3997
	return 0;
3998 3999
}

4000
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

4012 4013 4014 4015 4016 4017
	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
	req->sync.flags = READ_ONCE(sqe->sync_range_flags);
	return 0;
}

4018
static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
4019 4020 4021
{
	int ret;

4022 4023 4024 4025
	/* sync_file_range always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

4026
	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
4027 4028 4029
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
4030
	io_req_complete(req, ret);
4031 4032 4033
	return 0;
}

4034
#if defined(CONFIG_NET)
4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049
static int io_setup_async_msg(struct io_kiocb *req,
			      struct io_async_msghdr *kmsg)
{
	if (req->io)
		return -EAGAIN;
	if (io_alloc_async_ctx(req)) {
		if (kmsg->iov != kmsg->fast_iov)
			kfree(kmsg->iov);
		return -ENOMEM;
	}
	req->flags |= REQ_F_NEED_CLEANUP;
	memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
	return -EAGAIN;
}

4050 4051 4052 4053 4054 4055 4056 4057 4058
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
{
	iomsg->iov = iomsg->fast_iov;
	iomsg->msg.msg_name = &iomsg->addr;
	return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
				   req->sr_msg.msg_flags, &iomsg->iov);
}

4059
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4060
{
4061
	struct io_sr_msg *sr = &req->sr_msg;
4062
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
4063
	int ret;
4064

4065 4066 4067
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

4068
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
4069
	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4070
	sr->len = READ_ONCE(sqe->len);
4071

4072 4073 4074 4075 4076
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

4077
	if (!io || req->opcode == IORING_OP_SEND)
4078
		return 0;
4079 4080 4081
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
4082

4083
	ret = io_sendmsg_copy_hdr(req, &io->msg);
P
Pavel Begunkov 已提交
4084 4085 4086
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
4087 4088
}

4089 4090
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
J
Jens Axboe 已提交
4091
{
4092
	struct io_async_msghdr iomsg, *kmsg;
J
Jens Axboe 已提交
4093
	struct socket *sock;
4094
	unsigned flags;
J
Jens Axboe 已提交
4095 4096 4097
	int ret;

	sock = sock_from_file(req->file, &ret);
4098 4099
	if (unlikely(!sock))
		return ret;
4100

4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112
	if (req->io) {
		kmsg = &req->io->msg;
		kmsg->msg.msg_name = &req->io->msg.addr;
		/* if iov is set, it's allocated already */
		if (!kmsg->iov)
			kmsg->iov = kmsg->fast_iov;
		kmsg->msg.msg_iter.iov = kmsg->iov;
	} else {
		ret = io_sendmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
J
Jens Axboe 已提交
4113 4114
	}

4115 4116 4117 4118 4119
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4120

4121 4122 4123 4124 4125
	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
	if (force_nonblock && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
J
Jens Axboe 已提交
4126

4127
	if (kmsg->iov != kmsg->fast_iov)
4128
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
4129
	req->flags &= ~REQ_F_NEED_CLEANUP;
J
Jens Axboe 已提交
4130 4131
	if (ret < 0)
		req_set_fail_links(req);
4132
	__io_req_complete(req, ret, 0, cs);
4133
	return 0;
4134
}
J
Jens Axboe 已提交
4135

4136 4137
static int io_send(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4138
{
4139 4140 4141
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	struct iovec iov;
4142
	struct socket *sock;
4143
	unsigned flags;
4144 4145 4146
	int ret;

	sock = sock_from_file(req->file, &ret);
4147 4148
	if (unlikely(!sock))
		return ret;
4149

4150 4151
	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
	if (unlikely(ret))
4152
		return ret;;
4153

4154 4155 4156 4157
	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;
4158

4159 4160 4161 4162 4163
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4164

4165 4166 4167 4168 4169 4170
	msg.msg_flags = flags;
	ret = sock_sendmsg(sock, &msg);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4171 4172 4173

	if (ret < 0)
		req_set_fail_links(req);
4174
	__io_req_complete(req, ret, 0, cs);
4175 4176 4177
	return 0;
}

4178 4179
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
				 struct io_async_msghdr *iomsg)
4180 4181 4182 4183 4184 4185
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct iovec __user *uiov;
	size_t iov_len;
	int ret;

4186 4187
	ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
					&iomsg->uaddr, &uiov, &iov_len);
4188 4189 4190 4191 4192 4193
	if (ret)
		return ret;

	if (req->flags & REQ_F_BUFFER_SELECT) {
		if (iov_len > 1)
			return -EINVAL;
4194
		if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov)))
4195
			return -EFAULT;
4196 4197
		sr->len = iomsg->iov[0].iov_len;
		iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1,
4198
				sr->len);
4199
		iomsg->iov = NULL;
4200 4201
	} else {
		ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
4202
					&iomsg->iov, &iomsg->msg.msg_iter);
4203 4204 4205 4206 4207 4208 4209 4210 4211
		if (ret > 0)
			ret = 0;
	}

	return ret;
}

#ifdef CONFIG_COMPAT
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
4212
					struct io_async_msghdr *iomsg)
4213 4214 4215 4216 4217 4218 4219 4220
{
	struct compat_msghdr __user *msg_compat;
	struct io_sr_msg *sr = &req->sr_msg;
	struct compat_iovec __user *uiov;
	compat_uptr_t ptr;
	compat_size_t len;
	int ret;

4221
	msg_compat = (struct compat_msghdr __user *) sr->umsg;
4222
	ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238
					&ptr, &len);
	if (ret)
		return ret;

	uiov = compat_ptr(ptr);
	if (req->flags & REQ_F_BUFFER_SELECT) {
		compat_ssize_t clen;

		if (len > 1)
			return -EINVAL;
		if (!access_ok(uiov, sizeof(*uiov)))
			return -EFAULT;
		if (__get_user(clen, &uiov->iov_len))
			return -EFAULT;
		if (clen < 0)
			return -EINVAL;
4239 4240
		sr->len = iomsg->iov[0].iov_len;
		iomsg->iov = NULL;
4241 4242
	} else {
		ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
4243 4244
						&iomsg->iov,
						&iomsg->msg.msg_iter);
4245 4246 4247 4248 4249 4250 4251 4252
		if (ret < 0)
			return ret;
	}

	return 0;
}
#endif

4253 4254
static int io_recvmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
4255
{
4256 4257
	iomsg->msg.msg_name = &iomsg->addr;
	iomsg->iov = iomsg->fast_iov;
4258 4259 4260

#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
4261
		return __io_compat_recvmsg_copy_hdr(req, iomsg);
4262
#endif
4263

4264
	return __io_recvmsg_copy_hdr(req, iomsg);
4265 4266
}

4267
static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
4268
					       bool needs_lock)
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct io_buffer *kbuf;

	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
	if (IS_ERR(kbuf))
		return kbuf;

	sr->kbuf = kbuf;
	req->flags |= REQ_F_BUFFER_SELECTED;
	return kbuf;
4280 4281
}

4282 4283 4284 4285 4286
static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
{
	return io_put_kbuf(req, req->sr_msg.kbuf);
}

4287 4288
static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4289
{
4290
	struct io_sr_msg *sr = &req->sr_msg;
4291
	struct io_async_ctx *io = req->io;
P
Pavel Begunkov 已提交
4292
	int ret;
4293

4294 4295 4296
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

4297
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
4298
	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4299
	sr->len = READ_ONCE(sqe->len);
4300
	sr->bgid = READ_ONCE(sqe->buf_group);
4301

4302 4303 4304 4305 4306
#ifdef CONFIG_COMPAT
	if (req->ctx->compat)
		sr->msg_flags |= MSG_CMSG_COMPAT;
#endif

4307
	if (!io || req->opcode == IORING_OP_RECV)
4308
		return 0;
4309 4310 4311
	/* iovec is already imported */
	if (req->flags & REQ_F_NEED_CLEANUP)
		return 0;
4312

4313
	ret = io_recvmsg_copy_hdr(req, &io->msg);
P
Pavel Begunkov 已提交
4314 4315 4316
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
J
Jens Axboe 已提交
4317 4318
}

4319 4320
static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
J
Jens Axboe 已提交
4321
{
4322
	struct io_async_msghdr iomsg, *kmsg;
4323
	struct socket *sock;
4324
	struct io_buffer *kbuf;
4325
	unsigned flags;
4326
	int ret, cflags = 0;
4327 4328

	sock = sock_from_file(req->file, &ret);
4329 4330
	if (unlikely(!sock))
		return ret;
4331

4332 4333 4334 4335 4336 4337 4338 4339 4340 4341
	if (req->io) {
		kmsg = &req->io->msg;
		kmsg->msg.msg_name = &req->io->msg.addr;
		/* if iov is set, it's allocated already */
		if (!kmsg->iov)
			kmsg->iov = kmsg->fast_iov;
		kmsg->msg.msg_iter.iov = kmsg->iov;
	} else {
		ret = io_recvmsg_copy_hdr(req, &iomsg);
		if (ret)
4342
			return ret;
4343 4344
		kmsg = &iomsg;
	}
4345

4346
	if (req->flags & REQ_F_BUFFER_SELECT) {
4347
		kbuf = io_recv_buffer_select(req, !force_nonblock);
4348
		if (IS_ERR(kbuf))
4349
			return PTR_ERR(kbuf);
4350 4351 4352 4353
		kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
				1, req->sr_msg.len);
	}
4354

4355 4356 4357 4358 4359
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;
4360

4361 4362
	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
					kmsg->uaddr, flags);
4363 4364
	if (force_nonblock && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
4365 4366
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4367

4368 4369
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
4370
	if (kmsg->iov != kmsg->fast_iov)
4371
		kfree(kmsg->iov);
P
Pavel Begunkov 已提交
4372
	req->flags &= ~REQ_F_NEED_CLEANUP;
J
Jens Axboe 已提交
4373 4374
	if (ret < 0)
		req_set_fail_links(req);
4375
	__io_req_complete(req, ret, cflags, cs);
4376
	return 0;
J
Jens Axboe 已提交
4377
}
4378

4379 4380
static int io_recv(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4381
{
4382
	struct io_buffer *kbuf;
4383 4384 4385
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	void __user *buf = sr->buf;
4386
	struct socket *sock;
4387 4388
	struct iovec iov;
	unsigned flags;
4389
	int ret, cflags = 0;
4390 4391

	sock = sock_from_file(req->file, &ret);
4392 4393
	if (unlikely(!sock))
		return ret;
4394

4395
	if (req->flags & REQ_F_BUFFER_SELECT) {
4396
		kbuf = io_recv_buffer_select(req, !force_nonblock);
4397 4398
		if (IS_ERR(kbuf))
			return PTR_ERR(kbuf);
4399
		buf = u64_to_user_ptr(kbuf->addr);
4400
	}
4401

4402
	ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
4403 4404
	if (unlikely(ret))
		goto out_free;
4405

4406 4407 4408 4409 4410 4411
	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;
	msg.msg_iocb = NULL;
	msg.msg_flags = 0;
4412

4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423
	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (force_nonblock)
		flags |= MSG_DONTWAIT;

	ret = sock_recvmsg(sock, &msg, flags);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4424
out_free:
4425 4426
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
4427 4428
	if (ret < 0)
		req_set_fail_links(req);
4429
	__io_req_complete(req, ret, cflags, cs);
4430 4431 4432
	return 0;
}

4433
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4434
{
4435 4436
	struct io_accept *accept = &req->accept;

4437 4438
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
4439
	if (sqe->ioprio || sqe->len || sqe->buf_index)
4440 4441
		return -EINVAL;

4442 4443
	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4444
	accept->flags = READ_ONCE(sqe->accept_flags);
4445
	accept->nofile = rlimit(RLIMIT_NOFILE);
4446 4447
	return 0;
}
4448

4449 4450
static int io_accept(struct io_kiocb *req, bool force_nonblock,
		     struct io_comp_state *cs)
4451 4452
{
	struct io_accept *accept = &req->accept;
4453
	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
4454 4455
	int ret;

4456 4457 4458
	if (req->file->f_flags & O_NONBLOCK)
		req->flags |= REQ_F_NOWAIT;

4459
	ret = __sys_accept4_file(req->file, file_flags, accept->addr,
4460 4461
					accept->addr_len, accept->flags,
					accept->nofile);
4462
	if (ret == -EAGAIN && force_nonblock)
4463
		return -EAGAIN;
4464 4465 4466
	if (ret < 0) {
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
4467
		req_set_fail_links(req);
4468
	}
4469
	__io_req_complete(req, ret, 0, cs);
4470
	return 0;
4471 4472
}

4473
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4474
{
4475 4476
	struct io_connect *conn = &req->connect;
	struct io_async_ctx *io = req->io;
4477

4478 4479 4480 4481 4482
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

4483 4484 4485 4486 4487 4488 4489
	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	conn->addr_len =  READ_ONCE(sqe->addr2);

	if (!io)
		return 0;

	return move_addr_to_kernel(conn->addr, conn->addr_len,
4490
					&io->connect.address);
4491 4492
}

4493 4494
static int io_connect(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4495
{
4496
	struct io_async_ctx __io, *io;
4497
	unsigned file_flags;
4498
	int ret;
4499

4500 4501 4502
	if (req->io) {
		io = req->io;
	} else {
4503 4504 4505
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
4506 4507 4508 4509 4510
		if (ret)
			goto out;
		io = &__io;
	}

4511 4512 4513 4514
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
4515
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
4516 4517 4518
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
4519 4520 4521
			ret = -ENOMEM;
			goto out;
		}
4522
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
4523
		return -EAGAIN;
4524
	}
4525 4526
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
4527
out:
J
Jens Axboe 已提交
4528 4529
	if (ret < 0)
		req_set_fail_links(req);
4530
	__io_req_complete(req, ret, 0, cs);
4531
	return 0;
4532 4533 4534 4535
}
#else /* !CONFIG_NET */
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
4536 4537 4538
	return -EOPNOTSUPP;
}

4539 4540
static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4541 4542 4543 4544
{
	return -EOPNOTSUPP;
}

4545 4546
static int io_send(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4547 4548 4549 4550 4551 4552 4553 4554 4555 4556
{
	return -EOPNOTSUPP;
}

static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4557 4558
static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4559 4560 4561 4562
{
	return -EOPNOTSUPP;
}

4563 4564
static int io_recv(struct io_kiocb *req, bool force_nonblock,
		   struct io_comp_state *cs)
4565 4566 4567 4568 4569 4570 4571 4572 4573
{
	return -EOPNOTSUPP;
}

static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4574 4575
static int io_accept(struct io_kiocb *req, bool force_nonblock,
		     struct io_comp_state *cs)
4576 4577 4578
{
	return -EOPNOTSUPP;
}
4579

4580 4581 4582 4583 4584
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}

4585 4586
static int io_connect(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
4587
{
4588
	return -EOPNOTSUPP;
4589
}
4590
#endif /* CONFIG_NET */
4591

4592 4593 4594 4595 4596
struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};
4597

4598 4599 4600
static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
			   __poll_t mask, task_work_func_t func)
{
4601
	bool twa_signal_ok;
4602
	int ret;
4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

	trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);

	list_del_init(&poll->wait.entry);

	req->result = mask;
	init_task_work(&req->task_work, func);
4614 4615
	percpu_ref_get(&req->ctx->refs);

4616 4617 4618 4619 4620 4621 4622 4623
	/*
	 * If we using the signalfd wait_queue_head for this wakeup, then
	 * it's not safe to use TWA_SIGNAL as we could be recursing on the
	 * tsk->sighand->siglock on doing the wakeup. Should not be needed
	 * either, as the normal wakeup will suffice.
	 */
	twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh);

4624
	/*
4625 4626 4627 4628
	 * If this fails, then the task is exiting. When a task exits, the
	 * work gets canceled, so just cancel this request as well instead
	 * of executing it. We can't safely execute it anyway, as we may not
	 * have the needed state needed for it anyway.
4629
	 */
4630
	ret = io_req_task_work_add(req, &req->task_work, twa_signal_ok);
4631
	if (unlikely(ret)) {
4632 4633
		struct task_struct *tsk;

4634
		WRITE_ONCE(poll->canceled, true);
4635
		tsk = io_wq_get_task(req->ctx->io_wq);
4636 4637
		task_work_add(tsk, &req->task_work, 0);
		wake_up_process(tsk);
4638
	}
4639 4640 4641
	return 1;
}

4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
	__acquires(&req->ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->result && !READ_ONCE(poll->canceled)) {
		struct poll_table_struct pt = { ._key = poll->events };

		req->result = vfs_poll(req->file, &pt) & poll->events;
	}

	spin_lock_irq(&ctx->completion_lock);
	if (!req->result && !READ_ONCE(poll->canceled)) {
		add_wait_queue(poll->head, &poll->wait);
		return true;
	}

	return false;
}

4662
static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
4663
{
4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679
	/* pure poll stashes this in ->io, poll driven retry elsewhere */
	if (req->opcode == IORING_OP_POLL_ADD)
		return (struct io_poll_iocb *) req->io;
	return req->apoll->double_poll;
}

static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
{
	if (req->opcode == IORING_OP_POLL_ADD)
		return &req->poll;
	return &req->apoll->poll;
}

static void io_poll_remove_double(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = io_poll_get_double(req);
4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698

	lockdep_assert_held(&req->ctx->completion_lock);

	if (poll && poll->head) {
		struct wait_queue_head *head = poll->head;

		spin_lock(&head->lock);
		list_del_init(&poll->wait.entry);
		if (poll->wait.private)
			refcount_dec(&req->refs);
		poll->head = NULL;
		spin_unlock(&head->lock);
	}
}

static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

4699
	io_poll_remove_double(req);
4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716
	req->poll.done = true;
	io_cqring_fill_event(req, error ? error : mangle_poll(mask));
	io_commit_cqring(ctx);
}

static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	if (io_poll_rewait(req, &req->poll)) {
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}

	hash_del(&req->hash_node);
	io_poll_complete(req, req->result, 0);
	req->flags |= REQ_F_COMP_LOCKED;
4717
	*nxt = io_put_req_find_next(req);
4718 4719 4720 4721 4722 4723 4724 4725
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
}

static void io_poll_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4726
	struct io_ring_ctx *ctx = req->ctx;
4727 4728 4729
	struct io_kiocb *nxt = NULL;

	io_poll_task_handler(req, &nxt);
4730 4731
	if (nxt)
		__io_req_task_submit(nxt);
4732
	percpu_ref_put(&ctx->refs);
4733 4734 4735 4736 4737 4738
}

static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
			       int sync, void *key)
{
	struct io_kiocb *req = wait->private;
4739
	struct io_poll_iocb *poll = io_poll_get_single(req);
4740 4741 4742 4743 4744 4745
	__poll_t mask = key_to_poll(key);

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

4746
	if (poll && poll->head) {
4747 4748
		bool done;

4749 4750
		spin_lock(&poll->head->lock);
		done = list_empty(&poll->wait.entry);
4751
		if (!done)
4752
			list_del_init(&poll->wait.entry);
4753 4754
		/* make sure double remove sees this as being gone */
		wait->private = NULL;
4755
		spin_unlock(&poll->head->lock);
4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774
		if (!done)
			__io_async_wake(req, poll, mask, io_poll_task_func);
	}
	refcount_dec(&req->refs);
	return 1;
}

static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
			      wait_queue_func_t wake_func)
{
	poll->head = NULL;
	poll->done = false;
	poll->canceled = false;
	poll->events = events;
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, wake_func);
}

static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
4775 4776
			    struct wait_queue_head *head,
			    struct io_poll_iocb **poll_ptr)
4777 4778 4779 4780 4781 4782 4783 4784 4785 4786
{
	struct io_kiocb *req = pt->req;

	/*
	 * If poll->head is already set, it's because the file being polled
	 * uses multiple waitqueues for poll handling (eg one for read, one
	 * for write). Setup a separate io_poll_iocb if this happens.
	 */
	if (unlikely(poll->head)) {
		/* already have a 2nd entry, fail a third attempt */
4787
		if (*poll_ptr) {
4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798
			pt->error = -EINVAL;
			return;
		}
		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
		if (!poll) {
			pt->error = -ENOMEM;
			return;
		}
		io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);
		refcount_inc(&req->refs);
		poll->wait.private = req;
4799
		*poll_ptr = poll;
4800 4801 4802 4803
	}

	pt->error = 0;
	poll->head = head;
4804 4805 4806 4807 4808

	if (poll->events & EPOLLEXCLUSIVE)
		add_wait_queue_exclusive(head, &poll->wait);
	else
		add_wait_queue(head, &poll->wait);
4809 4810 4811 4812 4813 4814
}

static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
4815
	struct async_poll *apoll = pt->req->apoll;
4816

4817
	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
4818 4819
}

4820 4821 4822 4823 4824 4825 4826 4827
static void io_async_task_func(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct async_poll *apoll = req->apoll;
	struct io_ring_ctx *ctx = req->ctx;

	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);

4828
	if (io_poll_rewait(req, &apoll->poll)) {
4829
		spin_unlock_irq(&ctx->completion_lock);
4830
		percpu_ref_put(&ctx->refs);
4831
		return;
4832 4833
	}

4834
	/* If req is still hashed, it cannot have been canceled. Don't check. */
4835
	if (hash_hashed(&req->hash_node))
4836
		hash_del(&req->hash_node);
4837

4838
	io_poll_remove_double(req);
4839 4840
	spin_unlock_irq(&ctx->completion_lock);

4841 4842 4843 4844
	if (!READ_ONCE(apoll->poll.canceled))
		__io_req_task_submit(req);
	else
		__io_req_task_cancel(req, -ECANCELED);
4845

4846
	percpu_ref_put(&ctx->refs);
4847
	kfree(apoll->double_poll);
4848
	kfree(apoll);
4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880
}

static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->apoll->poll;

	trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
					key_to_poll(key));

	return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
}

static void io_poll_req_insert(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct hlist_head *list;

	list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	hlist_add_head(&req->hash_node, list);
}

static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
				      struct io_poll_iocb *poll,
				      struct io_poll_table *ipt, __poll_t mask,
				      wait_queue_func_t wake_func)
	__acquires(&ctx->completion_lock)
{
	struct io_ring_ctx *ctx = req->ctx;
	bool cancel = false;

4881
	io_init_poll_iocb(poll, mask, wake_func);
4882
	poll->file = req->file;
4883
	poll->wait.private = req;
4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918

	ipt->pt._key = mask;
	ipt->req = req;
	ipt->error = -EINVAL;

	mask = vfs_poll(req->file, &ipt->pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
		if (unlikely(list_empty(&poll->wait.entry))) {
			if (ipt->error)
				cancel = true;
			ipt->error = 0;
			mask = 0;
		}
		if (mask || ipt->error)
			list_del_init(&poll->wait.entry);
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
			io_poll_req_insert(req);
		spin_unlock(&poll->head->lock);
	}

	return mask;
}

static bool io_arm_poll_handler(struct io_kiocb *req)
{
	const struct io_op_def *def = &io_op_defs[req->opcode];
	struct io_ring_ctx *ctx = req->ctx;
	struct async_poll *apoll;
	struct io_poll_table ipt;
	__poll_t mask, ret;
4919
	int rw;
4920 4921 4922

	if (!req->file || !file_can_poll(req->file))
		return false;
4923
	if (req->flags & REQ_F_POLLED)
4924
		return false;
4925 4926 4927 4928 4929 4930 4931 4932
	if (def->pollin)
		rw = READ;
	else if (def->pollout)
		rw = WRITE;
	else
		return false;
	/* if we can't nonblock try, then no point in arming a poll handler */
	if (!io_file_supports_async(req->file, rw))
4933 4934 4935 4936 4937
		return false;

	apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
	if (unlikely(!apoll))
		return false;
4938
	apoll->double_poll = NULL;
4939 4940

	req->flags |= REQ_F_POLLED;
P
Pavel Begunkov 已提交
4941
	io_get_req_task(req);
4942 4943 4944
	req->apoll = apoll;
	INIT_HLIST_NODE(&req->hash_node);

4945
	mask = 0;
4946
	if (def->pollin)
4947
		mask |= POLLIN | POLLRDNORM;
4948 4949 4950 4951 4952 4953 4954 4955
	if (def->pollout)
		mask |= POLLOUT | POLLWRNORM;
	mask |= POLLERR | POLLPRI;

	ipt.pt._qproc = io_async_queue_proc;

	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
					io_async_wake);
4956
	if (ret || ipt.error) {
4957
		io_poll_remove_double(req);
4958
		spin_unlock_irq(&ctx->completion_lock);
4959
		kfree(apoll->double_poll);
4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970
		kfree(apoll);
		return false;
	}
	spin_unlock_irq(&ctx->completion_lock);
	trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
					apoll->poll.events);
	return true;
}

static bool __io_poll_remove_one(struct io_kiocb *req,
				 struct io_poll_iocb *poll)
4971
{
4972
	bool do_complete = false;
4973 4974 4975

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
4976 4977
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
4978
		do_complete = true;
4979 4980
	}
	spin_unlock(&poll->head->lock);
4981
	hash_del(&req->hash_node);
4982 4983 4984 4985 4986 4987 4988
	return do_complete;
}

static bool io_poll_remove_one(struct io_kiocb *req)
{
	bool do_complete;

4989 4990
	io_poll_remove_double(req);

4991 4992 4993
	if (req->opcode == IORING_OP_POLL_ADD) {
		do_complete = __io_poll_remove_one(req, &req->poll);
	} else {
4994 4995
		struct async_poll *apoll = req->apoll;

4996
		/* non-poll requests have submit ref still */
4997 4998
		do_complete = __io_poll_remove_one(req, &apoll->poll);
		if (do_complete) {
4999
			io_put_req(req);
5000
			kfree(apoll->double_poll);
5001 5002
			kfree(apoll);
		}
5003 5004
	}

5005 5006 5007 5008
	if (do_complete) {
		io_cqring_fill_event(req, -ECANCELED);
		io_commit_cqring(req->ctx);
		req->flags |= REQ_F_COMP_LOCKED;
5009
		req_set_fail_links(req);
5010 5011 5012 5013
		io_put_req(req);
	}

	return do_complete;
5014 5015 5016 5017
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
5018
	struct hlist_node *tmp;
5019
	struct io_kiocb *req;
5020
	int posted = 0, i;
5021 5022

	spin_lock_irq(&ctx->completion_lock);
5023 5024 5025 5026 5027
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node)
5028
			posted += io_poll_remove_one(req);
5029 5030
	}
	spin_unlock_irq(&ctx->completion_lock);
5031

5032 5033
	if (posted)
		io_cqring_ev_posted(ctx);
5034 5035
}

5036 5037
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
5038
	struct hlist_head *list;
5039 5040
	struct io_kiocb *req;

5041 5042
	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	hlist_for_each_entry(req, list, hash_node) {
5043 5044 5045
		if (sqe_addr != req->user_data)
			continue;
		if (io_poll_remove_one(req))
5046
			return 0;
5047
		return -EALREADY;
5048 5049 5050 5051 5052
	}

	return -ENOENT;
}

5053 5054
static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	req->poll.addr = READ_ONCE(sqe->addr);
	return 0;
}

5066 5067 5068 5069
/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
5070
static int io_poll_remove(struct io_kiocb *req)
5071 5072
{
	struct io_ring_ctx *ctx = req->ctx;
5073
	u64 addr;
5074
	int ret;
5075

5076
	addr = req->poll.addr;
5077
	spin_lock_irq(&ctx->completion_lock);
5078
	ret = io_poll_cancel(ctx, addr);
5079 5080
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
5081 5082
	if (ret < 0)
		req_set_fail_links(req);
5083
	io_req_complete(req, ret);
5084 5085 5086 5087 5088 5089
	return 0;
}

static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
5090 5091
	struct io_kiocb *req = wait->private;
	struct io_poll_iocb *poll = &req->poll;
5092

5093
	return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
5094 5095 5096 5097 5098 5099 5100
}

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

5101
	__io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->io);
5102 5103
}

5104
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5105 5106
{
	struct io_poll_iocb *poll = &req->poll;
5107
	u32 events;
5108 5109 5110 5111 5112

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
5113 5114
	if (!poll->file)
		return -EBADF;
5115

5116 5117 5118 5119
	events = READ_ONCE(sqe->poll32_events);
#ifdef __BIG_ENDIAN
	events = swahw32(events);
#endif
5120 5121
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
		       (events & EPOLLEXCLUSIVE);
5122

P
Pavel Begunkov 已提交
5123
	io_get_req_task(req);
5124 5125 5126
	return 0;
}

5127
static int io_poll_add(struct io_kiocb *req)
5128 5129 5130 5131 5132 5133
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	__poll_t mask;

5134
	INIT_HLIST_NODE(&req->hash_node);
5135
	ipt.pt._qproc = io_poll_queue_proc;
5136

5137 5138
	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
					io_poll_wake);
5139

J
Jens Axboe 已提交
5140
	if (mask) { /* no async, we'd stolen it */
5141
		ipt.error = 0;
5142
		io_poll_complete(req, mask, 0);
5143 5144 5145
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
5146 5147
	if (mask) {
		io_cqring_ev_posted(ctx);
5148
		io_put_req(req);
5149
	}
J
Jens Axboe 已提交
5150
	return ipt.error;
5151 5152
}

J
Jens Axboe 已提交
5153 5154
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
5155 5156 5157 5158
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
5159 5160 5161
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
5162 5163 5164
	atomic_set(&req->ctx->cq_timeouts,
		atomic_read(&req->ctx->cq_timeouts) + 1);

5165
	/*
5166 5167
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
5168
	 */
P
Pavel Begunkov 已提交
5169 5170
	if (!list_empty(&req->timeout.list))
		list_del_init(&req->timeout.list);
J
Jens Axboe 已提交
5171

5172
	io_cqring_fill_event(req, -ETIME);
J
Jens Axboe 已提交
5173 5174 5175 5176
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
5177
	req_set_fail_links(req);
J
Jens Axboe 已提交
5178 5179 5180 5181
	io_put_req(req);
	return HRTIMER_NORESTART;
}

5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198
static int __io_timeout_cancel(struct io_kiocb *req)
{
	int ret;

	list_del_init(&req->timeout.list);

	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
	if (ret == -1)
		return -EALREADY;

	req_set_fail_links(req);
	req->flags |= REQ_F_COMP_LOCKED;
	io_cqring_fill_event(req, -ECANCELED);
	io_put_req(req);
	return 0;
}

5199 5200 5201 5202 5203
static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
{
	struct io_kiocb *req;
	int ret = -ENOENT;

P
Pavel Begunkov 已提交
5204
	list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
5205 5206 5207 5208 5209 5210 5211 5212 5213
		if (user_data == req->user_data) {
			ret = 0;
			break;
		}
	}

	if (ret == -ENOENT)
		return ret;

5214
	return __io_timeout_cancel(req);
5215 5216
}

5217 5218
static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
5219 5220 5221
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
5222 5223 5224
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index || sqe->len)
5225 5226 5227 5228 5229 5230 5231 5232 5233 5234
		return -EINVAL;

	req->timeout.addr = READ_ONCE(sqe->addr);
	req->timeout.flags = READ_ONCE(sqe->timeout_flags);
	if (req->timeout.flags)
		return -EINVAL;

	return 0;
}

5235 5236 5237
/*
 * Remove or update an existing timeout command
 */
5238
static int io_timeout_remove(struct io_kiocb *req)
5239 5240
{
	struct io_ring_ctx *ctx = req->ctx;
5241
	int ret;
5242 5243

	spin_lock_irq(&ctx->completion_lock);
5244
	ret = io_timeout_cancel(ctx, req->timeout.addr);
5245

5246
	io_cqring_fill_event(req, ret);
5247 5248
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
5249
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
5250 5251
	if (ret < 0)
		req_set_fail_links(req);
5252
	io_put_req(req);
5253
	return 0;
J
Jens Axboe 已提交
5254 5255
}

5256
static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5257
			   bool is_timeout_link)
J
Jens Axboe 已提交
5258
{
5259
	struct io_timeout_data *data;
5260
	unsigned flags;
5261
	u32 off = READ_ONCE(sqe->off);
J
Jens Axboe 已提交
5262

5263
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
5264
		return -EINVAL;
5265
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
5266
		return -EINVAL;
5267
	if (off && is_timeout_link)
5268
		return -EINVAL;
5269 5270
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
5271
		return -EINVAL;
5272

5273
	req->timeout.off = off;
5274

5275
	if (!req->io && io_alloc_async_ctx(req))
5276 5277 5278
		return -ENOMEM;

	data = &req->io->timeout;
5279 5280 5281
	data->req = req;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
5282 5283
		return -EFAULT;

5284
	if (flags & IORING_TIMEOUT_ABS)
5285
		data->mode = HRTIMER_MODE_ABS;
5286
	else
5287
		data->mode = HRTIMER_MODE_REL;
5288

5289 5290 5291 5292
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}

5293
static int io_timeout(struct io_kiocb *req)
5294 5295
{
	struct io_ring_ctx *ctx = req->ctx;
5296
	struct io_timeout_data *data = &req->io->timeout;
5297
	struct list_head *entry;
5298
	u32 tail, off = req->timeout.off;
5299

5300
	spin_lock_irq(&ctx->completion_lock);
5301

J
Jens Axboe 已提交
5302 5303
	/*
	 * sqe->off holds how many events that need to occur for this
5304 5305
	 * timeout event to be satisfied. If it isn't set, then this is
	 * a pure timeout request, sequence isn't used.
J
Jens Axboe 已提交
5306
	 */
5307
	if (io_is_timeout_noseq(req)) {
5308 5309 5310
		entry = ctx->timeout_list.prev;
		goto add;
	}
J
Jens Axboe 已提交
5311

5312 5313
	tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
	req->timeout.target_seq = tail + off;
J
Jens Axboe 已提交
5314 5315 5316 5317 5318 5319

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	list_for_each_prev(entry, &ctx->timeout_list) {
P
Pavel Begunkov 已提交
5320 5321
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
						  timeout.list);
J
Jens Axboe 已提交
5322

5323
		if (io_is_timeout_noseq(nxt))
5324
			continue;
5325 5326
		/* nxt.seq is behind @tail, otherwise would've been completed */
		if (off >= nxt->timeout.target_seq - tail)
J
Jens Axboe 已提交
5327 5328
			break;
	}
5329
add:
P
Pavel Begunkov 已提交
5330
	list_add(&req->timeout.list, entry);
5331 5332
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
J
Jens Axboe 已提交
5333 5334 5335 5336
	spin_unlock_irq(&ctx->completion_lock);
	return 0;
}

5337 5338 5339 5340 5341 5342 5343
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

5344
static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
5345 5346 5347 5348
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;

5349
	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

5362 5363 5364
	return ret;
}

5365 5366
static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
5367
				     int success_ret)
5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
	if (ret != -ENOENT) {
		spin_lock_irqsave(&ctx->completion_lock, flags);
		goto done;
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr);
done:
5384 5385
	if (!ret)
		ret = success_ret;
5386 5387 5388 5389 5390
	io_cqring_fill_event(req, ret);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

J
Jens Axboe 已提交
5391 5392
	if (ret < 0)
		req_set_fail_links(req);
5393
	io_put_req(req);
5394 5395
}

5396 5397
static int io_async_cancel_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
5398
{
5399
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5400
		return -EINVAL;
5401 5402 5403
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
5404 5405
		return -EINVAL;

5406 5407 5408 5409
	req->cancel.addr = READ_ONCE(sqe->addr);
	return 0;
}

5410
static int io_async_cancel(struct io_kiocb *req)
5411 5412 5413
{
	struct io_ring_ctx *ctx = req->ctx;

5414
	io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
J
Jens Axboe 已提交
5415 5416 5417
	return 0;
}

5418 5419 5420
static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
5421 5422 5423
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->rw_flags)
5424 5425 5426 5427 5428 5429 5430 5431 5432 5433
		return -EINVAL;

	req->files_update.offset = READ_ONCE(sqe->off);
	req->files_update.nr_args = READ_ONCE(sqe->len);
	if (!req->files_update.nr_args)
		return -EINVAL;
	req->files_update.arg = READ_ONCE(sqe->addr);
	return 0;
}

5434 5435
static int io_files_update(struct io_kiocb *req, bool force_nonblock,
			   struct io_comp_state *cs)
5436 5437
{
	struct io_ring_ctx *ctx = req->ctx;
5438 5439
	struct io_uring_files_update up;
	int ret;
5440

5441
	if (force_nonblock)
5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
5453
	__io_req_complete(req, ret, 0, cs);
J
Jens Axboe 已提交
5454 5455 5456
	return 0;
}

5457 5458
static int io_req_defer_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
5459
{
J
Jens Axboe 已提交
5460
	ssize_t ret = 0;
5461

5462 5463 5464
	if (!sqe)
		return 0;

5465 5466
	if (io_alloc_async_ctx(req))
		return -EAGAIN;
5467 5468 5469
	ret = io_prep_work_files(req);
	if (unlikely(ret))
		return ret;
5470

5471 5472
	io_prep_async_work(req);

5473
	switch (req->opcode) {
J
Jens Axboe 已提交
5474 5475
	case IORING_OP_NOP:
		break;
5476 5477
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
5478
	case IORING_OP_READ:
5479
		ret = io_read_prep(req, sqe, true);
5480 5481 5482
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
5483
	case IORING_OP_WRITE:
5484
		ret = io_write_prep(req, sqe, true);
5485
		break;
5486
	case IORING_OP_POLL_ADD:
5487
		ret = io_poll_add_prep(req, sqe);
5488 5489
		break;
	case IORING_OP_POLL_REMOVE:
5490
		ret = io_poll_remove_prep(req, sqe);
5491
		break;
5492
	case IORING_OP_FSYNC:
5493
		ret = io_prep_fsync(req, sqe);
5494 5495
		break;
	case IORING_OP_SYNC_FILE_RANGE:
5496
		ret = io_prep_sfr(req, sqe);
5497
		break;
5498
	case IORING_OP_SENDMSG:
5499
	case IORING_OP_SEND:
5500
		ret = io_sendmsg_prep(req, sqe);
5501 5502
		break;
	case IORING_OP_RECVMSG:
5503
	case IORING_OP_RECV:
5504
		ret = io_recvmsg_prep(req, sqe);
5505
		break;
5506
	case IORING_OP_CONNECT:
5507
		ret = io_connect_prep(req, sqe);
5508
		break;
5509
	case IORING_OP_TIMEOUT:
5510
		ret = io_timeout_prep(req, sqe, false);
5511
		break;
5512
	case IORING_OP_TIMEOUT_REMOVE:
5513
		ret = io_timeout_remove_prep(req, sqe);
5514
		break;
5515
	case IORING_OP_ASYNC_CANCEL:
5516
		ret = io_async_cancel_prep(req, sqe);
5517
		break;
5518
	case IORING_OP_LINK_TIMEOUT:
5519
		ret = io_timeout_prep(req, sqe, true);
5520
		break;
5521
	case IORING_OP_ACCEPT:
5522
		ret = io_accept_prep(req, sqe);
5523
		break;
5524 5525 5526
	case IORING_OP_FALLOCATE:
		ret = io_fallocate_prep(req, sqe);
		break;
5527 5528 5529
	case IORING_OP_OPENAT:
		ret = io_openat_prep(req, sqe);
		break;
5530 5531 5532
	case IORING_OP_CLOSE:
		ret = io_close_prep(req, sqe);
		break;
5533 5534 5535
	case IORING_OP_FILES_UPDATE:
		ret = io_files_update_prep(req, sqe);
		break;
5536 5537 5538
	case IORING_OP_STATX:
		ret = io_statx_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5539 5540 5541
	case IORING_OP_FADVISE:
		ret = io_fadvise_prep(req, sqe);
		break;
J
Jens Axboe 已提交
5542 5543 5544
	case IORING_OP_MADVISE:
		ret = io_madvise_prep(req, sqe);
		break;
5545 5546 5547
	case IORING_OP_OPENAT2:
		ret = io_openat2_prep(req, sqe);
		break;
5548 5549 5550
	case IORING_OP_EPOLL_CTL:
		ret = io_epoll_ctl_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5551 5552 5553
	case IORING_OP_SPLICE:
		ret = io_splice_prep(req, sqe);
		break;
5554 5555 5556
	case IORING_OP_PROVIDE_BUFFERS:
		ret = io_provide_buffers_prep(req, sqe);
		break;
5557 5558 5559
	case IORING_OP_REMOVE_BUFFERS:
		ret = io_remove_buffers_prep(req, sqe);
		break;
P
Pavel Begunkov 已提交
5560 5561 5562
	case IORING_OP_TEE:
		ret = io_tee_prep(req, sqe);
		break;
5563
	default:
J
Jens Axboe 已提交
5564 5565 5566
		printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
				req->opcode);
		ret = -EINVAL;
5567
		break;
5568 5569
	}

5570
	return ret;
5571 5572
}

5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586
static u32 io_get_sequence(struct io_kiocb *req)
{
	struct io_kiocb *pos;
	struct io_ring_ctx *ctx = req->ctx;
	u32 total_submitted, nr_reqs = 1;

	if (req->flags & REQ_F_LINK_HEAD)
		list_for_each_entry(pos, &req->link_list, link_list)
			nr_reqs++;

	total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
	return total_submitted - nr_reqs;
}

5587
static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5588
{
5589
	struct io_ring_ctx *ctx = req->ctx;
5590
	struct io_defer_entry *de;
5591
	int ret;
5592
	u32 seq;
5593

B
Bob Liu 已提交
5594
	/* Still need defer if there is pending req in defer list. */
5595 5596 5597 5598 5599 5600 5601
	if (likely(list_empty_careful(&ctx->defer_list) &&
		!(req->flags & REQ_F_IO_DRAIN)))
		return 0;

	seq = io_get_sequence(req);
	/* Still a chance to pass the sequence check */
	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
5602 5603
		return 0;

5604 5605
	if (!req->io) {
		ret = io_req_defer_prep(req, sqe);
5606
		if (ret)
5607 5608
			return ret;
	}
5609
	io_prep_async_link(req);
5610 5611 5612
	de = kmalloc(sizeof(*de), GFP_KERNEL);
	if (!de)
		return -ENOMEM;
5613

5614
	spin_lock_irq(&ctx->completion_lock);
5615
	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
5616
		spin_unlock_irq(&ctx->completion_lock);
5617
		kfree(de);
5618 5619
		io_queue_async_work(req);
		return -EIOCBQUEUED;
5620 5621
	}

5622
	trace_io_uring_defer(ctx, req, req->user_data);
5623
	de->req = req;
5624
	de->seq = seq;
5625
	list_add_tail(&de->list, &ctx->defer_list);
5626 5627 5628 5629
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

5630
static void __io_clean_op(struct io_kiocb *req)
P
Pavel Begunkov 已提交
5631 5632 5633
{
	struct io_async_ctx *io = req->io;

5634 5635 5636 5637 5638
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
5639
			kfree((void *)(unsigned long)req->rw.addr);
5640 5641 5642
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_RECV:
5643
			kfree(req->sr_msg.kbuf);
5644 5645 5646
			break;
		}
		req->flags &= ~REQ_F_BUFFER_SELECTED;
P
Pavel Begunkov 已提交
5647 5648
	}

5649 5650 5651 5652 5653 5654 5655 5656
	if (req->flags & REQ_F_NEED_CLEANUP) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
		case IORING_OP_WRITE:
5657 5658
			if (io->rw.free_iovec)
				kfree(io->rw.free_iovec);
5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_SENDMSG:
			if (io->msg.iov != io->msg.fast_iov)
				kfree(io->msg.iov);
			break;
		case IORING_OP_SPLICE:
		case IORING_OP_TEE:
			io_put_file(req, req->splice.file_in,
				    (req->splice.flags & SPLICE_F_FD_IN_FIXED));
			break;
		}
		req->flags &= ~REQ_F_NEED_CLEANUP;
P
Pavel Begunkov 已提交
5672
	}
5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684

	if (req->flags & REQ_F_INFLIGHT) {
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
		req->flags &= ~REQ_F_INFLIGHT;
	}
P
Pavel Begunkov 已提交
5685 5686
}

5687
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5688
			bool force_nonblock, struct io_comp_state *cs)
J
Jens Axboe 已提交
5689
{
5690
	struct io_ring_ctx *ctx = req->ctx;
5691
	int ret;
J
Jens Axboe 已提交
5692

5693
	switch (req->opcode) {
J
Jens Axboe 已提交
5694
	case IORING_OP_NOP:
5695
		ret = io_nop(req, cs);
J
Jens Axboe 已提交
5696 5697
		break;
	case IORING_OP_READV:
5698
	case IORING_OP_READ_FIXED:
5699
	case IORING_OP_READ:
5700 5701 5702 5703 5704
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5705
		ret = io_read(req, force_nonblock, cs);
5706
		break;
5707
	case IORING_OP_WRITEV:
5708
	case IORING_OP_WRITE_FIXED:
5709
	case IORING_OP_WRITE:
5710 5711 5712 5713 5714
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
5715
		ret = io_write(req, force_nonblock, cs);
J
Jens Axboe 已提交
5716
		break;
C
Christoph Hellwig 已提交
5717
	case IORING_OP_FSYNC:
5718 5719 5720 5721 5722
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
5723
		ret = io_fsync(req, force_nonblock);
C
Christoph Hellwig 已提交
5724
		break;
5725
	case IORING_OP_POLL_ADD:
5726 5727 5728 5729 5730
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
5731
		ret = io_poll_add(req);
5732 5733
		break;
	case IORING_OP_POLL_REMOVE:
5734 5735 5736 5737 5738
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
5739
		ret = io_poll_remove(req);
5740
		break;
5741
	case IORING_OP_SYNC_FILE_RANGE:
5742 5743 5744 5745 5746
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
5747
		ret = io_sync_file_range(req, force_nonblock);
5748
		break;
J
Jens Axboe 已提交
5749
	case IORING_OP_SENDMSG:
5750
	case IORING_OP_SEND:
5751 5752 5753 5754 5755
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
5756
		if (req->opcode == IORING_OP_SENDMSG)
5757
			ret = io_sendmsg(req, force_nonblock, cs);
5758
		else
5759
			ret = io_send(req, force_nonblock, cs);
J
Jens Axboe 已提交
5760
		break;
J
Jens Axboe 已提交
5761
	case IORING_OP_RECVMSG:
5762
	case IORING_OP_RECV:
5763 5764 5765 5766 5767
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
5768
		if (req->opcode == IORING_OP_RECVMSG)
5769
			ret = io_recvmsg(req, force_nonblock, cs);
5770
		else
5771
			ret = io_recv(req, force_nonblock, cs);
J
Jens Axboe 已提交
5772
		break;
J
Jens Axboe 已提交
5773
	case IORING_OP_TIMEOUT:
5774 5775 5776 5777 5778
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
5779
		ret = io_timeout(req);
J
Jens Axboe 已提交
5780
		break;
5781
	case IORING_OP_TIMEOUT_REMOVE:
5782 5783 5784 5785 5786
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
5787
		ret = io_timeout_remove(req);
5788
		break;
5789
	case IORING_OP_ACCEPT:
5790 5791 5792 5793 5794
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
5795
		ret = io_accept(req, force_nonblock, cs);
5796
		break;
5797
	case IORING_OP_CONNECT:
5798 5799 5800 5801 5802
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
5803
		ret = io_connect(req, force_nonblock, cs);
5804
		break;
5805
	case IORING_OP_ASYNC_CANCEL:
5806 5807 5808 5809 5810
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
5811
		ret = io_async_cancel(req);
5812
		break;
5813 5814 5815 5816 5817 5818
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
5819
		ret = io_fallocate(req, force_nonblock);
5820
		break;
5821 5822 5823 5824 5825 5826
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
5827
		ret = io_openat(req, force_nonblock);
5828
		break;
5829 5830 5831 5832 5833 5834
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
5835
		ret = io_close(req, force_nonblock, cs);
5836
		break;
5837 5838 5839 5840 5841 5842
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
5843
		ret = io_files_update(req, force_nonblock, cs);
5844
		break;
5845 5846 5847 5848 5849 5850
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
5851
		ret = io_statx(req, force_nonblock);
5852
		break;
J
Jens Axboe 已提交
5853 5854 5855 5856 5857 5858
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
5859
		ret = io_fadvise(req, force_nonblock);
J
Jens Axboe 已提交
5860
		break;
J
Jens Axboe 已提交
5861 5862 5863 5864 5865 5866
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
5867
		ret = io_madvise(req, force_nonblock);
J
Jens Axboe 已提交
5868
		break;
5869 5870 5871 5872 5873 5874
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
5875
		ret = io_openat2(req, force_nonblock);
5876
		break;
5877 5878 5879 5880 5881 5882
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
5883
		ret = io_epoll_ctl(req, force_nonblock, cs);
5884
		break;
P
Pavel Begunkov 已提交
5885 5886 5887 5888 5889 5890
	case IORING_OP_SPLICE:
		if (sqe) {
			ret = io_splice_prep(req, sqe);
			if (ret < 0)
				break;
		}
5891
		ret = io_splice(req, force_nonblock);
P
Pavel Begunkov 已提交
5892
		break;
5893 5894 5895 5896 5897 5898
	case IORING_OP_PROVIDE_BUFFERS:
		if (sqe) {
			ret = io_provide_buffers_prep(req, sqe);
			if (ret)
				break;
		}
5899
		ret = io_provide_buffers(req, force_nonblock, cs);
5900
		break;
5901 5902 5903 5904 5905 5906
	case IORING_OP_REMOVE_BUFFERS:
		if (sqe) {
			ret = io_remove_buffers_prep(req, sqe);
			if (ret)
				break;
		}
5907
		ret = io_remove_buffers(req, force_nonblock, cs);
5908
		break;
P
Pavel Begunkov 已提交
5909 5910 5911 5912 5913 5914 5915 5916
	case IORING_OP_TEE:
		if (sqe) {
			ret = io_tee_prep(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_tee(req, force_nonblock);
		break;
J
Jens Axboe 已提交
5917 5918 5919 5920 5921
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
5922 5923 5924
	if (ret)
		return ret;

5925 5926
	/* If the op doesn't have a file, we're not polling for it */
	if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
5927 5928 5929 5930 5931 5932
		const bool in_async = io_wq_current_is_worker();

		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

J
Jens Axboe 已提交
5933
		io_iopoll_req_issued(req);
5934 5935 5936

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
5937 5938 5939
	}

	return 0;
J
Jens Axboe 已提交
5940 5941
}

5942
static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work)
J
Jens Axboe 已提交
5943 5944
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5945
	struct io_kiocb *timeout;
5946
	int ret = 0;
J
Jens Axboe 已提交
5947

5948 5949 5950
	timeout = io_prep_linked_timeout(req);
	if (timeout)
		io_queue_linked_timeout(timeout);
5951

5952 5953 5954
	/* if NO_CANCEL is set, we must still run the work */
	if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
				IO_WQ_WORK_CANCEL) {
5955
		ret = -ECANCELED;
5956
	}
5957

5958 5959
	if (!ret) {
		do {
5960
			ret = io_issue_sqe(req, NULL, false, NULL);
5961 5962 5963 5964 5965 5966 5967 5968 5969 5970
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
5971

5972
	if (ret) {
J
Jens Axboe 已提交
5973
		req_set_fail_links(req);
5974
		io_req_complete(req, ret);
5975
	}
J
Jens Axboe 已提交
5976

5977
	return io_steal_work(req);
J
Jens Axboe 已提交
5978 5979
}

5980 5981 5982 5983 5984
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

5985
	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
5986
	return table->files[index & IORING_FILE_TABLE_MASK];
5987 5988
}

5989 5990
static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
			int fd, struct file **out_file, bool fixed)
J
Jens Axboe 已提交
5991
{
5992
	struct io_ring_ctx *ctx = req->ctx;
5993
	struct file *file;
J
Jens Axboe 已提交
5994

5995
	if (fixed) {
5996
		if (unlikely(!ctx->file_data ||
J
Jens Axboe 已提交
5997 5998
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
5999
		fd = array_index_nospec(fd, ctx->nr_user_files);
6000
		file = io_file_from_index(ctx, fd);
6001 6002 6003 6004
		if (file) {
			req->fixed_file_refs = ctx->file_data->cur_refs;
			percpu_ref_get(req->fixed_file_refs);
		}
J
Jens Axboe 已提交
6005
	} else {
6006
		trace_io_uring_file_get(ctx, fd);
6007
		file = __io_file_get(state, fd);
J
Jens Axboe 已提交
6008 6009
	}

6010 6011 6012 6013 6014
	if (file || io_op_defs[req->opcode].needs_file_no_error) {
		*out_file = file;
		return 0;
	}
	return -EBADF;
J
Jens Axboe 已提交
6015 6016
}

6017
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
6018
			   int fd)
6019 6020 6021
{
	bool fixed;

6022
	fixed = (req->flags & REQ_F_FIXED_FILE) != 0;
6023
	if (unlikely(!fixed && io_async_submit(req->ctx)))
6024 6025 6026 6027 6028
		return -EBADF;

	return io_file_get(state, req, fd, &req->file, fixed);
}

6029
static int io_grab_files(struct io_kiocb *req)
6030 6031
{
	int ret = -EBADF;
6032
	struct io_ring_ctx *ctx = req->ctx;
6033

6034 6035
	io_req_init_async(req);

6036
	if (req->work.files || (req->flags & REQ_F_NO_FILE_TABLE))
6037
		return 0;
P
Pavel Begunkov 已提交
6038
	if (!ctx->ring_file)
6039 6040
		return -EBADF;

6041 6042 6043 6044 6045 6046 6047 6048
	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
P
Pavel Begunkov 已提交
6049
	if (fcheck(ctx->ring_fd) == ctx->ring_file) {
6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

6061 6062 6063 6064 6065 6066 6067
static inline int io_prep_work_files(struct io_kiocb *req)
{
	if (!io_op_defs[req->opcode].file_table)
		return 0;
	return io_grab_files(req);
}

6068
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
J
Jens Axboe 已提交
6069
{
6070 6071 6072
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
6073 6074 6075 6076 6077 6078 6079 6080 6081 6082
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
6083 6084 6085
	if (!list_empty(&req->link_list)) {
		prev = list_entry(req->link_list.prev, struct io_kiocb,
				  link_list);
6086
		if (refcount_inc_not_zero(&prev->refs)) {
6087
			list_del_init(&req->link_list);
6088 6089
			prev->flags &= ~REQ_F_LINK_TIMEOUT;
		} else
6090
			prev = NULL;
6091 6092 6093 6094 6095
	}

	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
J
Jens Axboe 已提交
6096
		req_set_fail_links(prev);
6097
		io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
6098
		io_put_req(prev);
6099
	} else {
6100
		io_req_complete(req, -ETIME);
6101 6102 6103 6104
	}
	return HRTIMER_NORESTART;
}

6105
static void __io_queue_linked_timeout(struct io_kiocb *req)
6106
{
6107 6108 6109 6110
	/*
	 * If the list is now empty, then our linked request finished before
	 * we got a chance to setup the timer
	 */
6111
	if (!list_empty(&req->link_list)) {
6112
		struct io_timeout_data *data = &req->io->timeout;
6113

6114 6115 6116
		data->timer.function = io_link_timeout_fn;
		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
				data->mode);
6117
	}
6118 6119 6120 6121 6122 6123 6124 6125
}

static void io_queue_linked_timeout(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	__io_queue_linked_timeout(req);
6126
	spin_unlock_irq(&ctx->completion_lock);
6127 6128

	/* drop submission reference */
6129 6130
	io_put_req(req);
}
6131

6132
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
6133 6134 6135
{
	struct io_kiocb *nxt;

6136
	if (!(req->flags & REQ_F_LINK_HEAD))
6137
		return NULL;
6138
	if (req->flags & REQ_F_LINK_TIMEOUT)
6139
		return NULL;
6140

6141 6142
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
					link_list);
6143
	if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
6144
		return NULL;
6145

6146 6147
	req->flags |= REQ_F_LINK_TIMEOUT;
	return nxt;
6148 6149
}

6150 6151
static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			   struct io_comp_state *cs)
J
Jens Axboe 已提交
6152
{
6153
	struct io_kiocb *linked_timeout;
6154
	struct io_kiocb *nxt;
6155
	const struct cred *old_creds = NULL;
6156
	int ret;
J
Jens Axboe 已提交
6157

6158 6159 6160
again:
	linked_timeout = io_prep_linked_timeout(req);

6161 6162
	if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
	    req->work.creds != current_cred()) {
6163 6164 6165 6166 6167 6168 6169 6170
		if (old_creds)
			revert_creds(old_creds);
		if (old_creds == req->work.creds)
			old_creds = NULL; /* restored original creds */
		else
			old_creds = override_creds(req->work.creds);
	}

6171
	ret = io_issue_sqe(req, sqe, true, cs);
6172 6173 6174 6175 6176

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
6177
	if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6178
		if (!io_arm_poll_handler(req)) {
6179
punt:
6180 6181
			ret = io_prep_work_files(req);
			if (unlikely(ret))
6182
				goto err;
6183 6184 6185 6186 6187
			/*
			 * Queued up for async execution, worker will release
			 * submit reference when the iocb is actually submitted.
			 */
			io_queue_async_work(req);
J
Jens Axboe 已提交
6188
		}
6189

6190 6191
		if (linked_timeout)
			io_queue_linked_timeout(linked_timeout);
6192
		goto exit;
J
Jens Axboe 已提交
6193
	}
6194

6195
	if (unlikely(ret)) {
6196
err:
6197 6198
		/* un-prep timeout, so it'll be killed as any other linked */
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
6199
		req_set_fail_links(req);
6200
		io_put_req(req);
6201
		io_req_complete(req, ret);
6202
		goto exit;
J
Jens Axboe 已提交
6203
	}
6204 6205 6206 6207 6208 6209

	/* drop submission reference */
	nxt = io_put_req_find_next(req);
	if (linked_timeout)
		io_queue_linked_timeout(linked_timeout);

6210 6211
	if (nxt) {
		req = nxt;
6212 6213 6214

		if (req->flags & REQ_F_FORCE_ASYNC)
			goto punt;
6215 6216
		goto again;
	}
6217
exit:
6218 6219
	if (old_creds)
		revert_creds(old_creds);
J
Jens Axboe 已提交
6220 6221
}

6222 6223
static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 struct io_comp_state *cs)
6224 6225 6226
{
	int ret;

6227
	ret = io_req_defer(req, sqe);
6228 6229
	if (ret) {
		if (ret != -EIOCBQUEUED) {
6230
fail_req:
J
Jens Axboe 已提交
6231
			req_set_fail_links(req);
6232 6233
			io_put_req(req);
			io_req_complete(req, ret);
6234
		}
6235
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
6236 6237
		if (!req->io) {
			ret = io_req_defer_prep(req, sqe);
6238
			if (unlikely(ret))
6239 6240 6241
				goto fail_req;
		}

J
Jens Axboe 已提交
6242 6243 6244 6245
		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
6246
		io_req_init_async(req);
J
Jens Axboe 已提交
6247 6248 6249
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
6250
		__io_queue_sqe(req, sqe, cs);
J
Jens Axboe 已提交
6251
	}
6252 6253
}

6254 6255
static inline void io_queue_link_head(struct io_kiocb *req,
				      struct io_comp_state *cs)
6256
{
6257
	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
6258 6259
		io_put_req(req);
		io_req_complete(req, -ECANCELED);
6260
	} else
6261
		io_queue_sqe(req, NULL, cs);
6262 6263
}

6264
static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
6265
			 struct io_kiocb **link, struct io_comp_state *cs)
J
Jens Axboe 已提交
6266
{
6267
	struct io_ring_ctx *ctx = req->ctx;
6268
	int ret;
J
Jens Axboe 已提交
6269 6270 6271 6272 6273 6274 6275 6276 6277

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
P
Pavel Begunkov 已提交
6278
		struct io_kiocb *head = *link;
J
Jens Axboe 已提交
6279

6280 6281 6282 6283 6284 6285 6286
		/*
		 * Taking sequential execution of a link, draining both sides
		 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
		 * requests in the link. So, it drains the head and the
		 * next after the link request. The last one is done via
		 * drain_next flag to persist the effect across calls.
		 */
6287
		if (req->flags & REQ_F_IO_DRAIN) {
6288 6289 6290
			head->flags |= REQ_F_IO_DRAIN;
			ctx->drain_next = 1;
		}
6291
		ret = io_req_defer_prep(req, sqe);
6292
		if (unlikely(ret)) {
J
Jens Axboe 已提交
6293
			/* fail even hard links since we don't submit */
P
Pavel Begunkov 已提交
6294
			head->flags |= REQ_F_FAIL_LINK;
6295
			return ret;
6296
		}
P
Pavel Begunkov 已提交
6297
		trace_io_uring_link(ctx, req, head);
6298
		io_get_req_task(req);
P
Pavel Begunkov 已提交
6299
		list_add_tail(&req->link_list, &head->link_list);
6300 6301

		/* last request of a link, enqueue the link */
6302
		if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
6303
			io_queue_link_head(head, cs);
6304 6305
			*link = NULL;
		}
J
Jens Axboe 已提交
6306
	} else {
6307 6308
		if (unlikely(ctx->drain_next)) {
			req->flags |= REQ_F_IO_DRAIN;
6309
			ctx->drain_next = 0;
6310
		}
6311
		if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
6312
			req->flags |= REQ_F_LINK_HEAD;
6313
			INIT_LIST_HEAD(&req->link_list);
6314

6315
			ret = io_req_defer_prep(req, sqe);
6316
			if (unlikely(ret))
6317 6318 6319
				req->flags |= REQ_F_FAIL_LINK;
			*link = req;
		} else {
6320
			io_queue_sqe(req, sqe, cs);
6321
		}
J
Jens Axboe 已提交
6322
	}
6323

6324
	return 0;
J
Jens Axboe 已提交
6325 6326
}

6327 6328 6329 6330 6331
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
6332 6333
	if (!list_empty(&state->comp.list))
		io_submit_flush_completions(&state->comp);
6334
	blk_finish_plug(&state->plug);
P
Pavel Begunkov 已提交
6335
	io_state_file_put(state);
J
Jens Axboe 已提交
6336
	if (state->free_reqs)
6337
		kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
6338 6339 6340 6341 6342 6343
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
6344
				  struct io_ring_ctx *ctx, unsigned int max_ios)
6345 6346
{
	blk_start_plug(&state->plug);
6347 6348 6349
#ifdef CONFIG_BLOCK
	state->plug.nowait = true;
#endif
6350 6351 6352
	state->comp.nr = 0;
	INIT_LIST_HEAD(&state->comp.list);
	state->comp.ctx = ctx;
J
Jens Axboe 已提交
6353
	state->free_reqs = 0;
6354 6355 6356 6357
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
6358 6359
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
6360
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
6361

6362 6363 6364 6365 6366 6367
	/*
	 * Ensure any loads from the SQEs are done at this point,
	 * since once we write the new head, the application could
	 * write new data to them.
	 */
	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
6368 6369 6370
}

/*
6371
 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
J
Jens Axboe 已提交
6372 6373 6374 6375 6376 6377
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
6378
static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
6379
{
6380
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
6381 6382 6383 6384 6385 6386 6387 6388 6389 6390
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
6391
	head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
6392 6393
	if (likely(head < ctx->sq_entries))
		return &ctx->sq_sqes[head];
J
Jens Axboe 已提交
6394 6395

	/* drop invalid entries */
6396
	ctx->cached_sq_dropped++;
6397
	WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
6398 6399 6400 6401 6402 6403
	return NULL;
}

static inline void io_consume_sqe(struct io_ring_ctx *ctx)
{
	ctx->cached_sq_head++;
J
Jens Axboe 已提交
6404 6405
}

6406 6407 6408 6409 6410 6411
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK|	\
				IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
				IOSQE_BUFFER_SELECT)

static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
		       const struct io_uring_sqe *sqe,
6412
		       struct io_submit_state *state)
6413
{
6414
	unsigned int sqe_flags;
6415
	int id;
6416

6417 6418 6419 6420 6421 6422 6423 6424
	req->opcode = READ_ONCE(sqe->opcode);
	req->user_data = READ_ONCE(sqe->user_data);
	req->io = NULL;
	req->file = NULL;
	req->ctx = ctx;
	req->flags = 0;
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
P
Pavel Begunkov 已提交
6425
	req->task = current;
6426
	req->result = 0;
6427 6428 6429 6430

	if (unlikely(req->opcode >= IORING_OP_LAST))
		return -EINVAL;

6431 6432
	if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
		return -EFAULT;
6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444

	sqe_flags = READ_ONCE(sqe->flags);
	/* enforce forwards compatibility on users */
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
		return -EINVAL;

	if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
	    !io_op_defs[req->opcode].buffer_select)
		return -EOPNOTSUPP;

	id = READ_ONCE(sqe->personality);
	if (id) {
6445
		io_req_init_async(req);
6446 6447 6448 6449 6450 6451 6452
		req->work.creds = idr_find(&ctx->personality_idr, id);
		if (unlikely(!req->work.creds))
			return -EINVAL;
		get_cred(req->work.creds);
	}

	/* same numerical values with corresponding REQ_F_*, safe to copy */
6453
	req->flags |= sqe_flags;
6454

6455 6456 6457 6458
	if (!io_op_defs[req->opcode].needs_file)
		return 0;

	return io_req_set_file(state, req, READ_ONCE(sqe->fd));
6459 6460
}

6461
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
6462
			  struct file *ring_file, int ring_fd)
J
Jens Axboe 已提交
6463
{
6464
	struct io_submit_state state;
J
Jens Axboe 已提交
6465 6466
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
J
Jens Axboe 已提交
6467

6468
	/* if we have a backlog and couldn't flush it all, return BUSY */
6469 6470 6471 6472 6473
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}
J
Jens Axboe 已提交
6474

6475 6476
	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
6477

6478 6479
	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;
J
Jens Axboe 已提交
6480

6481
	io_submit_state_start(&state, ctx, nr);
J
Jens Axboe 已提交
6482

P
Pavel Begunkov 已提交
6483 6484 6485
	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

J
Jens Axboe 已提交
6486
	for (i = 0; i < nr; i++) {
6487
		const struct io_uring_sqe *sqe;
6488
		struct io_kiocb *req;
6489
		int err;
6490

6491 6492 6493 6494 6495
		sqe = io_get_sqe(ctx);
		if (unlikely(!sqe)) {
			io_consume_sqe(ctx);
			break;
		}
6496
		req = io_alloc_req(ctx, &state);
6497 6498 6499
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
6500
			break;
6501
		}
6502

6503
		err = io_init_req(ctx, req, sqe, &state);
6504
		io_consume_sqe(ctx);
6505 6506 6507
		/* will complete beyond this point, count as submitted */
		submitted++;

6508
		if (unlikely(err)) {
6509
fail_req:
6510 6511
			io_put_req(req);
			io_req_complete(req, err);
6512 6513
			break;
		}
6514

6515
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6516
						true, io_async_submit(ctx));
6517
		err = io_submit_sqe(req, sqe, &link, &state.comp);
6518 6519
		if (err)
			goto fail_req;
J
Jens Axboe 已提交
6520 6521
	}

6522 6523 6524 6525 6526
	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
J
Jens Axboe 已提交
6527
	if (link)
6528
		io_queue_link_head(link, &state.comp);
6529
	io_submit_state_end(&state);
J
Jens Axboe 已提交
6530

6531 6532 6533
	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

J
Jens Axboe 已提交
6534 6535 6536
	return submitted;
}

6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551
static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
{
	/* Tell userspace we may need a wakeup call */
	spin_lock_irq(&ctx->completion_lock);
	ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
	spin_unlock_irq(&ctx->completion_lock);
}

static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
	spin_unlock_irq(&ctx->completion_lock);
}

J
Jens Axboe 已提交
6552 6553 6554
static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
6555
	const struct cred *old_cred;
J
Jens Axboe 已提交
6556 6557
	DEFINE_WAIT(wait);
	unsigned long timeout;
6558
	int ret = 0;
J
Jens Axboe 已提交
6559

6560
	complete(&ctx->sq_thread_comp);
6561

6562
	old_cred = override_creds(ctx->creds);
J
Jens Axboe 已提交
6563

6564
	timeout = jiffies + ctx->sq_thread_idle;
6565
	while (!kthread_should_park()) {
6566
		unsigned int to_submit;
J
Jens Axboe 已提交
6567

6568
		if (!list_empty(&ctx->iopoll_list)) {
J
Jens Axboe 已提交
6569 6570
			unsigned nr_events = 0;

6571
			mutex_lock(&ctx->uring_lock);
6572
			if (!list_empty(&ctx->iopoll_list) && !need_resched())
6573
				io_do_iopoll(ctx, &nr_events, 0);
6574
			else
J
Jens Axboe 已提交
6575
				timeout = jiffies + ctx->sq_thread_idle;
6576
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
6577 6578
		}

6579
		to_submit = io_sqring_entries(ctx);
6580 6581 6582 6583 6584

		/*
		 * If submit got -EBUSY, flag us as needing the application
		 * to enter the kernel to reap and flush events.
		 */
6585
		if (!to_submit || ret == -EBUSY || need_resched()) {
6586 6587 6588 6589 6590 6591
			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
6592
			io_sq_thread_drop_mm();
6593

J
Jens Axboe 已提交
6594 6595 6596
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
6597 6598 6599
			 * to sleep. The exception is if we got EBUSY doing
			 * more IO, we should wait for the application to
			 * reap events and wake us up.
J
Jens Axboe 已提交
6600
			 */
6601
			if (!list_empty(&ctx->iopoll_list) || need_resched() ||
6602 6603
			    (!time_after(jiffies, timeout) && ret != -EBUSY &&
			    !percpu_ref_is_dying(&ctx->refs))) {
6604
				io_run_task_work();
6605
				cond_resched();
J
Jens Axboe 已提交
6606 6607 6608 6609 6610 6611
				continue;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);

6612 6613
			/*
			 * While doing polled IO, before going to sleep, we need
6614 6615 6616 6617
			 * to check if there are new reqs added to iopoll_list,
			 * it is because reqs may have been punted to io worker
			 * and will be added to iopoll_list later, hence check
			 * the iopoll_list again.
6618 6619
			 */
			if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6620
			    !list_empty_careful(&ctx->iopoll_list)) {
6621 6622 6623 6624
				finish_wait(&ctx->sqo_wait, &wait);
				continue;
			}

6625
			io_ring_set_wakeup_flag(ctx);
J
Jens Axboe 已提交
6626

6627
			to_submit = io_sqring_entries(ctx);
6628
			if (!to_submit || ret == -EBUSY) {
6629
				if (kthread_should_park()) {
J
Jens Axboe 已提交
6630 6631 6632
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
6633
				if (io_run_task_work()) {
6634
					finish_wait(&ctx->sqo_wait, &wait);
6635
					io_ring_clear_wakeup_flag(ctx);
6636 6637
					continue;
				}
J
Jens Axboe 已提交
6638 6639 6640 6641 6642
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

6643
				io_ring_clear_wakeup_flag(ctx);
6644
				ret = 0;
J
Jens Axboe 已提交
6645 6646 6647 6648
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

6649
			io_ring_clear_wakeup_flag(ctx);
J
Jens Axboe 已提交
6650 6651
		}

6652
		mutex_lock(&ctx->uring_lock);
6653 6654
		if (likely(!percpu_ref_is_dying(&ctx->refs)))
			ret = io_submit_sqes(ctx, to_submit, NULL, -1);
6655
		mutex_unlock(&ctx->uring_lock);
6656
		timeout = jiffies + ctx->sq_thread_idle;
J
Jens Axboe 已提交
6657 6658
	}

6659
	io_run_task_work();
6660

6661
	io_sq_thread_drop_mm();
6662
	revert_creds(old_cred);
6663

6664
	kthread_parkme();
6665

J
Jens Axboe 已提交
6666 6667 6668
	return 0;
}

6669 6670 6671 6672 6673 6674 6675
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

6676
static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
6677 6678 6679 6680
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
6681
	 * Wake up if we have enough events, or if a timeout occurred since we
6682 6683 6684
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
6685
	return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
6686 6687 6688 6689 6690 6691 6692 6693 6694
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

6695 6696
	/* use noflush == true, as we can't safely rely on locking context */
	if (!io_should_wake(iowq, true))
6697 6698 6699 6700 6701
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
6702 6703 6704 6705 6706 6707 6708
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
6709 6710 6711 6712 6713 6714 6715 6716 6717
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
6718
	struct io_rings *rings = ctx->rings;
6719
	int ret = 0;
J
Jens Axboe 已提交
6720

6721 6722 6723
	do {
		if (io_cqring_events(ctx, false) >= min_events)
			return 0;
6724
		if (!io_run_task_work())
6725 6726
			break;
	} while (1);
J
Jens Axboe 已提交
6727 6728

	if (sig) {
6729 6730 6731
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
6732
						      sigsz);
6733 6734
		else
#endif
6735
			ret = set_user_sigmask(sig, sigsz);
6736

J
Jens Axboe 已提交
6737 6738 6739 6740
		if (ret)
			return ret;
	}

6741
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
6742
	trace_io_uring_cqring_wait(ctx, min_events);
6743 6744 6745
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
6746
		/* make sure we run task_work before checking for signals */
6747 6748
		if (io_run_task_work())
			continue;
6749
		if (signal_pending(current)) {
6750 6751 6752 6753 6754 6755 6756 6757
			if (current->jobctl & JOBCTL_TASK_WORK) {
				spin_lock_irq(&current->sighand->siglock);
				current->jobctl &= ~JOBCTL_TASK_WORK;
				recalc_sigpending();
				spin_unlock_irq(&current->sighand->siglock);
				continue;
			}
			ret = -EINTR;
6758 6759
			break;
		}
6760 6761 6762
		if (io_should_wake(&iowq, false))
			break;
		schedule();
6763 6764 6765
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

6766
	restore_saved_sigmask_unless(ret == -EINTR);
J
Jens Axboe 已提交
6767

6768
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
6769 6770
}

J
Jens Axboe 已提交
6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

6784 6785 6786 6787 6788 6789 6790
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
6791 6792 6793
#endif
}

6794 6795 6796 6797 6798 6799 6800 6801
static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}

J
Jens Axboe 已提交
6802 6803
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
6804
	struct fixed_file_data *data = ctx->file_data;
6805
	struct fixed_file_ref_node *ref_node = NULL;
6806 6807
	unsigned nr_tables, i;

6808
	if (!data)
J
Jens Axboe 已提交
6809 6810
		return -ENXIO;

6811
	spin_lock(&data->lock);
6812 6813 6814
	if (!list_empty(&data->ref_list))
		ref_node = list_first_entry(&data->ref_list,
				struct fixed_file_ref_node, node);
6815
	spin_unlock(&data->lock);
6816 6817 6818 6819 6820 6821
	if (ref_node)
		percpu_ref_kill(&ref_node->refs);

	percpu_ref_kill(&data->refs);

	/* wait for all refs nodes to complete */
6822
	flush_delayed_work(&ctx->file_put_work);
6823
	wait_for_completion(&data->done);
6824

J
Jens Axboe 已提交
6825
	__io_sqe_files_unregister(ctx);
6826 6827
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
6828 6829
		kfree(data->table[i].files);
	kfree(data->table);
6830 6831
	percpu_ref_exit(&data->refs);
	kfree(data);
6832
	ctx->file_data = NULL;
J
Jens Axboe 已提交
6833 6834 6835 6836
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
6837 6838 6839
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
6840
		wait_for_completion(&ctx->sq_thread_comp);
6841 6842 6843 6844 6845
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
6846
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
6847 6848 6849 6850 6851
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
6852 6853
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
6854 6855
	io_sq_thread_stop(ctx);

6856 6857 6858
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872
	}
}

#if defined(CONFIG_UNIX)
/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
6873
	int i, nr_files;
J
Jens Axboe 已提交
6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

6887
	nr_files = 0;
J
Jens Axboe 已提交
6888 6889
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
6890 6891 6892
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
6893
			continue;
6894
		fpl->fp[nr_files] = get_file(file);
6895 6896
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
6897 6898
	}

6899 6900 6901 6902
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
6903
		skb->destructor = unix_destruct_scm;
6904 6905
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
6906

6907 6908 6909 6910 6911 6912
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
6943 6944 6945 6946
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

6959 6960 6961 6962 6963 6964
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
6965
		struct fixed_file_table *table = &ctx->file_data->table[i];
6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
6980
		struct fixed_file_table *table = &ctx->file_data->table[i];
6981 6982 6983 6984 6985
		kfree(table->files);
	}
	return 1;
}

6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048
static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
	fput(file);
#endif
}

struct io_file_put {
7049
	struct list_head list;
7050 7051 7052
	struct file *file;
};

7053
static void __io_file_put_work(struct fixed_file_ref_node *ref_node)
7054
{
7055 7056
	struct fixed_file_data *file_data = ref_node->file_data;
	struct io_ring_ctx *ctx = file_data->ctx;
7057
	struct io_file_put *pfile, *tmp;
7058 7059

	list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) {
7060
		list_del(&pfile->list);
7061 7062
		io_ring_file_put(ctx, pfile->file);
		kfree(pfile);
7063
	}
7064

7065 7066 7067
	spin_lock(&file_data->lock);
	list_del(&ref_node->node);
	spin_unlock(&file_data->lock);
7068 7069 7070 7071

	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
	percpu_ref_put(&file_data->refs);
7072
}
7073

7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091
static void io_file_put_work(struct work_struct *work)
{
	struct io_ring_ctx *ctx;
	struct llist_node *node;

	ctx = container_of(work, struct io_ring_ctx, file_put_work.work);
	node = llist_del_all(&ctx->file_put_llist);

	while (node) {
		struct fixed_file_ref_node *ref_node;
		struct llist_node *next = node->next;

		ref_node = llist_entry(node, struct fixed_file_ref_node, llist);
		__io_file_put_work(ref_node);
		node = next;
	}
}

7092
static void io_file_data_ref_zero(struct percpu_ref *ref)
7093
{
7094
	struct fixed_file_ref_node *ref_node;
7095 7096 7097
	struct io_ring_ctx *ctx;
	bool first_add;
	int delay = HZ;
7098

7099
	ref_node = container_of(ref, struct fixed_file_ref_node, refs);
7100
	ctx = ref_node->file_data->ctx;
7101

7102 7103
	if (percpu_ref_is_dying(&ctx->file_data->refs))
		delay = 0;
7104

7105 7106 7107 7108 7109
	first_add = llist_add(&ref_node->llist, &ctx->file_put_llist);
	if (!delay)
		mod_delayed_work(system_wq, &ctx->file_put_work, 0);
	else if (first_add)
		queue_delayed_work(system_wq, &ctx->file_put_work, delay);
7110
}
7111

7112 7113
static struct fixed_file_ref_node *alloc_fixed_file_ref_node(
			struct io_ring_ctx *ctx)
7114
{
7115
	struct fixed_file_ref_node *ref_node;
7116

7117 7118 7119
	ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
	if (!ref_node)
		return ERR_PTR(-ENOMEM);
7120

7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135
	if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero,
			    0, GFP_KERNEL)) {
		kfree(ref_node);
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&ref_node->node);
	INIT_LIST_HEAD(&ref_node->file_list);
	ref_node->file_data = ctx->file_data;
	return ref_node;
}

static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node)
{
	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
7136 7137
}

J
Jens Axboe 已提交
7138 7139 7140 7141
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
7142
	unsigned nr_tables;
7143
	struct file *file;
J
Jens Axboe 已提交
7144 7145
	int fd, ret = 0;
	unsigned i;
7146
	struct fixed_file_ref_node *ref_node;
J
Jens Axboe 已提交
7147

7148
	if (ctx->file_data)
J
Jens Axboe 已提交
7149 7150 7151 7152 7153 7154
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

7155 7156 7157 7158 7159
	ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
	if (!ctx->file_data)
		return -ENOMEM;
	ctx->file_data->ctx = ctx;
	init_completion(&ctx->file_data->done);
7160
	INIT_LIST_HEAD(&ctx->file_data->ref_list);
7161
	spin_lock_init(&ctx->file_data->lock);
7162

7163
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
7164 7165
	ctx->file_data->table = kcalloc(nr_tables,
					sizeof(struct fixed_file_table),
7166
					GFP_KERNEL);
7167 7168 7169
	if (!ctx->file_data->table) {
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7170
		return -ENOMEM;
7171 7172
	}

7173
	if (percpu_ref_init(&ctx->file_data->refs, io_file_ref_kill,
7174 7175 7176 7177
				PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7178
		return -ENOMEM;
7179
	}
J
Jens Axboe 已提交
7180

7181
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
7182 7183 7184 7185
		percpu_ref_exit(&ctx->file_data->refs);
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
7186 7187 7188
		return -ENOMEM;
	}

7189
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
7190 7191 7192
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
7193 7194 7195
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
7196 7197 7198 7199 7200
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
7201

7202
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7203
		index = i & IORING_FILE_TABLE_MASK;
7204
		file = fget(fd);
J
Jens Axboe 已提交
7205 7206

		ret = -EBADF;
7207
		if (!file)
J
Jens Axboe 已提交
7208
			break;
7209

J
Jens Axboe 已提交
7210 7211 7212 7213 7214 7215 7216
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
7217 7218
		if (file->f_op == &io_uring_fops) {
			fput(file);
J
Jens Axboe 已提交
7219 7220 7221
			break;
		}
		ret = 0;
7222
		table->files[index] = file;
J
Jens Axboe 已提交
7223 7224 7225
	}

	if (ret) {
7226 7227 7228 7229 7230 7231
		for (i = 0; i < ctx->nr_user_files; i++) {
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
7232
			kfree(ctx->file_data->table[i].files);
J
Jens Axboe 已提交
7233

7234
		percpu_ref_exit(&ctx->file_data->refs);
7235 7236 7237
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
7238 7239 7240 7241 7242
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
7243
	if (ret) {
J
Jens Axboe 已提交
7244
		io_sqe_files_unregister(ctx);
7245 7246
		return ret;
	}
J
Jens Axboe 已提交
7247

7248 7249 7250 7251 7252 7253 7254
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node)) {
		io_sqe_files_unregister(ctx);
		return PTR_ERR(ref_node);
	}

	ctx->file_data->cur_refs = &ref_node->refs;
7255
	spin_lock(&ctx->file_data->lock);
7256
	list_add(&ref_node->node, &ctx->file_data->ref_list);
7257
	spin_unlock(&ctx->file_data->lock);
7258
	percpu_ref_get(&ctx->file_data->refs);
J
Jens Axboe 已提交
7259 7260 7261
	return ret;
}

7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304
static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

7305
static int io_queue_file_removal(struct fixed_file_data *data,
7306
				 struct file *file)
7307
{
7308
	struct io_file_put *pfile;
7309 7310
	struct percpu_ref *refs = data->cur_refs;
	struct fixed_file_ref_node *ref_node;
7311 7312

	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
7313 7314
	if (!pfile)
		return -ENOMEM;
7315

7316
	ref_node = container_of(refs, struct fixed_file_ref_node, refs);
7317
	pfile->file = file;
7318 7319
	list_add(&pfile->list, &ref_node->file_list);

7320
	return 0;
7321 7322 7323 7324 7325 7326 7327
}

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *up,
				 unsigned nr_args)
{
	struct fixed_file_data *data = ctx->file_data;
7328
	struct fixed_file_ref_node *ref_node;
7329
	struct file *file;
7330 7331 7332
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;
7333
	bool needs_switch = false;
7334

7335
	if (check_add_overflow(up->offset, nr_args, &done))
7336 7337 7338 7339
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

7340 7341 7342 7343
	ref_node = alloc_fixed_file_ref_node(ctx);
	if (IS_ERR(ref_node))
		return PTR_ERR(ref_node);

7344
	done = 0;
7345
	fds = u64_to_user_ptr(up->fds);
7346
	while (nr_args) {
7347 7348 7349
		struct fixed_file_table *table;
		unsigned index;

7350 7351 7352 7353 7354
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
7355 7356
		i = array_index_nospec(up->offset, ctx->nr_user_files);
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7357 7358
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
7359
			file = table->files[index];
7360 7361 7362
			err = io_queue_file_removal(data, file);
			if (err)
				break;
7363
			table->files[index] = NULL;
7364
			needs_switch = true;
7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384
		}
		if (fd != -1) {
			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
7385
			table->files[index] = file;
7386
			err = io_sqe_file_register(ctx, file, i);
7387
			if (err) {
7388
				table->files[index] = NULL;
7389
				fput(file);
7390
				break;
7391
			}
7392 7393 7394
		}
		nr_args--;
		done++;
7395 7396 7397
		up->offset++;
	}

7398 7399
	if (needs_switch) {
		percpu_ref_kill(data->cur_refs);
7400
		spin_lock(&data->lock);
7401 7402
		list_add(&ref_node->node, &data->ref_list);
		data->cur_refs = &ref_node->refs;
7403
		spin_unlock(&data->lock);
7404 7405 7406
		percpu_ref_get(&ctx->file_data->refs);
	} else
		destroy_fixed_file_ref_node(ref_node);
7407 7408 7409

	return done ? done : err;
}
7410

7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426
static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}
7427

7428
static void io_free_work(struct io_wq_work *work)
7429 7430 7431
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

7432
	/* Consider that io_steal_work() relies on this ref */
7433 7434 7435
	io_put_req(req);
}

7436 7437 7438 7439 7440 7441 7442 7443 7444 7445
static int io_init_wq_offload(struct io_ring_ctx *ctx,
			      struct io_uring_params *p)
{
	struct io_wq_data data;
	struct fd f;
	struct io_ring_ctx *ctx_attach;
	unsigned int concurrency;
	int ret = 0;

	data.user = ctx->user;
7446
	data.free_work = io_free_work;
7447
	data.do_work = io_wq_submit_work;
7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482

	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
		/* Do QD, or 4 * CPUS, whatever is smallest */
		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

		ctx->io_wq = io_wq_create(concurrency, &data);
		if (IS_ERR(ctx->io_wq)) {
			ret = PTR_ERR(ctx->io_wq);
			ctx->io_wq = NULL;
		}
		return ret;
	}

	f = fdget(p->wq_fd);
	if (!f.file)
		return -EBADF;

	if (f.file->f_op != &io_uring_fops) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx_attach = f.file->private_data;
	/* @io_wq is protected by holding the fd */
	if (!io_wq_get(ctx_attach->io_wq, &data)) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx->io_wq = ctx_attach->io_wq;
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
7483 7484
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
7485 7486 7487
{
	int ret;

J
Jens Axboe 已提交
7488
	if (ctx->flags & IORING_SETUP_SQPOLL) {
7489 7490 7491 7492
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

7493 7494 7495 7496
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
7497
		if (p->flags & IORING_SETUP_SQ_AFF) {
7498
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
7499

7500
			ret = -EINVAL;
7501 7502
			if (cpu >= nr_cpu_ids)
				goto err;
7503
			if (!cpu_online(cpu))
7504 7505
				goto err;

J
Jens Axboe 已提交
7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

7525 7526
	ret = io_init_wq_offload(ctx, p);
	if (ret)
J
Jens Axboe 已提交
7527 7528 7529 7530
		goto err;

	return 0;
err:
7531
	io_finish_async(ctx);
J
Jens Axboe 已提交
7532 7533 7534
	return ret;
}

7535 7536
static inline void __io_unaccount_mem(struct user_struct *user,
				      unsigned long nr_pages)
J
Jens Axboe 已提交
7537 7538 7539 7540
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

7541 7542
static inline int __io_account_mem(struct user_struct *user,
				   unsigned long nr_pages)
J
Jens Axboe 已提交
7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

7560 7561
static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			     enum io_mem_account acct)
7562
{
7563
	if (ctx->limit_mem)
7564
		__io_unaccount_mem(ctx->user, nr_pages);
7565

7566 7567 7568 7569 7570 7571
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm -= nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7572 7573
}

7574 7575
static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
			  enum io_mem_account acct)
7576
{
7577 7578 7579 7580 7581 7582 7583 7584
	int ret;

	if (ctx->limit_mem) {
		ret = __io_account_mem(ctx->user, nr_pages);
		if (ret)
			return ret;
	}

7585 7586 7587 7588 7589 7590
	if (ctx->sqo_mm) {
		if (acct == ACCT_LOCKED)
			ctx->sqo_mm->locked_vm += nr_pages;
		else if (acct == ACCT_PINNED)
			atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
	}
7591 7592 7593 7594

	return 0;
}

J
Jens Axboe 已提交
7595 7596
static void io_mem_free(void *ptr)
{
7597 7598 7599 7600
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
7601

7602
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

7631 7632 7633
	if (sq_offset)
		*sq_offset = off;

7634 7635 7636 7637 7638 7639 7640 7641 7642 7643
	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	return off;
}

J
Jens Axboe 已提交
7644 7645
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
7646
	size_t pages;
J
Jens Axboe 已提交
7647

7648 7649 7650 7651
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
7652

7653
	return pages;
J
Jens Axboe 已提交
7654 7655
}

7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
7667
			unpin_user_page(imu->bvec[j].bv_page);
7668

7669
		io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
7670
		kvfree(imu->bvec);
7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

7694
		dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
7732
			goto err;
7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

7752
		ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
7753 7754
		if (ret)
			goto err;
7755 7756 7757

		ret = 0;
		if (!pages || nr_pages > got_pages) {
7758 7759
			kvfree(vmas);
			kvfree(pages);
7760
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
7761
						GFP_KERNEL);
7762
			vmas = kvmalloc_array(nr_pages,
7763 7764 7765 7766
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
7767
				io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7768 7769 7770 7771 7772
				goto err;
			}
			got_pages = nr_pages;
		}

7773
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
7774 7775 7776
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
7777
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7778 7779 7780 7781
			goto err;
		}

		ret = 0;
7782
		mmap_read_lock(current->mm);
7783
		pret = pin_user_pages(ubuf, nr_pages,
7784 7785
				      FOLL_WRITE | FOLL_LONGTERM,
				      pages, vmas);
7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
7800
		mmap_read_unlock(current->mm);
7801 7802 7803 7804 7805
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
7806
			if (pret > 0)
7807
				unpin_user_pages(pages, pret);
7808
			io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7809
			kvfree(imu->bvec);
7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
7832 7833
	kvfree(pages);
	kvfree(vmas);
7834 7835
	return 0;
err:
7836 7837
	kvfree(pages);
	kvfree(vmas);
7838 7839 7840 7841
	io_sqe_buffer_unregister(ctx);
	return ret;
}

7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

7874 7875 7876 7877 7878
static int __io_destroy_buffers(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	struct io_buffer *buf = p;

7879
	__io_remove_buffers(ctx, buf, id, -1U);
7880 7881 7882 7883 7884 7885 7886 7887 7888
	return 0;
}

static void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
	idr_destroy(&ctx->io_buffer_idr);
}

J
Jens Axboe 已提交
7889 7890
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
7891
	io_finish_async(ctx);
7892
	io_sqe_buffer_unregister(ctx);
7893
	if (ctx->sqo_mm) {
J
Jens Axboe 已提交
7894
		mmdrop(ctx->sqo_mm);
7895 7896
		ctx->sqo_mm = NULL;
	}
J
Jens Axboe 已提交
7897

J
Jens Axboe 已提交
7898
	io_sqe_files_unregister(ctx);
7899
	io_eventfd_unregister(ctx);
7900
	io_destroy_buffers(ctx);
J
Jens Axboe 已提交
7901
	idr_destroy(&ctx->personality_idr);
J
Jens Axboe 已提交
7902

J
Jens Axboe 已提交
7903
#if defined(CONFIG_UNIX)
7904 7905
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
7906
		sock_release(ctx->ring_sock);
7907
	}
J
Jens Axboe 已提交
7908 7909
#endif

7910
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
7911 7912 7913 7914
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	free_uid(ctx->user);
7915
	put_cred(ctx->creds);
7916
	kfree(ctx->cancel_hash);
7917
	kmem_cache_free(req_cachep, ctx->fallback_req);
J
Jens Axboe 已提交
7918 7919 7920 7921 7922 7923 7924 7925 7926
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
7927 7928 7929 7930
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
7931
	smp_rmb();
7932 7933
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
7934
		mask |= EPOLLOUT | EPOLLWRNORM;
7935
	if (io_cqring_events(ctx, false))
J
Jens Axboe 已提交
7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958
static int io_remove_personalities(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	const struct cred *cred;

	cred = idr_remove(&ctx->personality_idr, id);
	if (cred)
		put_cred(cred);
	return 0;
}

7959 7960
static void io_ring_exit_work(struct work_struct *work)
{
7961 7962
	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
					       exit_work);
7963

7964 7965 7966 7967 7968 7969
	/*
	 * If we're doing polled IO and end up having requests being
	 * submitted async (out-of-line), then completions can come in while
	 * we're waiting for refs to drop. We need to reap these manually,
	 * as nobody else will be looking for them.
	 */
7970
	do {
7971 7972
		if (ctx->rings)
			io_cqring_overflow_flush(ctx, true);
7973 7974
		io_iopoll_try_reap_events(ctx);
	} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
7975 7976 7977
	io_ring_ctx_free(ctx);
}

J
Jens Axboe 已提交
7978 7979 7980 7981 7982 7983
static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
7984
	io_kill_timeouts(ctx);
7985
	io_poll_remove_all(ctx);
7986 7987 7988 7989

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

7990 7991 7992
	/* if we failed setting up the ctx, we might not have any rings */
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);
7993
	io_iopoll_try_reap_events(ctx);
7994
	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
7995 7996 7997 7998 7999 8000

	/*
	 * Do this upfront, so we won't have a grace period where the ring
	 * is closed but resources aren't reaped yet. This can cause
	 * spurious failure in setting up a new ring.
	 */
8001 8002
	io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
			 ACCT_LOCKED);
8003

8004
	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
8005 8006 8007 8008 8009 8010 8011
	/*
	 * Use system_unbound_wq to avoid spawning tons of event kworkers
	 * if we're exiting a ton of rings at the same time. It just adds
	 * noise and overhead, there's no discernable change in runtime
	 * over using system_wq.
	 */
	queue_work(system_unbound_wq, &ctx->exit_work);
J
Jens Axboe 已提交
8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

8023 8024 8025 8026 8027 8028 8029
static bool io_wq_files_match(struct io_wq_work *work, void *data)
{
	struct files_struct *files = data;

	return work->files == files;
}

8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047
/*
 * Returns true if 'preq' is the link parent of 'req'
 */
static bool io_match_link(struct io_kiocb *preq, struct io_kiocb *req)
{
	struct io_kiocb *link;

	if (!(preq->flags & REQ_F_LINK_HEAD))
		return false;

	list_for_each_entry(link, &preq->link_list, link_list) {
		if (link == req)
			return true;
	}

	return false;
}

8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069
static inline bool io_match_files(struct io_kiocb *req,
				       struct files_struct *files)
{
	return (req->flags & REQ_F_WORK_INITIALIZED) && req->work.files == files;
}

static bool io_match_link_files(struct io_kiocb *req,
				struct files_struct *files)
{
	struct io_kiocb *link;

	if (io_match_files(req, files))
		return true;
	if (req->flags & REQ_F_LINK_HEAD) {
		list_for_each_entry(link, &req->link_list, link_list) {
			if (io_match_files(link, files))
				return true;
		}
	}
	return false;
}

8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116
/*
 * We're looking to cancel 'req' because it's holding on to our files, but
 * 'req' could be a link to another request. See if it is, and cancel that
 * parent request if so.
 */
static bool io_poll_remove_link(struct io_ring_ctx *ctx, struct io_kiocb *req)
{
	struct hlist_node *tmp;
	struct io_kiocb *preq;
	bool found = false;
	int i;

	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(preq, tmp, list, hash_node) {
			found = io_match_link(preq, req);
			if (found) {
				io_poll_remove_one(preq);
				break;
			}
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	return found;
}

static bool io_timeout_remove_link(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	struct io_kiocb *preq;
	bool found = false;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry(preq, &ctx->timeout_list, timeout.list) {
		found = io_match_link(preq, req);
		if (found) {
			__io_timeout_cancel(preq);
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	return found;
}

8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143
static bool io_cancel_link_cb(struct io_wq_work *work, void *data)
{
	return io_match_link(container_of(work, struct io_kiocb, work), data);
}

static void io_attempt_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
{
	enum io_wq_cancel cret;

	/* cancel this particular work, if it's running */
	cret = io_wq_cancel_work(ctx->io_wq, &req->work);
	if (cret != IO_WQ_CANCEL_NOTFOUND)
		return;

	/* find links that hold this pending, cancel those */
	cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_link_cb, req, true);
	if (cret != IO_WQ_CANCEL_NOTFOUND)
		return;

	/* if we have a poll link holding this pending, cancel that */
	if (io_poll_remove_link(ctx, req))
		return;

	/* final option, timeout link is holding this req pending */
	io_timeout_remove_link(ctx, req);
}

8144 8145 8146 8147 8148 8149 8150 8151
static void io_cancel_defer_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
	struct io_defer_entry *de = NULL;
	LIST_HEAD(list);

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_reverse(de, &ctx->defer_list, list) {
8152
		if (io_match_link_files(de->req, files)) {
8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168
			list_cut_position(&list, &ctx->defer_list, &de->list);
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);

	while (!list_empty(&list)) {
		de = list_first_entry(&list, struct io_defer_entry, list);
		list_del_init(&de->list);
		req_set_fail_links(de->req);
		io_put_req(de->req);
		io_req_complete(de->req, -ECANCELED);
		kfree(de);
	}
}

8169 8170 8171
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
8172 8173 8174
	if (list_empty_careful(&ctx->inflight_list))
		return;

8175
	io_cancel_defer_files(ctx, files);
8176 8177 8178
	/* cancel all at once, should be faster than doing it one by one*/
	io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);

8179
	while (!list_empty_careful(&ctx->inflight_list)) {
8180 8181
		struct io_kiocb *cancel_req = NULL, *req;
		DEFINE_WAIT(wait);
8182 8183 8184

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
8185 8186 8187 8188 8189 8190 8191
			if (req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
8192
		}
8193
		if (cancel_req)
8194
			prepare_to_wait(&ctx->inflight_wait, &wait,
8195
						TASK_UNINTERRUPTIBLE);
8196 8197
		spin_unlock_irq(&ctx->inflight_lock);

8198 8199
		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
8200
			break;
8201 8202 8203
		/* cancel this request, or head link requests */
		io_attempt_cancel(ctx, cancel_req);
		io_put_req(cancel_req);
8204 8205
		/* cancellations _may_ trigger task work */
		io_run_task_work();
8206
		schedule();
8207
		finish_wait(&ctx->inflight_wait, &wait);
8208 8209 8210
	}
}

8211
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
8212
{
8213 8214
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct task_struct *task = data;
8215

8216
	return req->task == task;
8217 8218
}

8219 8220 8221 8222 8223
static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
8224 8225 8226 8227

	/*
	 * If the task is going away, cancel work it may have pending
	 */
8228 8229
	if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
		io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
8230

8231 8232 8233
	return 0;
}

8234 8235
static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
J
Jens Axboe 已提交
8236 8237
{
	struct io_ring_ctx *ctx = file->private_data;
8238
	loff_t offset = pgoff << PAGE_SHIFT;
J
Jens Axboe 已提交
8239 8240 8241 8242 8243
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
8244 8245
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
8246 8247 8248 8249 8250
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
8251
		return ERR_PTR(-EINVAL);
J
Jens Axboe 已提交
8252 8253 8254
	}

	page = virt_to_head_page(ptr);
8255
	if (sz > page_size(page))
8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271
		return ERR_PTR(-EINVAL);

	return ptr;
}

#ifdef CONFIG_MMU

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);
J
Jens Axboe 已提交
8272 8273 8274 8275 8276

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303
#else /* !CONFIG_MMU */

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
}

static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
{
	return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
}

static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len,
	unsigned long pgoff, unsigned long flags)
{
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, pgoff, len);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	return (unsigned long) ptr;
}

#endif /* !CONFIG_MMU */

J
Jens Axboe 已提交
8304 8305 8306 8307 8308 8309 8310 8311 8312
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

8313
	io_run_task_work();
8314

J
Jens Axboe 已提交
8315
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
8331 8332 8333 8334 8335
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
8336
	ret = 0;
J
Jens Axboe 已提交
8337
	if (ctx->flags & IORING_SETUP_SQPOLL) {
8338 8339
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false);
J
Jens Axboe 已提交
8340 8341 8342
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
8343
	} else if (to_submit) {
J
Jens Axboe 已提交
8344
		mutex_lock(&ctx->uring_lock);
8345
		submitted = io_submit_sqes(ctx, to_submit, f.file, fd);
J
Jens Axboe 已提交
8346
		mutex_unlock(&ctx->uring_lock);
8347 8348 8349

		if (submitted != to_submit)
			goto out;
J
Jens Axboe 已提交
8350 8351 8352 8353
	}
	if (flags & IORING_ENTER_GETEVENTS) {
		min_complete = min(min_complete, ctx->cq_entries);

8354 8355 8356 8357 8358 8359 8360 8361
		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
8362
			ret = io_iopoll_check(ctx, min_complete);
J
Jens Axboe 已提交
8363 8364 8365
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
8366 8367
	}

8368
out:
8369
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
8370 8371 8372 8373 8374
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

8375
#ifdef CONFIG_PROC_FS
8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436
static int io_uring_show_cred(int id, void *p, void *data)
{
	const struct cred *cred = p;
	struct seq_file *m = data;
	struct user_namespace *uns = seq_user_ns(m);
	struct group_info *gi;
	kernel_cap_t cap;
	unsigned __capi;
	int g;

	seq_printf(m, "%5d\n", id);
	seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
	seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
	seq_puts(m, "\n\tGroups:\t");
	gi = cred->group_info;
	for (g = 0; g < gi->ngroups; g++) {
		seq_put_decimal_ull(m, g ? " " : "",
					from_kgid_munged(uns, gi->gid[g]));
	}
	seq_puts(m, "\n\tCapEff:\t");
	cap = cred->cap_effective;
	CAP_FOR_EACH_U32(__capi)
		seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
	seq_putc(m, '\n');
	return 0;
}

static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	int i;

	mutex_lock(&ctx->uring_lock);
	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (!idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447
	seq_printf(m, "PollList:\n");
	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list = &ctx->cancel_hash[i];
		struct io_kiocb *req;

		hlist_for_each_entry(req, list, hash_node)
			seq_printf(m, "  op=%d, task_works=%d\n", req->opcode,
					req->task->task_works != NULL);
	}
	spin_unlock_irq(&ctx->completion_lock);
8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459
	mutex_unlock(&ctx->uring_lock);
}

static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
{
	struct io_ring_ctx *ctx = f->private_data;

	if (percpu_ref_tryget(&ctx->refs)) {
		__io_uring_show_fdinfo(ctx, m);
		percpu_ref_put(&ctx->refs);
	}
}
8460
#endif
8461

J
Jens Axboe 已提交
8462 8463
static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
8464
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
8465
	.mmap		= io_uring_mmap,
8466 8467 8468 8469
#ifndef CONFIG_MMU
	.get_unmapped_area = io_uring_nommu_get_unmapped_area,
	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
#endif
J
Jens Axboe 已提交
8470 8471
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
8472
#ifdef CONFIG_PROC_FS
8473
	.show_fdinfo	= io_uring_show_fdinfo,
8474
#endif
J
Jens Axboe 已提交
8475 8476 8477 8478 8479
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
8480 8481
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
8482

8483 8484 8485 8486
	/* make sure these are sane, as we already accounted them */
	ctx->sq_entries = p->sq_entries;
	ctx->cq_entries = p->cq_entries;

8487 8488 8489 8490 8491 8492
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
8493 8494
		return -ENOMEM;

8495 8496 8497 8498 8499 8500 8501 8502
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
J
Jens Axboe 已提交
8503 8504

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
8505 8506 8507
	if (size == SIZE_MAX) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8508
		return -EOVERFLOW;
8509
	}
J
Jens Axboe 已提交
8510 8511

	ctx->sq_sqes = io_mem_alloc(size);
8512 8513 8514
	if (!ctx->sq_sqes) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
8515
		return -ENOMEM;
8516
	}
J
Jens Axboe 已提交
8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

8564 8565
static int io_uring_create(unsigned entries, struct io_uring_params *p,
			   struct io_uring_params __user *params)
J
Jens Axboe 已提交
8566 8567 8568
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
8569
	bool limit_mem;
J
Jens Axboe 已提交
8570 8571
	int ret;

8572
	if (!entries)
J
Jens Axboe 已提交
8573
		return -EINVAL;
8574 8575 8576 8577 8578
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}
J
Jens Axboe 已提交
8579 8580 8581 8582 8583

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
8584 8585 8586
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
8587 8588
	 */
	p->sq_entries = roundup_pow_of_two(entries);
8589 8590 8591 8592 8593 8594
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
8595
		if (p->cq_entries < p->sq_entries)
8596
			return -EINVAL;
8597 8598 8599 8600 8601
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
8602 8603 8604 8605
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
8606 8607

	user = get_uid(current_user());
8608
	limit_mem = !capable(CAP_IPC_LOCK);
J
Jens Axboe 已提交
8609

8610
	if (limit_mem) {
8611
		ret = __io_account_mem(user,
J
Jens Axboe 已提交
8612 8613 8614 8615 8616 8617 8618 8619 8620
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
8621
		if (limit_mem)
8622
			__io_unaccount_mem(user, ring_pages(p->sq_entries,
J
Jens Axboe 已提交
8623 8624 8625 8626 8627 8628
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->user = user;
8629
	ctx->creds = get_current_cred();
J
Jens Axboe 已提交
8630

8631 8632 8633
	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

8634 8635 8636 8637 8638 8639 8640 8641 8642 8643
	/*
	 * Account memory _before_ installing the file descriptor. Once
	 * the descriptor is installed, it can get closed at any time. Also
	 * do this before hitting the general error path, as ring freeing
	 * will un-account as well.
	 */
	io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
		       ACCT_LOCKED);
	ctx->limit_mem = limit_mem;

J
Jens Axboe 已提交
8644 8645 8646 8647
	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
8648
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
8649 8650 8651 8652
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
8653 8654 8655 8656 8657 8658 8659
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
8660 8661

	memset(&p->cq_off, 0, sizeof(p->cq_off));
8662 8663 8664 8665 8666 8667
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
8668
	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
8669

8670 8671
	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
8672 8673
			IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
			IORING_FEAT_POLL_32BITS;
8674 8675 8676 8677 8678

	if (copy_to_user(params, p, sizeof(*p))) {
		ret = -EFAULT;
		goto err;
	}
8679

8680 8681 8682 8683 8684 8685 8686 8687
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

8688
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
J
Jens Axboe 已提交
8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
8712
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8713
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
8714
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
J
Jens Axboe 已提交
8715 8716
		return -EINVAL;

8717
	return  io_uring_create(entries, &p, params);
J
Jens Axboe 已提交
8718 8719 8720 8721 8722 8723 8724 8725
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764
static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}

8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803
static int io_register_personality(struct io_ring_ctx *ctx)
{
	const struct cred *creds = get_current_cred();
	int id;

	id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
				USHRT_MAX, GFP_KERNEL);
	if (id < 0)
		put_cred(creds);
	return id;
}

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}

static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}

8804 8805
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
8806 8807
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
8808 8809 8810
{
	int ret;

8811 8812 8813 8814 8815 8816 8817 8818
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

8819
	if (io_register_op_must_quiesce(opcode)) {
8820
		percpu_ref_kill(&ctx->refs);
8821

8822 8823 8824 8825 8826 8827 8828 8829 8830
		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
8831
		ret = wait_for_completion_interruptible(&ctx->ref_comp);
8832
		mutex_lock(&ctx->uring_lock);
8833 8834 8835 8836 8837
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
8838
	}
8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
8850 8851 8852 8853 8854 8855 8856 8857 8858
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
8859 8860 8861
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
8862
	case IORING_REGISTER_EVENTFD:
8863
	case IORING_REGISTER_EVENTFD_ASYNC:
8864 8865 8866 8867
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
8868 8869 8870 8871 8872 8873
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
8874 8875 8876 8877 8878 8879 8880
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
8881 8882 8883 8884 8885 8886
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
8899 8900 8901 8902 8903
	default:
		ret = -EINVAL;
		break;
	}

8904
	if (io_register_op_must_quiesce(opcode)) {
8905 8906
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
8907
out:
8908
		reinit_completion(&ctx->ref_comp);
8909
	}
8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
8933 8934
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
8935 8936 8937 8938 8939
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
8940 8941
static int __init io_uring_init(void)
{
8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
P
Pavel Begunkov 已提交
8957
	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
8958 8959 8960 8961 8962
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
8963 8964
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
8965 8966 8967 8968 8969 8970 8971 8972
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
P
Pavel Begunkov 已提交
8973
	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
8974 8975 8976
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
P
Pavel Begunkov 已提交
8977
	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);
8978

8979
	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
8980
	BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
J
Jens Axboe 已提交
8981 8982 8983 8984
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);