io_uring.c 165.5 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46 47 48
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
#include <linux/refcount.h>
#include <linux/uio.h>
49
#include <linux/bits.h>
J
Jens Axboe 已提交
50 51 52 53 54 55 56 57 58 59

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/mmu_context.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
60
#include <linux/kthread.h>
J
Jens Axboe 已提交
61
#include <linux/blkdev.h>
62
#include <linux/bvec.h>
J
Jens Axboe 已提交
63 64 65
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
66
#include <net/scm.h>
J
Jens Axboe 已提交
67 68 69 70
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
71 72
#include <linux/sizes.h>
#include <linux/hugetlb.h>
73
#include <linux/highmem.h>
74 75
#include <linux/namei.h>
#include <linux/fsnotify.h>
J
Jens Axboe 已提交
76
#include <linux/fadvise.h>
77
#include <linux/eventpoll.h>
J
Jens Axboe 已提交
78

79 80 81
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
82 83 84
#include <uapi/linux/io_uring.h>

#include "internal.h"
85
#include "io-wq.h"
J
Jens Axboe 已提交
86

87
#define IORING_MAX_ENTRIES	32768
88
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
89 90 91 92 93 94 95 96

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
97 98 99 100 101 102

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
103
/*
104 105
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
106 107 108 109
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
110
struct io_rings {
S
Stefan Bühler 已提交
111 112 113 114
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
115 116 117
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
118
	 */
119
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
120
	/*
121
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
122 123
	 * ring_entries - 1)
	 */
124 125 126
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
127 128 129 130 131 132 133 134 135 136 137 138
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
139
	u32			sq_dropped;
S
Stefan Bühler 已提交
140 141 142 143 144 145 146 147 148
	/*
	 * Runtime flags
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
149
	u32			sq_flags;
S
Stefan Bühler 已提交
150 151 152
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
L
LimingWu 已提交
153
	 * there are not more requests pending than there is space in
S
Stefan Bühler 已提交
154 155 156 157 158 159 160 161 162
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
163
	u32			cq_overflow;
S
Stefan Bühler 已提交
164 165 166 167 168 169 170
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
171
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
172 173
};

174 175 176 177 178 179 180
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

181 182
struct fixed_file_table {
	struct file		**files;
183 184
};

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
enum {
	FFD_F_ATOMIC,
};

struct fixed_file_data {
	struct fixed_file_table		*table;
	struct io_ring_ctx		*ctx;

	struct percpu_ref		refs;
	struct llist_head		put_llist;
	unsigned long			state;
	struct work_struct		ref_work;
	struct completion		done;
};

J
Jens Axboe 已提交
200 201 202 203 204 205 206
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
207 208 209 210
		int			compat: 1;
		int			account_mem: 1;
		int			cq_overflow_flushed: 1;
		int			drain_next: 1;
211
		int			eventfd_async: 1;
J
Jens Axboe 已提交
212

213 214 215 216 217 218 219 220 221 222 223 224
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
225 226 227
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
228
		unsigned		sq_thread_idle;
229
		unsigned		cached_sq_dropped;
230
		atomic_t		cached_cq_overflow;
231
		unsigned long		sq_check_overflow;
232 233

		struct list_head	defer_list;
J
Jens Axboe 已提交
234
		struct list_head	timeout_list;
235
		struct list_head	cq_overflow_list;
236 237

		wait_queue_head_t	inflight_wait;
238
		struct io_uring_sqe	*sq_sqes;
J
Jens Axboe 已提交
239 240
	} ____cacheline_aligned_in_smp;

241 242
	struct io_rings	*rings;

J
Jens Axboe 已提交
243
	/* IO offload */
244
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
245
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
246
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
247
	wait_queue_head_t	sqo_wait;
248

J
Jens Axboe 已提交
249 250 251 252 253
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
254
	struct fixed_file_data	*file_data;
J
Jens Axboe 已提交
255
	unsigned		nr_user_files;
P
Pavel Begunkov 已提交
256 257
	int 			ring_fd;
	struct file 		*ring_file;
J
Jens Axboe 已提交
258

259 260 261 262
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
263 264
	struct user_struct	*user;

265
	const struct cred	*creds;
266

267 268 269
	/* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
	struct completion	*completions;

270 271 272
	/* if all else fails... */
	struct io_kiocb		*fallback_req;

273 274 275 276
#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif

277 278
	struct idr		personality_idr;

279 280 281 282 283
	struct {
		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		unsigned		cq_mask;
		atomic_t		cq_timeouts;
284
		unsigned long		cq_check_overflow;
285 286 287 288
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
		struct eventfd_ctx	*cq_ev_fd;
	} ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
289 290 291 292 293 294 295 296

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
297 298
		struct llist_head	poll_llist;

J
Jens Axboe 已提交
299 300 301 302 303 304 305
		/*
		 * ->poll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct list_head	poll_list;
306 307
		struct hlist_head	*cancel_hash;
		unsigned		cancel_hash_bits;
308
		bool			poll_multi_file;
309

310 311
		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
312 313 314
	} ____cacheline_aligned_in_smp;
};

J
Jens Axboe 已提交
315 316 317 318
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
319 320
struct io_poll_iocb {
	struct file			*file;
321 322 323 324
	union {
		struct wait_queue_head	*head;
		u64			addr;
	};
325
	__poll_t			events;
J
Jens Axboe 已提交
326
	bool				done;
327
	bool				canceled;
328
	struct wait_queue_entry		wait;
329 330
};

331 332 333 334 335 336
struct io_close {
	struct file			*file;
	struct file			*put_file;
	int				fd;
};

337 338 339 340 341
struct io_timeout_data {
	struct io_kiocb			*req;
	struct hrtimer			timer;
	struct timespec64		ts;
	enum hrtimer_mode		mode;
342
	u32				seq_offset;
343 344
};

345 346 347 348 349 350 351 352 353 354 355 356
struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
};

struct io_sync {
	struct file			*file;
	loff_t				len;
	loff_t				off;
	int				flags;
357
	int				mode;
358 359
};

360 361 362 363 364
struct io_cancel {
	struct file			*file;
	u64				addr;
};

365 366 367 368
struct io_timeout {
	struct file			*file;
	u64				addr;
	int				flags;
369
	unsigned			count;
370 371
};

372 373 374 375 376 377 378
struct io_rw {
	/* NOTE: kiocb has the file as the first member, so don't do it here */
	struct kiocb			kiocb;
	u64				addr;
	u64				len;
};

379 380 381 382 383 384
struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
};

385 386
struct io_sr_msg {
	struct file			*file;
387 388 389 390
	union {
		struct user_msghdr __user *msg;
		void __user		*buf;
	};
391
	int				msg_flags;
392
	size_t				len;
393 394
};

395 396 397
struct io_open {
	struct file			*file;
	int				dfd;
398 399 400
	union {
		unsigned		mask;
	};
401
	struct filename			*filename;
402
	struct statx __user		*buffer;
403
	struct open_how			how;
404 405
};

406 407 408 409 410 411 412
struct io_files_update {
	struct file			*file;
	u64				arg;
	u32				nr_args;
	u32				offset;
};

J
Jens Axboe 已提交
413 414 415 416 417 418 419
struct io_fadvise {
	struct file			*file;
	u64				offset;
	u32				len;
	u32				advice;
};

J
Jens Axboe 已提交
420 421 422 423 424 425 426
struct io_madvise {
	struct file			*file;
	u64				addr;
	u32				len;
	u32				advice;
};

427 428 429 430 431 432
struct io_epoll {
	struct file			*file;
	int				epfd;
	int				op;
	int				fd;
	struct epoll_event		event;
433 434
};

435 436 437 438
struct io_async_connect {
	struct sockaddr_storage		address;
};

439 440 441 442 443 444 445
struct io_async_msghdr {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	struct sockaddr __user		*uaddr;
	struct msghdr			msg;
};

446 447 448 449 450 451 452
struct io_async_rw {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	ssize_t				nr_segs;
	ssize_t				size;
};

453 454 455 456
struct io_async_open {
	struct filename			*filename;
};

457
struct io_async_ctx {
458 459
	union {
		struct io_async_rw	rw;
460
		struct io_async_msghdr	msg;
461
		struct io_async_connect	connect;
462
		struct io_timeout_data	timeout;
463
		struct io_async_open	open;
464
	};
465 466
};

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,

	REQ_F_LINK_NEXT_BIT,
	REQ_F_FAIL_LINK_BIT,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_IOPOLL_COMPLETED_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_TIMEOUT_BIT,
	REQ_F_ISREG_BIT,
	REQ_F_MUST_PUNT_BIT,
	REQ_F_TIMEOUT_NOSEQ_BIT,
	REQ_F_COMP_LOCKED_BIT,
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),

	/* already grabbed next link */
	REQ_F_LINK_NEXT		= BIT(REQ_F_LINK_NEXT_BIT),
	/* fail rest of links */
	REQ_F_FAIL_LINK		= BIT(REQ_F_FAIL_LINK_BIT),
	/* on inflight list */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* polled IO has completed */
	REQ_F_IOPOLL_COMPLETED	= BIT(REQ_F_IOPOLL_COMPLETED_BIT),
	/* has linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* timeout request */
	REQ_F_TIMEOUT		= BIT(REQ_F_TIMEOUT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* must be punted even for NONBLOCK */
	REQ_F_MUST_PUNT		= BIT(REQ_F_MUST_PUNT_BIT),
	/* no timeout sequence */
	REQ_F_TIMEOUT_NOSEQ	= BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
	/* completion under lock */
	REQ_F_COMP_LOCKED	= BIT(REQ_F_COMP_LOCKED_BIT),
};

J
Jens Axboe 已提交
526 527 528 529 530 531
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
532
struct io_kiocb {
533
	union {
J
Jens Axboe 已提交
534
		struct file		*file;
535
		struct io_rw		rw;
536
		struct io_poll_iocb	poll;
537 538
		struct io_accept	accept;
		struct io_sync		sync;
539
		struct io_cancel	cancel;
540
		struct io_timeout	timeout;
541
		struct io_connect	connect;
542
		struct io_sr_msg	sr_msg;
543
		struct io_open		open;
544
		struct io_close		close;
545
		struct io_files_update	files_update;
J
Jens Axboe 已提交
546
		struct io_fadvise	fadvise;
J
Jens Axboe 已提交
547
		struct io_madvise	madvise;
548
		struct io_epoll		epoll;
549
	};
J
Jens Axboe 已提交
550

551
	struct io_async_ctx		*io;
P
Pavel Begunkov 已提交
552 553 554 555
	/*
	 * llist_node is only used for poll deferred completions
	 */
	struct llist_node		llist_node;
556 557 558
	bool				has_user;
	bool				in_async;
	bool				needs_fixed_file;
559
	u8				opcode;
J
Jens Axboe 已提交
560 561

	struct io_ring_ctx	*ctx;
562 563
	union {
		struct list_head	list;
564
		struct hlist_node	hash_node;
565
	};
J
Jens Axboe 已提交
566
	struct list_head	link_list;
J
Jens Axboe 已提交
567
	unsigned int		flags;
J
Jens Axboe 已提交
568
	refcount_t		refs;
J
Jens Axboe 已提交
569
	u64			user_data;
J
Jens Axboe 已提交
570
	u32			result;
571
	u32			sequence;
J
Jens Axboe 已提交
572

573 574
	struct list_head	inflight_entry;

575
	struct io_wq_work	work;
J
Jens Axboe 已提交
576 577 578
};

#define IO_PLUG_THRESHOLD		2
J
Jens Axboe 已提交
579
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
580

581 582 583
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
584 585 586 587 588 589 590
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
	unsigned		int free_reqs;
	unsigned		int cur_req;

591 592 593 594 595 596 597 598 599 600
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		used_refs;
	unsigned int		ios_left;
};

601 602 603 604 605 606 607 608 609 610 611 612 613
struct io_op_def {
	/* needs req->io allocated for deferral/async */
	unsigned		async_ctx : 1;
	/* needs current->mm setup, does mm access */
	unsigned		needs_mm : 1;
	/* needs req->file assigned */
	unsigned		needs_file : 1;
	/* needs req->file assigned IFF fd is >= 0 */
	unsigned		fd_non_neg : 1;
	/* hash wq insertion if file is a regular file */
	unsigned		hash_reg_file : 1;
	/* unbound wq insertion if file is a non-regular file */
	unsigned		unbound_nonreg_file : 1;
614 615
	/* opcode is not supported by this kernel */
	unsigned		not_supported : 1;
616 617
	/* needs file table */
	unsigned		file_table : 1;
618 619 620
};

static const struct io_op_def io_op_defs[] = {
621 622
	[IORING_OP_NOP] = {},
	[IORING_OP_READV] = {
623 624 625 626 627
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
628
	[IORING_OP_WRITEV] = {
629 630 631 632 633 634
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
635
	[IORING_OP_FSYNC] = {
636 637
		.needs_file		= 1,
	},
638
	[IORING_OP_READ_FIXED] = {
639 640 641
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
642
	[IORING_OP_WRITE_FIXED] = {
643 644 645 646
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
647
	[IORING_OP_POLL_ADD] = {
648 649 650
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
651 652
	[IORING_OP_POLL_REMOVE] = {},
	[IORING_OP_SYNC_FILE_RANGE] = {
653 654
		.needs_file		= 1,
	},
655
	[IORING_OP_SENDMSG] = {
656 657 658 659 660
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
661
	[IORING_OP_RECVMSG] = {
662 663 664 665 666
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
667
	[IORING_OP_TIMEOUT] = {
668 669 670
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
671 672
	[IORING_OP_TIMEOUT_REMOVE] = {},
	[IORING_OP_ACCEPT] = {
673 674 675
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
676
		.file_table		= 1,
677
	},
678 679
	[IORING_OP_ASYNC_CANCEL] = {},
	[IORING_OP_LINK_TIMEOUT] = {
680 681 682
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
683
	[IORING_OP_CONNECT] = {
684 685 686 687 688
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
689
	[IORING_OP_FALLOCATE] = {
690 691
		.needs_file		= 1,
	},
692
	[IORING_OP_OPENAT] = {
693 694
		.needs_file		= 1,
		.fd_non_neg		= 1,
695
		.file_table		= 1,
696
	},
697
	[IORING_OP_CLOSE] = {
698
		.needs_file		= 1,
699
		.file_table		= 1,
700
	},
701
	[IORING_OP_FILES_UPDATE] = {
702
		.needs_mm		= 1,
703
		.file_table		= 1,
704
	},
705
	[IORING_OP_STATX] = {
706 707 708 709
		.needs_mm		= 1,
		.needs_file		= 1,
		.fd_non_neg		= 1,
	},
710
	[IORING_OP_READ] = {
711 712 713 714
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
715
	[IORING_OP_WRITE] = {
716 717 718 719
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
720
	[IORING_OP_FADVISE] = {
J
Jens Axboe 已提交
721 722
		.needs_file		= 1,
	},
723
	[IORING_OP_MADVISE] = {
J
Jens Axboe 已提交
724 725
		.needs_mm		= 1,
	},
726
	[IORING_OP_SEND] = {
727 728 729 730
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
731
	[IORING_OP_RECV] = {
732 733 734 735
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
736
	[IORING_OP_OPENAT2] = {
737 738
		.needs_file		= 1,
		.fd_non_neg		= 1,
739
		.file_table		= 1,
740
	},
741 742 743 744
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.file_table		= 1,
	},
745 746
};

747
static void io_wq_submit_work(struct io_wq_work **workptr);
748
static void io_cqring_fill_event(struct io_kiocb *req, long res);
749
static void io_put_req(struct io_kiocb *req);
750
static void __io_double_put_req(struct io_kiocb *req);
751 752
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
static void io_queue_linked_timeout(struct io_kiocb *req);
753 754 755
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *ip,
				 unsigned nr_args);
756
static int io_grab_files(struct io_kiocb *req);
757

J
Jens Axboe 已提交
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

779
	complete(&ctx->completions[0]);
J
Jens Axboe 已提交
780 781 782 783 784
}

static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;
785
	int hash_bits;
J
Jens Axboe 已提交
786 787 788 789 790

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

791 792 793 794
	ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
	if (!ctx->fallback_req)
		goto err;

795 796 797 798
	ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
	if (!ctx->completions)
		goto err;

799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
	/*
	 * Use 5 bits less than the max cq entries, that should give us around
	 * 32 entries per hash list if totally full and uniformly spread.
	 */
	hash_bits = ilog2(p->cq_entries);
	hash_bits -= 5;
	if (hash_bits <= 0)
		hash_bits = 1;
	ctx->cancel_hash_bits = hash_bits;
	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
					GFP_KERNEL);
	if (!ctx->cancel_hash)
		goto err;
	__hash_init(ctx->cancel_hash, 1U << hash_bits);

814
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
815 816
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
		goto err;
J
Jens Axboe 已提交
817 818 819

	ctx->flags = p->flags;
	init_waitqueue_head(&ctx->cq_wait);
820
	INIT_LIST_HEAD(&ctx->cq_overflow_list);
821 822
	init_completion(&ctx->completions[0]);
	init_completion(&ctx->completions[1]);
823
	idr_init(&ctx->personality_idr);
J
Jens Axboe 已提交
824 825 826
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
827
	init_llist_head(&ctx->poll_llist);
J
Jens Axboe 已提交
828
	INIT_LIST_HEAD(&ctx->poll_list);
829
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
830
	INIT_LIST_HEAD(&ctx->timeout_list);
831 832 833
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
J
Jens Axboe 已提交
834
	return ctx;
835
err:
836 837
	if (ctx->fallback_req)
		kmem_cache_free(req_cachep, ctx->fallback_req);
838
	kfree(ctx->completions);
839
	kfree(ctx->cancel_hash);
840 841
	kfree(ctx);
	return NULL;
J
Jens Axboe 已提交
842 843
}

B
Bob Liu 已提交
844
static inline bool __req_need_defer(struct io_kiocb *req)
845
{
846 847
	struct io_ring_ctx *ctx = req->ctx;

848 849
	return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
					+ atomic_read(&ctx->cached_cq_overflow);
850 851
}

B
Bob Liu 已提交
852
static inline bool req_need_defer(struct io_kiocb *req)
853
{
854
	if (unlikely(req->flags & REQ_F_IO_DRAIN))
B
Bob Liu 已提交
855
		return __req_need_defer(req);
856

B
Bob Liu 已提交
857
	return false;
858 859
}

860
static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
861 862 863
{
	struct io_kiocb *req;

864
	req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
B
Bob Liu 已提交
865
	if (req && !req_need_defer(req)) {
866 867 868 869 870 871 872
		list_del_init(&req->list);
		return req;
	}

	return NULL;
}

J
Jens Axboe 已提交
873 874
static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
{
875 876 877
	struct io_kiocb *req;

	req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
878 879 880
	if (req) {
		if (req->flags & REQ_F_TIMEOUT_NOSEQ)
			return NULL;
881
		if (!__req_need_defer(req)) {
882 883 884
			list_del_init(&req->list);
			return req;
		}
885 886 887
	}

	return NULL;
J
Jens Axboe 已提交
888 889
}

890
static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
891
{
892
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
893

894 895
	/* order cqe stores with ring update */
	smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
896

897 898 899
	if (wq_has_sleeper(&ctx->cq_wait)) {
		wake_up_interruptible(&ctx->cq_wait);
		kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
J
Jens Axboe 已提交
900 901 902
	}
}

903 904 905 906 907 908
static inline void io_req_work_grab_env(struct io_kiocb *req,
					const struct io_op_def *def)
{
	if (!req->work.mm && def->needs_mm) {
		mmgrab(current->mm);
		req->work.mm = current->mm;
J
Jens Axboe 已提交
909
	}
910 911
	if (!req->work.creds)
		req->work.creds = get_current_cred();
J
Jens Axboe 已提交
912 913
}

914
static inline void io_req_work_drop_env(struct io_kiocb *req)
915
{
916 917 918 919 920 921 922 923
	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
924 925
}

926 927
static inline bool io_prep_async_work(struct io_kiocb *req,
				      struct io_kiocb **link)
928
{
929
	const struct io_op_def *def = &io_op_defs[req->opcode];
930
	bool do_hashed = false;
931

932 933
	if (req->flags & REQ_F_ISREG) {
		if (def->hash_reg_file)
934
			do_hashed = true;
935 936
	} else {
		if (def->unbound_nonreg_file)
937
			req->work.flags |= IO_WQ_WORK_UNBOUND;
938
	}
939 940

	io_req_work_grab_env(req, def);
941

942
	*link = io_prep_linked_timeout(req);
943 944 945
	return do_hashed;
}

946
static inline void io_queue_async_work(struct io_kiocb *req)
947
{
948
	struct io_ring_ctx *ctx = req->ctx;
949 950 951 952
	struct io_kiocb *link;
	bool do_hashed;

	do_hashed = io_prep_async_work(req, &link);
953 954 955 956 957 958 959 960 961

	trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
					req->flags);
	if (!do_hashed) {
		io_wq_enqueue(ctx->io_wq, &req->work);
	} else {
		io_wq_enqueue_hashed(ctx->io_wq, &req->work,
					file_inode(req->file));
	}
962 963 964

	if (link)
		io_queue_linked_timeout(link);
965 966
}

J
Jens Axboe 已提交
967 968 969 970
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

971
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
J
Jens Axboe 已提交
972 973
	if (ret != -1) {
		atomic_inc(&req->ctx->cq_timeouts);
974
		list_del_init(&req->list);
975
		io_cqring_fill_event(req, 0);
976
		io_put_req(req);
J
Jens Axboe 已提交
977 978 979 980 981 982 983 984 985 986 987 988 989
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

990 991 992 993
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

J
Jens Axboe 已提交
994 995 996
	while ((req = io_get_timeout_req(ctx)) != NULL)
		io_kill_timeout(req);

997 998
	__io_commit_cqring(ctx);

999
	while ((req = io_get_deferred_req(ctx)) != NULL)
1000
		io_queue_async_work(req);
1001 1002
}

J
Jens Axboe 已提交
1003 1004
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
1005
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1006 1007 1008
	unsigned tail;

	tail = ctx->cached_cq_tail;
1009 1010 1011 1012 1013
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
1014
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
1015 1016 1017
		return NULL;

	ctx->cached_cq_tail++;
1018
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
1019 1020
}

1021 1022
static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
{
1023 1024
	if (!ctx->cq_ev_fd)
		return false;
1025 1026 1027 1028 1029
	if (!ctx->eventfd_async)
		return true;
	return io_wq_current_is_worker() || in_interrupt();
}

1030
static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
1031 1032 1033 1034 1035
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
1036
	if (trigger_ev)
1037 1038 1039
		eventfd_signal(ctx->cq_ev_fd, 1);
}

1040 1041 1042 1043 1044
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
{
	__io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
}

1045 1046
/* Returns true if there are no backlogged entries after the flush */
static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1047 1048 1049 1050 1051 1052 1053 1054 1055
{
	struct io_rings *rings = ctx->rings;
	struct io_uring_cqe *cqe;
	struct io_kiocb *req;
	unsigned long flags;
	LIST_HEAD(list);

	if (!force) {
		if (list_empty_careful(&ctx->cq_overflow_list))
1056
			return true;
1057 1058
		if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
		    rings->cq_ring_entries))
1059
			return false;
1060 1061 1062 1063 1064 1065
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/* if force is set, the ring is going away. always drop after that */
	if (force)
1066
		ctx->cq_overflow_flushed = 1;
1067

1068
	cqe = NULL;
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	while (!list_empty(&ctx->cq_overflow_list)) {
		cqe = io_get_cqring(ctx);
		if (!cqe && !force)
			break;

		req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
						list);
		list_move(&req->list, &list);
		if (cqe) {
			WRITE_ONCE(cqe->user_data, req->user_data);
			WRITE_ONCE(cqe->res, req->result);
			WRITE_ONCE(cqe->flags, 0);
		} else {
			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
		}
	}

	io_commit_cqring(ctx);
1088 1089 1090 1091
	if (cqe) {
		clear_bit(0, &ctx->sq_check_overflow);
		clear_bit(0, &ctx->cq_check_overflow);
	}
1092 1093 1094 1095 1096 1097
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	while (!list_empty(&list)) {
		req = list_first_entry(&list, struct io_kiocb, list);
		list_del(&req->list);
1098
		io_put_req(req);
1099
	}
1100 1101

	return cqe != NULL;
1102 1103
}

1104
static void io_cqring_fill_event(struct io_kiocb *req, long res)
J
Jens Axboe 已提交
1105
{
1106
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1107 1108
	struct io_uring_cqe *cqe;

1109
	trace_io_uring_complete(ctx, req->user_data, res);
J
Jens Axboe 已提交
1110

J
Jens Axboe 已提交
1111 1112 1113 1114 1115 1116
	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
1117
	if (likely(cqe)) {
1118
		WRITE_ONCE(cqe->user_data, req->user_data);
J
Jens Axboe 已提交
1119
		WRITE_ONCE(cqe->res, res);
1120
		WRITE_ONCE(cqe->flags, 0);
1121
	} else if (ctx->cq_overflow_flushed) {
1122 1123
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
1124
	} else {
1125 1126 1127 1128
		if (list_empty(&ctx->cq_overflow_list)) {
			set_bit(0, &ctx->sq_check_overflow);
			set_bit(0, &ctx->cq_check_overflow);
		}
1129 1130 1131
		refcount_inc(&req->refs);
		req->result = res;
		list_add_tail(&req->list, &ctx->cq_overflow_list);
J
Jens Axboe 已提交
1132 1133 1134
	}
}

1135
static void io_cqring_add_event(struct io_kiocb *req, long res)
J
Jens Axboe 已提交
1136
{
1137
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1138 1139 1140
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
1141
	io_cqring_fill_event(req, res);
J
Jens Axboe 已提交
1142 1143 1144
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
1145
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1146 1147
}

1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
static inline bool io_is_fallback_req(struct io_kiocb *req)
{
	return req == (struct io_kiocb *)
			((unsigned long) req->ctx->fallback_req & ~1UL);
}

static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	req = ctx->fallback_req;
	if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
		return req;

	return NULL;
}

J
Jens Axboe 已提交
1165 1166
static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
				   struct io_submit_state *state)
J
Jens Axboe 已提交
1167
{
1168
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
1169 1170
	struct io_kiocb *req;

J
Jens Axboe 已提交
1171
	if (!state) {
1172
		req = kmem_cache_alloc(req_cachep, gfp);
J
Jens Axboe 已提交
1173
		if (unlikely(!req))
1174
			goto fallback;
J
Jens Axboe 已提交
1175 1176 1177 1178 1179
	} else if (!state->free_reqs) {
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
1180 1181 1182 1183 1184 1185 1186 1187 1188
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
1189
				goto fallback;
1190 1191
			ret = 1;
		}
J
Jens Axboe 已提交
1192 1193 1194 1195 1196 1197 1198
		state->free_reqs = ret - 1;
		state->cur_req = 1;
		req = state->reqs[0];
	} else {
		req = state->reqs[state->cur_req];
		state->free_reqs--;
		state->cur_req++;
J
Jens Axboe 已提交
1199 1200
	}

1201
got_it:
1202
	req->io = NULL;
1203
	req->file = NULL;
J
Jens Axboe 已提交
1204 1205
	req->ctx = ctx;
	req->flags = 0;
1206 1207
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
J
Jens Axboe 已提交
1208
	req->result = 0;
1209
	INIT_IO_WORK(&req->work, io_wq_submit_work);
J
Jens Axboe 已提交
1210
	return req;
1211 1212 1213 1214
fallback:
	req = io_get_fallback_req(ctx);
	if (req)
		goto got_it;
1215
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
1216 1217 1218
	return NULL;
}

1219
static void __io_req_do_free(struct io_kiocb *req)
J
Jens Axboe 已提交
1220
{
1221 1222 1223 1224 1225 1226
	if (likely(!io_is_fallback_req(req)))
		kmem_cache_free(req_cachep, req);
	else
		clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
}

1227
static void __io_req_aux_free(struct io_kiocb *req)
J
Jens Axboe 已提交
1228
{
1229 1230
	struct io_ring_ctx *ctx = req->ctx;

1231
	kfree(req->io);
1232 1233 1234 1235 1236
	if (req->file) {
		if (req->flags & REQ_F_FIXED_FILE)
			percpu_ref_put(&ctx->file_data->refs);
		else
			fput(req->file);
J
Jens Axboe 已提交
1237
	}
1238 1239

	io_req_work_drop_env(req);
J
Jens Axboe 已提交
1240 1241
}

J
Jens Axboe 已提交
1242
static void __io_free_req(struct io_kiocb *req)
J
Jens Axboe 已提交
1243
{
1244
	__io_req_aux_free(req);
1245 1246

	if (req->flags & REQ_F_INFLIGHT) {
1247
		struct io_ring_ctx *ctx = req->ctx;
1248 1249 1250 1251 1252 1253 1254 1255
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
	}
1256 1257 1258

	percpu_ref_put(&req->ctx->refs);
	__io_req_do_free(req);
1259 1260
}

1261 1262 1263 1264 1265 1266 1267 1268
struct req_batch {
	void *reqs[IO_IOPOLL_BATCH];
	int to_free;
	int need_iter;
};

static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
{
1269 1270
	int fixed_refs = rb->to_free;

1271 1272 1273 1274 1275 1276
	if (!rb->to_free)
		return;
	if (rb->need_iter) {
		int i, inflight = 0;
		unsigned long flags;

1277
		fixed_refs = 0;
1278 1279 1280
		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

1281
			if (req->flags & REQ_F_FIXED_FILE) {
1282
				req->file = NULL;
1283 1284
				fixed_refs++;
			}
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
			if (req->flags & REQ_F_INFLIGHT)
				inflight++;
			__io_req_aux_free(req);
		}
		if (!inflight)
			goto do_free;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

1296
			if (req->flags & REQ_F_INFLIGHT) {
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
				list_del(&req->inflight_entry);
				if (!--inflight)
					break;
			}
		}
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);

		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
	}
do_free:
	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
1309 1310
	if (fixed_refs)
		percpu_ref_put_many(&ctx->file_data->refs, fixed_refs);
1311 1312
	percpu_ref_put_many(&ctx->refs, rb->to_free);
	rb->to_free = rb->need_iter = 0;
1313 1314
}

1315
static bool io_link_cancel_timeout(struct io_kiocb *req)
1316
{
1317
	struct io_ring_ctx *ctx = req->ctx;
1318 1319
	int ret;

1320
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1321
	if (ret != -1) {
1322
		io_cqring_fill_event(req, -ECANCELED);
1323 1324
		io_commit_cqring(ctx);
		req->flags &= ~REQ_F_LINK;
1325
		io_put_req(req);
1326 1327 1328 1329
		return true;
	}

	return false;
1330 1331
}

1332
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
J
Jens Axboe 已提交
1333
{
1334 1335
	struct io_ring_ctx *ctx = req->ctx;
	bool wake_ev = false;
J
Jens Axboe 已提交
1336

1337 1338 1339 1340
	/* Already got next link */
	if (req->flags & REQ_F_LINK_NEXT)
		return;

J
Jens Axboe 已提交
1341 1342 1343 1344 1345
	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
1346 1347 1348
	while (!list_empty(&req->link_list)) {
		struct io_kiocb *nxt = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
1349

1350 1351 1352
		if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
			     (nxt->flags & REQ_F_TIMEOUT))) {
			list_del_init(&nxt->link_list);
1353 1354 1355 1356
			wake_ev |= io_link_cancel_timeout(nxt);
			req->flags &= ~REQ_F_LINK_TIMEOUT;
			continue;
		}
J
Jens Axboe 已提交
1357

1358 1359 1360
		list_del_init(&req->link_list);
		if (!list_empty(&nxt->link_list))
			nxt->flags |= REQ_F_LINK;
1361
		*nxtptr = nxt;
1362
		break;
J
Jens Axboe 已提交
1363
	}
1364

1365
	req->flags |= REQ_F_LINK_NEXT;
1366 1367
	if (wake_ev)
		io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1368 1369 1370 1371 1372 1373 1374
}

/*
 * Called if REQ_F_LINK is set, and we fail the head request
 */
static void io_fail_links(struct io_kiocb *req)
{
1375 1376 1377 1378
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
J
Jens Axboe 已提交
1379 1380

	while (!list_empty(&req->link_list)) {
1381 1382
		struct io_kiocb *link = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
J
Jens Axboe 已提交
1383

1384
		list_del_init(&link->link_list);
1385
		trace_io_uring_fail_link(req, link);
1386 1387

		if ((req->flags & REQ_F_LINK_TIMEOUT) &&
1388
		    link->opcode == IORING_OP_LINK_TIMEOUT) {
1389
			io_link_cancel_timeout(link);
1390
		} else {
1391
			io_cqring_fill_event(link, -ECANCELED);
1392
			__io_double_put_req(link);
1393
		}
1394
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
1395
	}
1396 1397 1398 1399

	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1400 1401
}

1402
static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
J
Jens Axboe 已提交
1403
{
1404
	if (likely(!(req->flags & REQ_F_LINK)))
1405 1406
		return;

J
Jens Axboe 已提交
1407 1408 1409 1410 1411 1412
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
1413 1414
	if (req->flags & REQ_F_FAIL_LINK) {
		io_fail_links(req);
1415 1416
	} else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
			REQ_F_LINK_TIMEOUT) {
1417 1418 1419 1420 1421 1422
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;

		/*
		 * If this is a timeout link, we could be racing with the
		 * timeout timer. Grab the completion lock for this case to
1423
		 * protect against that.
1424 1425 1426 1427 1428 1429
		 */
		spin_lock_irqsave(&ctx->completion_lock, flags);
		io_req_link_next(req, nxt);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
		io_req_link_next(req, nxt);
J
Jens Axboe 已提交
1430
	}
1431
}
J
Jens Axboe 已提交
1432

1433 1434
static void io_free_req(struct io_kiocb *req)
{
1435 1436 1437
	struct io_kiocb *nxt = NULL;

	io_req_find_next(req, &nxt);
1438
	__io_free_req(req);
1439 1440 1441

	if (nxt)
		io_queue_async_work(nxt);
1442 1443
}

1444 1445 1446 1447
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
1448
__attribute__((nonnull))
1449
static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
1450
{
1451
	io_req_find_next(req, nxtptr);
1452

1453
	if (refcount_dec_and_test(&req->refs))
1454
		__io_free_req(req);
J
Jens Axboe 已提交
1455 1456
}

1457 1458 1459 1460
static void io_put_req(struct io_kiocb *req)
{
	if (refcount_dec_and_test(&req->refs))
		io_free_req(req);
J
Jens Axboe 已提交
1461 1462
}

1463 1464 1465 1466 1467
/*
 * Must only be used if we don't need to care about links, usually from
 * within the completion handling itself.
 */
static void __io_double_put_req(struct io_kiocb *req)
1468 1469 1470 1471 1472 1473
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		__io_free_req(req);
}

1474 1475 1476 1477 1478 1479 1480
static void io_double_put_req(struct io_kiocb *req)
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		io_free_req(req);
}

1481
static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
1482
{
1483 1484
	struct io_rings *rings = ctx->rings;

1485 1486 1487 1488 1489 1490 1491 1492
	if (test_bit(0, &ctx->cq_check_overflow)) {
		/*
		 * noflush == true is from the waitqueue handler, just ensure
		 * we wake up the task, and the next invocation will flush the
		 * entries. We cannot safely to it from here.
		 */
		if (noflush && !list_empty(&ctx->cq_overflow_list))
			return -1U;
1493

1494 1495
		io_cqring_overflow_flush(ctx, false);
	}
1496

1497 1498
	/* See comment at the top of this file */
	smp_rmb();
1499
	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
1500 1501
}

1502 1503 1504 1505 1506 1507 1508 1509
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

1510
static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
1511
{
1512 1513
	if ((req->flags & REQ_F_LINK) || io_is_fallback_req(req))
		return false;
1514

1515 1516 1517 1518 1519 1520 1521
	if (!(req->flags & REQ_F_FIXED_FILE) || req->io)
		rb->need_iter++;

	rb->reqs[rb->to_free++] = req;
	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		io_free_req_many(req->ctx, rb);
	return true;
1522 1523
}

J
Jens Axboe 已提交
1524 1525 1526 1527 1528 1529
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
1530
	struct req_batch rb;
J
Jens Axboe 已提交
1531 1532
	struct io_kiocb *req;

1533
	rb.to_free = rb.need_iter = 0;
J
Jens Axboe 已提交
1534 1535 1536 1537
	while (!list_empty(done)) {
		req = list_first_entry(done, struct io_kiocb, list);
		list_del(&req->list);

1538
		io_cqring_fill_event(req, req->result);
J
Jens Axboe 已提交
1539 1540
		(*nr_events)++;

1541 1542 1543
		if (refcount_dec_and_test(&req->refs) &&
		    !io_req_multi_free(&rb, req))
			io_free_req(req);
J
Jens Axboe 已提交
1544 1545
	}

J
Jens Axboe 已提交
1546
	io_commit_cqring(ctx);
1547
	io_free_req_many(ctx, &rb);
J
Jens Axboe 已提交
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565
}

static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
1566
		struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595

		/*
		 * Move completed entries to our local list. If we find a
		 * request that requires polling, break out and complete
		 * the done list first, if we have entries there.
		 */
		if (req->flags & REQ_F_IOPOLL_COMPLETED) {
			list_move_tail(&req->list, &done);
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
1596
 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
J
Jens Axboe 已提交
1597 1598 1599 1600 1601 1602
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
1603
	while (!list_empty(&ctx->poll_list) && !need_resched()) {
J
Jens Axboe 已提交
1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
	while (!list_empty(&ctx->poll_list)) {
		unsigned int nr_events = 0;

		io_iopoll_getevents(ctx, &nr_events, 1);
1630 1631 1632 1633 1634 1635

		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
		 */
		cond_resched();
J
Jens Axboe 已提交
1636 1637 1638 1639
	}
	mutex_unlock(&ctx->uring_lock);
}

1640 1641
static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			    long min)
J
Jens Axboe 已提交
1642
{
1643
	int iters = 0, ret = 0;
1644

J
Jens Axboe 已提交
1645 1646 1647
	do {
		int tmin = 0;

1648 1649 1650 1651 1652
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
1653
		if (io_cqring_events(ctx, false))
1654 1655
			break;

1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
			mutex_lock(&ctx->uring_lock);
		}

J
Jens Axboe 已提交
1671 1672 1673 1674 1675 1676 1677 1678 1679
		if (*nr_events < min)
			tmin = min - *nr_events;

		ret = io_iopoll_getevents(ctx, nr_events, tmin);
		if (ret <= 0)
			break;
		ret = 0;
	} while (min && !*nr_events && !need_resched());

1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694
	return ret;
}

static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			   long min)
{
	int ret;

	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
	ret = __io_iopoll_check(ctx, nr_events, min);
1695
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
1696 1697 1698
	return ret;
}

1699
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
1700
{
1701 1702 1703 1704 1705 1706
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
1707

1708
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
1709
	}
1710
	file_end_write(req->file);
J
Jens Axboe 已提交
1711 1712
}

J
Jens Axboe 已提交
1713 1714 1715 1716 1717 1718
static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}

1719
static void io_complete_rw_common(struct kiocb *kiocb, long res)
J
Jens Axboe 已提交
1720
{
1721
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
1722

1723 1724
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1725

J
Jens Axboe 已提交
1726 1727
	if (res != req->result)
		req_set_fail_links(req);
1728
	io_cqring_add_event(req, res);
1729 1730 1731 1732
}

static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
1733
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1734 1735

	io_complete_rw_common(kiocb, res);
1736
	io_put_req(req);
J
Jens Axboe 已提交
1737 1738
}

1739 1740
static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
{
1741
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1742
	struct io_kiocb *nxt = NULL;
1743 1744

	io_complete_rw_common(kiocb, res);
1745 1746 1747
	io_put_req_find_next(req, &nxt);

	return nxt;
J
Jens Axboe 已提交
1748 1749
}

J
Jens Axboe 已提交
1750 1751
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
1752
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
1753

1754 1755
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1756

J
Jens Axboe 已提交
1757 1758
	if (res != req->result)
		req_set_fail_links(req);
J
Jens Axboe 已提交
1759
	req->result = res;
J
Jens Axboe 已提交
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
	if (res != -EAGAIN)
		req->flags |= REQ_F_IOPOLL_COMPLETED;
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->poll_list)) {
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

		list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
						list);
1786
		if (list_req->file != req->file)
J
Jens Axboe 已提交
1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
	if (req->flags & REQ_F_IOPOLL_COMPLETED)
		list_add(&req->list, &ctx->poll_list);
	else
		list_add_tail(&req->list, &ctx->poll_list);
}

1800
static void io_file_put(struct io_submit_state *state)
1801
{
1802
	if (state->file) {
1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826
		int diff = state->has_refs - state->used_refs;

		if (diff)
			fput_many(state->file, diff);
		state->file = NULL;
	}
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
static struct file *io_file_get(struct io_submit_state *state, int fd)
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
			state->used_refs++;
			state->ios_left--;
			return state->file;
		}
1827
		io_file_put(state);
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->has_refs = state->ios_left;
	state->used_refs = 1;
	state->ios_left--;
	return state->file;
}

J
Jens Axboe 已提交
1840 1841 1842 1843 1844 1845 1846 1847 1848
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
static bool io_file_supports_async(struct file *file)
{
	umode_t mode = file_inode(file)->i_mode;

1849
	if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
J
Jens Axboe 已提交
1850 1851 1852 1853 1854 1855 1856
		return true;
	if (S_ISREG(mode) && file->f_op != &io_uring_fops)
		return true;

	return false;
}

1857 1858
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		      bool force_nonblock)
J
Jens Axboe 已提交
1859
{
J
Jens Axboe 已提交
1860
	struct io_ring_ctx *ctx = req->ctx;
1861
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
1862 1863
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
1864

J
Jens Axboe 已提交
1865 1866
	if (!req->file)
		return -EBADF;
J
Jens Axboe 已提交
1867

1868 1869 1870
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

J
Jens Axboe 已提交
1871
	kiocb->ki_pos = READ_ONCE(sqe->off);
1872 1873 1874 1875
	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
		req->flags |= REQ_F_CUR_POS;
		kiocb->ki_pos = req->file->f_pos;
	}
J
Jens Axboe 已提交
1876 1877 1878 1879 1880 1881 1882
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
1883
			return ret;
J
Jens Axboe 已提交
1884 1885 1886 1887 1888 1889 1890

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
J
Jens Axboe 已提交
1891
		return ret;
1892 1893

	/* don't allow async punt if RWF_NOWAIT was requested */
1894 1895
	if ((kiocb->ki_flags & IOCB_NOWAIT) ||
	    (req->file->f_flags & O_NONBLOCK))
1896 1897 1898
		req->flags |= REQ_F_NOWAIT;

	if (force_nonblock)
J
Jens Axboe 已提交
1899
		kiocb->ki_flags |= IOCB_NOWAIT;
1900

J
Jens Axboe 已提交
1901 1902 1903
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
1904
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
1905

J
Jens Axboe 已提交
1906 1907
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
1908
		req->result = 0;
J
Jens Axboe 已提交
1909
	} else {
J
Jens Axboe 已提交
1910 1911
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
1912 1913
		kiocb->ki_complete = io_complete_rw;
	}
1914

1915 1916
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
1917 1918
	/* we own ->private, reuse it for the buffer index */
	req->rw.kiocb.private = (void *) (unsigned long)
1919
					READ_ONCE(sqe->buf_index);
J
Jens Axboe 已提交
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

1944 1945 1946
static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
		       bool in_async)
{
1947 1948 1949 1950
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

	if (req->flags & REQ_F_CUR_POS)
		req->file->f_pos = kiocb->ki_pos;
1951
	if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
1952 1953 1954 1955 1956
		*nxt = __io_complete_rw(kiocb, ret);
	else
		io_rw_done(kiocb, ret);
}

1957
static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
1958
			       struct iov_iter *iter)
1959
{
1960 1961
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
1962 1963 1964 1965 1966 1967 1968 1969 1970
	struct io_mapped_ubuf *imu;
	unsigned index, buf_index;
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

1971
	buf_index = (unsigned long) req->rw.kiocb.private;
1972 1973 1974 1975 1976
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
1977
	buf_addr = req->rw.addr;
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
2023
			iter->count -= bvec->bv_len + offset;
2024 2025 2026 2027
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

2028
	return len;
2029 2030
}

2031 2032
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter)
J
Jens Axboe 已提交
2033
{
2034 2035
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
2036 2037
	u8 opcode;

2038
	opcode = req->opcode;
2039
	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2040
		*iovec = NULL;
2041
		return io_import_fixed(req, rw, iter);
2042
	}
J
Jens Axboe 已提交
2043

2044 2045 2046 2047
	/* buffer index only valid with fixed read/write */
	if (req->rw.kiocb.private)
		return -EINVAL;

2048 2049 2050 2051 2052 2053 2054
	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
		ssize_t ret;
		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
		return ret;
	}

2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
	if (req->io) {
		struct io_async_rw *iorw = &req->io->rw;

		*iovec = iorw->iov;
		iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
		if (iorw->iov == iorw->fast_iov)
			*iovec = NULL;
		return iorw->size;
	}

2065
	if (!req->has_user)
J
Jens Axboe 已提交
2066 2067 2068
		return -EFAULT;

#ifdef CONFIG_COMPAT
2069
	if (req->ctx->compat)
J
Jens Axboe 已提交
2070 2071 2072 2073 2074 2075 2076
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

2077
/*
2078 2079
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
2080
 */
2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
2097
		struct iovec iovec;
2098 2099
		ssize_t nr;

2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

2110 2111 2112 2113 2114 2115 2116 2117
		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, &kiocb->ki_pos);
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, &kiocb->ki_pos);
		}

2118 2119 2120
		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

2135
static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
			  struct iovec *iovec, struct iovec *fast_iov,
			  struct iov_iter *iter)
{
	req->io->rw.nr_segs = iter->nr_segs;
	req->io->rw.size = io_size;
	req->io->rw.iov = iovec;
	if (!req->io->rw.iov) {
		req->io->rw.iov = req->io->rw.fast_iov;
		memcpy(req->io->rw.iov, fast_iov,
			sizeof(struct iovec) * iter->nr_segs);
	}
}

2149
static int io_alloc_async_ctx(struct io_kiocb *req)
2150
{
2151 2152
	if (!io_op_defs[req->opcode].async_ctx)
		return 0;
2153
	req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
2154
	return req->io == NULL;
2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
}

static void io_rw_async(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct iovec *iov = NULL;

	if (req->io->rw.iov != req->io->rw.fast_iov)
		iov = req->io->rw.iov;
	io_wq_submit_work(workptr);
	kfree(iov);
}

static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
			     struct iovec *iovec, struct iovec *fast_iov,
			     struct iov_iter *iter)
{
2172
	if (!io_op_defs[req->opcode].async_ctx)
2173
		return 0;
2174 2175 2176
	if (!req->io) {
		if (io_alloc_async_ctx(req))
			return -ENOMEM;
2177

2178 2179
		io_req_map_rw(req, io_size, iovec, fast_iov, iter);
	}
2180 2181
	req->work.func = io_rw_async;
	return 0;
2182 2183
}

2184 2185
static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
2186
{
2187 2188
	struct io_async_ctx *io;
	struct iov_iter iter;
2189 2190
	ssize_t ret;

2191 2192 2193
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2194

2195 2196
	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;
2197

2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
	if (!req->io)
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
	ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2211 2212
}

2213
static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
2214
		   bool force_nonblock)
J
Jens Axboe 已提交
2215 2216
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2217
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2218
	struct iov_iter iter;
2219
	size_t iov_count;
2220
	ssize_t io_size, ret;
J
Jens Axboe 已提交
2221

2222
	ret = io_import_iovec(READ, req, &iovec, &iter);
2223 2224
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2225

2226 2227
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2228
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2229

2230
	req->result = 0;
2231
	io_size = ret;
J
Jens Axboe 已提交
2232
	if (req->flags & REQ_F_LINK)
2233 2234 2235 2236 2237 2238
		req->result = io_size;

	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
2239
	if (force_nonblock && !io_file_supports_async(req->file)) {
2240 2241 2242
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}
J
Jens Axboe 已提交
2243

2244
	iov_count = iov_iter_count(&iter);
2245
	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2246 2247 2248
	if (!ret) {
		ssize_t ret2;

2249 2250
		if (req->file->f_op->read_iter)
			ret2 = call_read_iter(req->file, kiocb, &iter);
2251
		else
2252
			ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
2253

2254
		/* Catch -EAGAIN return for forced non-blocking submission */
2255
		if (!force_nonblock || ret2 != -EAGAIN) {
2256
			kiocb_done(kiocb, ret2, nxt, req->in_async);
2257 2258
		} else {
copy_iov:
2259
			ret = io_setup_async_rw(req, io_size, iovec,
2260 2261 2262 2263 2264
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
J
Jens Axboe 已提交
2265
	}
2266
out_free:
2267 2268
	if (!io_wq_current_is_worker())
		kfree(iovec);
J
Jens Axboe 已提交
2269 2270 2271
	return ret;
}

2272 2273
static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 bool force_nonblock)
2274
{
2275 2276
	struct io_async_ctx *io;
	struct iov_iter iter;
2277 2278
	ssize_t ret;

2279 2280 2281
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2282

2283 2284
	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
		return -EBADF;
2285

2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298
	if (!req->io)
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
	ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2299 2300
}

2301
static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
2302
		    bool force_nonblock)
J
Jens Axboe 已提交
2303 2304
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2305
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2306
	struct iov_iter iter;
2307
	size_t iov_count;
2308
	ssize_t ret, io_size;
J
Jens Axboe 已提交
2309

2310
	ret = io_import_iovec(WRITE, req, &iovec, &iter);
2311 2312
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2313

2314 2315
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2316
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2317

2318
	req->result = 0;
2319
	io_size = ret;
J
Jens Axboe 已提交
2320
	if (req->flags & REQ_F_LINK)
2321
		req->result = io_size;
J
Jens Axboe 已提交
2322

2323 2324 2325 2326 2327 2328 2329 2330
	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
	if (force_nonblock && !io_file_supports_async(req->file)) {
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}
2331

2332 2333 2334
	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
2335
		goto copy_iov;
2336

2337
	iov_count = iov_iter_count(&iter);
2338
	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2339
	if (!ret) {
2340 2341
		ssize_t ret2;

J
Jens Axboe 已提交
2342 2343 2344 2345 2346 2347 2348
		/*
		 * Open-code file_start_write here to grab freeze protection,
		 * which will be released by another thread in
		 * io_complete_rw().  Fool lockdep by telling it the lock got
		 * released so that it doesn't complain about the held lock when
		 * we return to userspace.
		 */
2349
		if (req->flags & REQ_F_ISREG) {
2350
			__sb_start_write(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
2351
						SB_FREEZE_WRITE, true);
2352
			__sb_writers_release(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
2353 2354 2355
						SB_FREEZE_WRITE);
		}
		kiocb->ki_flags |= IOCB_WRITE;
2356

2357 2358
		if (req->file->f_op->write_iter)
			ret2 = call_write_iter(req->file, kiocb, &iter);
2359
		else
2360
			ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
2361
		if (!force_nonblock || ret2 != -EAGAIN) {
2362
			kiocb_done(kiocb, ret2, nxt, req->in_async);
2363 2364
		} else {
copy_iov:
2365
			ret = io_setup_async_rw(req, io_size, iovec,
2366 2367 2368 2369 2370
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
J
Jens Axboe 已提交
2371
	}
2372
out_free:
2373 2374
	if (!io_wq_current_is_worker())
		kfree(iovec);
J
Jens Axboe 已提交
2375 2376 2377 2378 2379 2380
	return ret;
}

/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
2381
static int io_nop(struct io_kiocb *req)
J
Jens Axboe 已提交
2382 2383 2384
{
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
2385 2386 2387
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

2388
	io_cqring_add_event(req, 0);
2389
	io_put_req(req);
J
Jens Axboe 已提交
2390 2391 2392
	return 0;
}

2393
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
C
Christoph Hellwig 已提交
2394
{
J
Jens Axboe 已提交
2395
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
2396

J
Jens Axboe 已提交
2397 2398
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
2399

J
Jens Axboe 已提交
2400
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
2401
		return -EINVAL;
2402
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
2403 2404
		return -EINVAL;

2405 2406 2407 2408 2409 2410
	req->sync.flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
C
Christoph Hellwig 已提交
2411 2412 2413
	return 0;
}

2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
static bool io_req_cancelled(struct io_kiocb *req)
{
	if (req->work.flags & IO_WQ_WORK_CANCEL) {
		req_set_fail_links(req);
		io_cqring_add_event(req, -ECANCELED);
		io_put_req(req);
		return true;
	}

	return false;
}

2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
static void io_link_work_cb(struct io_wq_work **workptr)
{
	struct io_wq_work *work = *workptr;
	struct io_kiocb *link = work->data;

	io_queue_linked_timeout(link);
	work->func = io_wq_submit_work;
}

static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
{
	struct io_kiocb *link;

	io_prep_async_work(nxt, &link);
	*workptr = &nxt->work;
	if (link) {
		nxt->work.flags |= IO_WQ_WORK_CB;
		nxt->work.func = io_link_work_cb;
		nxt->work.data = link;
	}
}

2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
static void io_fsync_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	loff_t end = req->sync.off + req->sync.len;
	struct io_kiocb *nxt = NULL;
	int ret;

	if (io_req_cancelled(req))
		return;

2458
	ret = vfs_fsync_range(req->file, req->sync.off,
2459 2460 2461 2462 2463 2464 2465
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
2466
		io_wq_assign_next(workptr, nxt);
2467 2468
}

2469 2470
static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
C
Christoph Hellwig 已提交
2471
{
2472
	struct io_wq_work *work, *old_work;
C
Christoph Hellwig 已提交
2473 2474

	/* fsync always requires a blocking context */
2475 2476 2477
	if (force_nonblock) {
		io_put_req(req);
		req->work.func = io_fsync_finish;
C
Christoph Hellwig 已提交
2478
		return -EAGAIN;
2479
	}
C
Christoph Hellwig 已提交
2480

2481 2482 2483 2484
	work = old_work = &req->work;
	io_fsync_finish(&work);
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);
C
Christoph Hellwig 已提交
2485 2486 2487
	return 0;
}

2488
static void io_fallocate_finish(struct io_wq_work **workptr)
2489 2490 2491 2492 2493
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;
	int ret;

2494 2495
	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
				req->sync.len);
2496 2497 2498 2499 2500
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
2501
		io_wq_assign_next(workptr, nxt);
2502 2503
}

2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517
static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
	return 0;
}

static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
			bool force_nonblock)
2518
{
2519
	struct io_wq_work *work, *old_work;
2520

2521
	/* fallocate always requiring blocking context */
2522 2523
	if (force_nonblock) {
		io_put_req(req);
2524
		req->work.func = io_fallocate_finish;
2525
		return -EAGAIN;
2526
	}
2527

2528
	work = old_work = &req->work;
2529
	io_fallocate_finish(&work);
2530 2531
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);
2532

2533 2534 2535
	return 0;
}

2536
static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2537
{
2538
	const char __user *fname;
2539
	int ret;
2540

2541 2542
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
2543

2544
	req->open.dfd = READ_ONCE(sqe->fd);
2545
	req->open.how.mode = READ_ONCE(sqe->len);
2546
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2547
	req->open.how.flags = READ_ONCE(sqe->open_flags);
2548

2549
	req->open.filename = getname(fname);
2550 2551 2552 2553 2554
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}
2555

2556
	return 0;
2557 2558
}

2559
static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
2560
{
2561 2562 2563
	struct open_how __user *how;
	const char __user *fname;
	size_t len;
J
Jens Axboe 已提交
2564 2565
	int ret;

2566
	if (sqe->ioprio || sqe->buf_index)
J
Jens Axboe 已提交
2567 2568
		return -EINVAL;

2569 2570 2571 2572
	req->open.dfd = READ_ONCE(sqe->fd);
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	len = READ_ONCE(sqe->len);
J
Jens Axboe 已提交
2573

2574 2575
	if (len < OPEN_HOW_SIZE_VER0)
		return -EINVAL;
2576

2577 2578 2579 2580
	ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
					len);
	if (ret)
		return ret;
2581

2582 2583
	if (!(req->open.how.flags & O_PATH) && force_o_largefile())
		req->open.how.flags |= O_LARGEFILE;
J
Jens Axboe 已提交
2584

2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
	req->open.filename = getname(fname);
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}

	return 0;
}

static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
2597 2598 2599 2600 2601
{
	struct open_flags op;
	struct file *file;
	int ret;

2602
	if (force_nonblock)
2603 2604
		return -EAGAIN;

2605
	ret = build_open_flags(&req->open.how, &op);
2606 2607 2608
	if (ret)
		goto err;

2609
	ret = get_unused_fd_flags(req->open.how.flags);
2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
	if (ret < 0)
		goto err;

	file = do_filp_open(req->open.dfd, req->open.filename, &op);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
	} else {
		fsnotify_open(file);
		fd_install(ret, file);
	}
err:
	putname(req->open.filename);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2630 2631 2632 2633 2634 2635 2636
static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
	req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
	return io_openat2(req, nxt, force_nonblock);
}

2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682
static int io_epoll_ctl_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_EPOLL)
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->epoll.epfd = READ_ONCE(sqe->fd);
	req->epoll.op = READ_ONCE(sqe->len);
	req->epoll.fd = READ_ONCE(sqe->off);

	if (ep_op_has_event(req->epoll.op)) {
		struct epoll_event __user *ev;

		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
		if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
			return -EFAULT;
	}

	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,
			bool force_nonblock)
{
#if defined(CONFIG_EPOLL)
	struct io_epoll *ie = &req->epoll;
	int ret;

	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	if (sqe->ioprio || sqe->buf_index || sqe->off)
		return -EINVAL;

	req->madvise.addr = READ_ONCE(sqe->addr);
	req->madvise.len = READ_ONCE(sqe->len);
	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747
static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr)
		return -EINVAL;

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}

static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
	struct io_fadvise *fa = &req->fadvise;
	int ret;

	/* DONTNEED may block, others _should_ not */
	if (fa->advice == POSIX_FADV_DONTNEED && force_nonblock)
		return -EAGAIN;

	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2748 2749
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
2750
	const char __user *fname;
2751 2752 2753 2754 2755 2756 2757 2758
	unsigned lookup_flags;
	int ret;

	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->open.dfd = READ_ONCE(sqe->fd);
	req->open.mask = READ_ONCE(sqe->len);
2759
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2760
	req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
2761
	req->open.how.flags = READ_ONCE(sqe->statx_flags);
2762

2763
	if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags))
2764 2765
		return -EINVAL;

2766
	req->open.filename = getname_flags(fname, lookup_flags, NULL);
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}

	return 0;
}

static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	struct io_open *ctx = &req->open;
	unsigned lookup_flags;
	struct path path;
	struct kstat stat;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

2788
	if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799
		return -EINVAL;

retry:
	/* filename_lookup() drops it, keep a reference */
	ctx->filename->refcnt++;

	ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
				NULL);
	if (ret)
		goto err;

2800
	ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816
	path_put(&path);
	if (retry_estale(ret, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
	if (!ret)
		ret = cp_statx(&stat, ctx->buffer);
err:
	putname(ctx->filename);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832
static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	/*
	 * If we queue this for async, it must not be cancellable. That would
	 * leave the 'file' in an undeterminate state.
	 */
	req->work.flags |= IO_WQ_WORK_NO_CANCEL;

	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
	    sqe->rw_flags || sqe->buf_index)
		return -EINVAL;
	if (sqe->flags & IOSQE_FIXED_FILE)
		return -EINVAL;

	req->close.fd = READ_ONCE(sqe->fd);
	if (req->file->f_op == &io_uring_fops ||
P
Pavel Begunkov 已提交
2833
	    req->close.fd == req->ctx->ring_fd)
2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874
		return -EBADF;

	return 0;
}

static void io_close_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;

	/* Invoked with files, we need to do the close */
	if (req->work.files) {
		int ret;

		ret = filp_close(req->close.put_file, req->work.files);
		if (ret < 0) {
			req_set_fail_links(req);
		}
		io_cqring_add_event(req, ret);
	}

	fput(req->close.put_file);

	/* we bypassed the re-issue, drop the submission reference */
	io_put_req(req);
	io_put_req_find_next(req, &nxt);
	if (nxt)
		io_wq_assign_next(workptr, nxt);
}

static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	int ret;

	req->close.put_file = NULL;
	ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
	if (ret < 0)
		return ret;

	/* if the file has a flush method, be safe and punt to async */
2875
	if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
		goto eagain;

	/*
	 * No ->flush(), safely close from here and just punt the
	 * fput() to async context.
	 */
	ret = filp_close(req->close.put_file, current->files);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);

	if (io_wq_current_is_worker()) {
		struct io_wq_work *old_work, *work;

		old_work = work = &req->work;
		io_close_finish(&work);
		if (work && work != old_work)
			*nxt = container_of(work, struct io_kiocb, work);
		return 0;
	}

eagain:
	req->work.func = io_close_finish;
	return -EAGAIN;
}

2903
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
	req->sync.flags = READ_ONCE(sqe->sync_range_flags);
	return 0;
}

static void io_sync_file_range_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;
	int ret;

	if (io_req_cancelled(req))
		return;

2930
	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
2931 2932 2933 2934 2935 2936
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
2937
		io_wq_assign_next(workptr, nxt);
2938 2939
}

2940
static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
2941 2942
			      bool force_nonblock)
{
2943
	struct io_wq_work *work, *old_work;
2944 2945

	/* sync_file_range always requires a blocking context */
2946 2947 2948
	if (force_nonblock) {
		io_put_req(req);
		req->work.func = io_sync_file_range_finish;
2949
		return -EAGAIN;
2950
	}
2951

2952 2953 2954 2955
	work = old_work = &req->work;
	io_sync_file_range_finish(&work);
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);
2956 2957 2958
	return 0;
}

2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971
#if defined(CONFIG_NET)
static void io_sendrecv_async(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct iovec *iov = NULL;

	if (req->io->rw.iov != req->io->rw.fast_iov)
		iov = req->io->msg.iov;
	io_wq_submit_work(workptr);
	kfree(iov);
}
#endif

2972
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2973
{
J
Jens Axboe 已提交
2974
#if defined(CONFIG_NET)
2975
	struct io_sr_msg *sr = &req->sr_msg;
2976
	struct io_async_ctx *io = req->io;
2977

2978 2979
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
2980
	sr->len = READ_ONCE(sqe->len);
2981

2982
	if (!io || req->opcode == IORING_OP_SEND)
2983 2984
		return 0;

2985
	io->msg.iov = io->msg.fast_iov;
2986
	return sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
2987
					&io->msg.iov);
2988
#else
2989
	return -EOPNOTSUPP;
2990 2991 2992
#endif
}

2993 2994
static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
J
Jens Axboe 已提交
2995
{
2996
#if defined(CONFIG_NET)
2997
	struct io_async_msghdr *kmsg = NULL;
J
Jens Axboe 已提交
2998 2999 3000 3001 3002 3003 3004 3005
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
3006
		struct io_async_ctx io;
3007
		struct sockaddr_storage addr;
J
Jens Axboe 已提交
3008 3009
		unsigned flags;

3010
		if (req->io) {
3011 3012 3013 3014 3015 3016
			kmsg = &req->io->msg;
			kmsg->msg.msg_name = &addr;
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
3017
		} else {
3018 3019
			struct io_sr_msg *sr = &req->sr_msg;

3020 3021
			kmsg = &io.msg;
			kmsg->msg.msg_name = &addr;
3022 3023 3024 3025

			io.msg.iov = io.msg.fast_iov;
			ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.iov);
3026
			if (ret)
3027
				return ret;
3028
		}
J
Jens Axboe 已提交
3029

3030 3031 3032 3033 3034 3035
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3036
		ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3037
		if (force_nonblock && ret == -EAGAIN) {
3038 3039 3040 3041 3042 3043
			if (req->io)
				return -EAGAIN;
			if (io_alloc_async_ctx(req))
				return -ENOMEM;
			memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
			req->work.func = io_sendrecv_async;
3044
			return -EAGAIN;
3045
		}
3046 3047
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
3048 3049
	}

3050
	if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
3051
		kfree(kmsg->iov);
3052
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3053 3054
	if (ret < 0)
		req_set_fail_links(req);
3055
	io_put_req_find_next(req, nxt);
3056
	return 0;
3057 3058
#else
	return -EOPNOTSUPP;
J
Jens Axboe 已提交
3059
#endif
3060
}
J
Jens Axboe 已提交
3061

3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094
static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

		ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
						&msg.msg_iter);
		if (ret)
			return ret;

		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3095 3096
		msg.msg_flags = flags;
		ret = sock_sendmsg(sock, &msg);
3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3113 3114
static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
3115 3116
{
#if defined(CONFIG_NET)
3117
	struct io_sr_msg *sr = &req->sr_msg;
3118 3119 3120 3121
	struct io_async_ctx *io = req->io;

	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3122
	sr->len = READ_ONCE(sqe->len);
3123

3124
	if (!io || req->opcode == IORING_OP_RECV)
3125
		return 0;
3126

3127
	io->msg.iov = io->msg.fast_iov;
3128
	return recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3129
					&io->msg.uaddr, &io->msg.iov);
J
Jens Axboe 已提交
3130
#else
3131
	return -EOPNOTSUPP;
J
Jens Axboe 已提交
3132 3133 3134
#endif
}

3135 3136
static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
J
Jens Axboe 已提交
3137 3138
{
#if defined(CONFIG_NET)
3139
	struct io_async_msghdr *kmsg = NULL;
3140 3141 3142 3143 3144 3145 3146 3147
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
3148
		struct io_async_ctx io;
3149 3150 3151 3152
		struct sockaddr_storage addr;
		unsigned flags;

		if (req->io) {
3153 3154 3155 3156 3157 3158
			kmsg = &req->io->msg;
			kmsg->msg.msg_name = &addr;
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
3159
		} else {
3160 3161
			struct io_sr_msg *sr = &req->sr_msg;

3162 3163
			kmsg = &io.msg;
			kmsg->msg.msg_name = &addr;
3164 3165 3166 3167 3168

			io.msg.iov = io.msg.fast_iov;
			ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.uaddr,
					&io.msg.iov);
3169
			if (ret)
3170
				return ret;
3171 3172
		}

3173 3174 3175 3176 3177 3178 3179 3180
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

		ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
						kmsg->uaddr, flags);
3181
		if (force_nonblock && ret == -EAGAIN) {
3182 3183 3184 3185 3186 3187
			if (req->io)
				return -EAGAIN;
			if (io_alloc_async_ctx(req))
				return -ENOMEM;
			memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
			req->work.func = io_sendrecv_async;
3188
			return -EAGAIN;
3189 3190 3191 3192 3193
		}
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

3194
	if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
3195
		kfree(kmsg->iov);
3196
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3197 3198
	if (ret < 0)
		req_set_fail_links(req);
3199 3200
	io_put_req_find_next(req, nxt);
	return 0;
J
Jens Axboe 已提交
3201 3202 3203 3204
#else
	return -EOPNOTSUPP;
#endif
}
3205

3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240
static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

		ret = import_single_range(READ, sr->buf, sr->len, &iov,
						&msg.msg_iter);
		if (ret)
			return ret;

		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;
		msg.msg_iocb = NULL;
		msg.msg_flags = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3241
		ret = sock_recvmsg(sock, &msg, flags);
3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


3259
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3260 3261
{
#if defined(CONFIG_NET)
3262 3263
	struct io_accept *accept = &req->accept;

3264 3265
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
3266
	if (sqe->ioprio || sqe->len || sqe->buf_index)
3267 3268
		return -EINVAL;

3269 3270
	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3271 3272 3273 3274 3275 3276
	accept->flags = READ_ONCE(sqe->accept_flags);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}
3277

3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289
#if defined(CONFIG_NET)
static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
		       bool force_nonblock)
{
	struct io_accept *accept = &req->accept;
	unsigned file_flags;
	int ret;

	file_flags = force_nonblock ? O_NONBLOCK : 0;
	ret = __sys_accept4_file(req->file, file_flags, accept->addr,
					accept->addr_len, accept->flags);
	if (ret == -EAGAIN && force_nonblock)
3290
		return -EAGAIN;
3291 3292
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
J
Jens Axboe 已提交
3293 3294
	if (ret < 0)
		req_set_fail_links(req);
3295
	io_cqring_add_event(req, ret);
3296
	io_put_req_find_next(req, nxt);
3297
	return 0;
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308
}

static void io_accept_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;

	if (io_req_cancelled(req))
		return;
	__io_accept(req, &nxt, false);
	if (nxt)
3309
		io_wq_assign_next(workptr, nxt);
3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325
}
#endif

static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
#if defined(CONFIG_NET)
	int ret;

	ret = __io_accept(req, nxt, force_nonblock);
	if (ret == -EAGAIN && force_nonblock) {
		req->work.func = io_accept_finish;
		io_put_req(req);
		return -EAGAIN;
	}
	return 0;
J
Jens Axboe 已提交
3326 3327 3328 3329
#else
	return -EOPNOTSUPP;
#endif
}
3330

3331
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3332 3333
{
#if defined(CONFIG_NET)
3334 3335
	struct io_connect *conn = &req->connect;
	struct io_async_ctx *io = req->io;
3336

3337 3338 3339 3340 3341
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

3342 3343 3344 3345 3346 3347 3348
	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	conn->addr_len =  READ_ONCE(sqe->addr2);

	if (!io)
		return 0;

	return move_addr_to_kernel(conn->addr, conn->addr_len,
3349
					&io->connect.address);
3350
#else
3351
	return -EOPNOTSUPP;
3352 3353 3354
#endif
}

3355 3356
static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
3357 3358
{
#if defined(CONFIG_NET)
3359
	struct io_async_ctx __io, *io;
3360
	unsigned file_flags;
3361
	int ret;
3362

3363 3364 3365
	if (req->io) {
		io = req->io;
	} else {
3366 3367 3368
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
3369 3370 3371 3372 3373
		if (ret)
			goto out;
		io = &__io;
	}

3374 3375 3376 3377
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
3378
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
3379 3380 3381
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
3382 3383 3384
			ret = -ENOMEM;
			goto out;
		}
3385
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
3386
		return -EAGAIN;
3387
	}
3388 3389
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
3390
out:
J
Jens Axboe 已提交
3391 3392
	if (ret < 0)
		req_set_fail_links(req);
3393 3394 3395 3396 3397 3398 3399 3400
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3401 3402 3403 3404 3405 3406
static void io_poll_remove_one(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = &req->poll;

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
3407 3408
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
3409
		io_queue_async_work(req);
3410 3411
	}
	spin_unlock(&poll->head->lock);
3412
	hash_del(&req->hash_node);
3413 3414 3415 3416
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
3417
	struct hlist_node *tmp;
3418
	struct io_kiocb *req;
3419
	int i;
3420 3421

	spin_lock_irq(&ctx->completion_lock);
3422 3423 3424 3425 3426 3427
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node)
			io_poll_remove_one(req);
3428 3429 3430 3431
	}
	spin_unlock_irq(&ctx->completion_lock);
}

3432 3433
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
3434
	struct hlist_head *list;
3435 3436
	struct io_kiocb *req;

3437 3438 3439
	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	hlist_for_each_entry(req, list, hash_node) {
		if (sqe_addr == req->user_data) {
3440 3441 3442
			io_poll_remove_one(req);
			return 0;
		}
3443 3444 3445 3446 3447
	}

	return -ENOENT;
}

3448 3449
static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	req->poll.addr = READ_ONCE(sqe->addr);
	return 0;
}

3461 3462 3463 3464
/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
3465
static int io_poll_remove(struct io_kiocb *req)
3466 3467
{
	struct io_ring_ctx *ctx = req->ctx;
3468
	u64 addr;
3469
	int ret;
3470

3471
	addr = req->poll.addr;
3472
	spin_lock_irq(&ctx->completion_lock);
3473
	ret = io_poll_cancel(ctx, addr);
3474 3475
	spin_unlock_irq(&ctx->completion_lock);

3476
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3477 3478
	if (ret < 0)
		req_set_fail_links(req);
3479
	io_put_req(req);
3480 3481 3482
	return 0;
}

3483
static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
3484
{
3485 3486
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
3487
	req->poll.done = true;
3488 3489 3490 3491
	if (error)
		io_cqring_fill_event(req, error);
	else
		io_cqring_fill_event(req, mangle_poll(mask));
J
Jens Axboe 已提交
3492
	io_commit_cqring(ctx);
3493 3494
}

3495
static void io_poll_complete_work(struct io_wq_work **workptr)
3496
{
3497
	struct io_wq_work *work = *workptr;
3498 3499 3500 3501
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_poll_iocb *poll = &req->poll;
	struct poll_table_struct pt = { ._key = poll->events };
	struct io_ring_ctx *ctx = req->ctx;
3502
	struct io_kiocb *nxt = NULL;
3503
	__poll_t mask = 0;
3504
	int ret = 0;
3505

3506
	if (work->flags & IO_WQ_WORK_CANCEL) {
3507
		WRITE_ONCE(poll->canceled, true);
3508 3509 3510 3511
		ret = -ECANCELED;
	} else if (READ_ONCE(poll->canceled)) {
		ret = -ECANCELED;
	}
3512

3513
	if (ret != -ECANCELED)
3514 3515 3516 3517 3518 3519 3520 3521 3522 3523
		mask = vfs_poll(poll->file, &pt) & poll->events;

	/*
	 * Note that ->ki_cancel callers also delete iocb from active_reqs after
	 * calling ->ki_cancel.  We need the ctx_lock roundtrip here to
	 * synchronize with them.  In the cancellation case the list_del_init
	 * itself is not actually needed, but harmless so we keep it in to
	 * avoid further branches in the fast path.
	 */
	spin_lock_irq(&ctx->completion_lock);
3524
	if (!mask && ret != -ECANCELED) {
3525
		add_wait_queue(poll->head, &poll->wait);
3526 3527 3528
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}
3529
	hash_del(&req->hash_node);
3530
	io_poll_complete(req, mask, ret);
3531 3532
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
3533
	io_cqring_ev_posted(ctx);
3534

J
Jens Axboe 已提交
3535 3536
	if (ret < 0)
		req_set_fail_links(req);
3537
	io_put_req_find_next(req, &nxt);
3538
	if (nxt)
3539
		io_wq_assign_next(workptr, nxt);
3540 3541
}

3542 3543 3544
static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
{
	struct io_kiocb *req, *tmp;
3545
	struct req_batch rb;
3546

3547
	rb.to_free = rb.need_iter = 0;
3548 3549 3550 3551 3552
	spin_lock_irq(&ctx->completion_lock);
	llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
		hash_del(&req->hash_node);
		io_poll_complete(req, req->result, 0);

3553 3554 3555 3556
		if (refcount_dec_and_test(&req->refs) &&
		    !io_req_multi_free(&rb, req)) {
			req->flags |= REQ_F_COMP_LOCKED;
			io_free_req(req);
3557 3558 3559 3560 3561
		}
	}
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
3562
	io_free_req_many(ctx, &rb);
3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574
}

static void io_poll_flush(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct llist_node *nodes;

	nodes = llist_del_all(&req->ctx->poll_llist);
	if (nodes)
		__io_poll_flush(req->ctx, nodes);
}

3575 3576 3577 3578 3579 3580 3581 3582
static void io_poll_trigger_evfd(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);

	eventfd_signal(req->ctx->cq_ev_fd, 1);
	io_put_req(req);
}

3583 3584 3585
static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
3586
	struct io_poll_iocb *poll = wait->private;
3587 3588 3589 3590 3591
	struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
	struct io_ring_ctx *ctx = req->ctx;
	__poll_t mask = key_to_poll(key);

	/* for instances that support it check for an event match first: */
J
Jens Axboe 已提交
3592 3593
	if (mask && !(mask & poll->events))
		return 0;
3594

3595
	list_del_init(&poll->wait.entry);
3596

3597 3598 3599 3600 3601 3602
	/*
	 * Run completion inline if we can. We're using trylock here because
	 * we are violating the completion_lock -> poll wq lock ordering.
	 * If we have a link timeout we're going to need the completion_lock
	 * for finalizing the request, mark us as having grabbed that already.
	 */
3603 3604
	if (mask) {
		unsigned long flags;
3605

3606 3607
		if (llist_empty(&ctx->poll_llist) &&
		    spin_trylock_irqsave(&ctx->completion_lock, flags)) {
3608 3609
			bool trigger_ev;

3610 3611 3612
			hash_del(&req->hash_node);
			io_poll_complete(req, mask, 0);

3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623
			trigger_ev = io_should_trigger_evfd(ctx);
			if (trigger_ev && eventfd_signal_count()) {
				trigger_ev = false;
				req->work.func = io_poll_trigger_evfd;
			} else {
				req->flags |= REQ_F_COMP_LOCKED;
				io_put_req(req);
				req = NULL;
			}
			spin_unlock_irqrestore(&ctx->completion_lock, flags);
			__io_cqring_ev_posted(ctx, trigger_ev);
3624 3625 3626 3627 3628 3629 3630 3631 3632
		} else {
			req->result = mask;
			req->llist_node.next = NULL;
			/* if the list wasn't empty, we're done */
			if (!llist_add(&req->llist_node, &ctx->poll_llist))
				req = NULL;
			else
				req->work.func = io_poll_flush;
		}
3633
	}
3634 3635
	if (req)
		io_queue_async_work(req);
3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657

	return 1;
}

struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

	if (unlikely(pt->req->poll.head)) {
		pt->error = -EINVAL;
		return;
	}

	pt->error = 0;
	pt->req->poll.head = head;
3658
	add_wait_queue(head, &pt->req->poll.wait);
3659 3660
}

3661 3662 3663
static void io_poll_req_insert(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
3664 3665 3666 3667
	struct hlist_head *list;

	list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	hlist_add_head(&req->hash_node, list);
3668 3669
}

3670
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3671 3672 3673 3674 3675 3676 3677 3678
{
	struct io_poll_iocb *poll = &req->poll;
	u16 events;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
3679 3680
	if (!poll->file)
		return -EBADF;
3681 3682 3683

	events = READ_ONCE(sqe->poll_events);
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695
	return 0;
}

static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	bool cancel = false;
	__poll_t mask;

	INIT_IO_WORK(&req->work, io_poll_complete_work);
3696
	INIT_HLIST_NODE(&req->hash_node);
3697 3698

	poll->head = NULL;
J
Jens Axboe 已提交
3699
	poll->done = false;
3700 3701 3702 3703 3704 3705 3706 3707
	poll->canceled = false;

	ipt.pt._qproc = io_poll_queue_proc;
	ipt.pt._key = poll->events;
	ipt.req = req;
	ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */

	/* initialized the list so that we can do list_empty checks */
3708 3709 3710
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, io_poll_wake);
	poll->wait.private = poll;
3711

3712 3713
	INIT_LIST_HEAD(&req->list);

3714 3715 3716
	mask = vfs_poll(poll->file, &ipt.pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
3717 3718
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
3719
		if (unlikely(list_empty(&poll->wait.entry))) {
J
Jens Axboe 已提交
3720 3721 3722 3723 3724 3725
			if (ipt.error)
				cancel = true;
			ipt.error = 0;
			mask = 0;
		}
		if (mask || ipt.error)
3726
			list_del_init(&poll->wait.entry);
J
Jens Axboe 已提交
3727 3728 3729
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
3730
			io_poll_req_insert(req);
J
Jens Axboe 已提交
3731 3732 3733
		spin_unlock(&poll->head->lock);
	}
	if (mask) { /* no async, we'd stolen it */
3734
		ipt.error = 0;
3735
		io_poll_complete(req, mask, 0);
3736 3737 3738
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
3739 3740
	if (mask) {
		io_cqring_ev_posted(ctx);
3741
		io_put_req_find_next(req, nxt);
3742
	}
J
Jens Axboe 已提交
3743
	return ipt.error;
3744 3745
}

J
Jens Axboe 已提交
3746 3747
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
3748 3749 3750 3751
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
3752 3753 3754 3755 3756
	unsigned long flags;

	atomic_inc(&ctx->cq_timeouts);

	spin_lock_irqsave(&ctx->completion_lock, flags);
3757
	/*
3758 3759
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
3760
	 */
3761
	if (!list_empty(&req->list)) {
3762
		struct io_kiocb *prev;
J
Jens Axboe 已提交
3763

3764 3765
		/*
		 * Adjust the reqs sequence before the current one because it
3766
		 * will consume a slot in the cq_ring and the cq_tail
3767 3768 3769 3770 3771 3772 3773 3774
		 * pointer will be increased, otherwise other timeout reqs may
		 * return in advance without waiting for enough wait_nr.
		 */
		prev = req;
		list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
			prev->sequence++;
		list_del_init(&req->list);
	}
J
Jens Axboe 已提交
3775

3776
	io_cqring_fill_event(req, -ETIME);
J
Jens Axboe 已提交
3777 3778 3779 3780
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
3781
	req_set_fail_links(req);
J
Jens Axboe 已提交
3782 3783 3784 3785
	io_put_req(req);
	return HRTIMER_NORESTART;
}

3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801
static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
{
	struct io_kiocb *req;
	int ret = -ENOENT;

	list_for_each_entry(req, &ctx->timeout_list, list) {
		if (user_data == req->user_data) {
			list_del_init(&req->list);
			ret = 0;
			break;
		}
	}

	if (ret == -ENOENT)
		return ret;

3802
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
3803 3804 3805
	if (ret == -1)
		return -EALREADY;

J
Jens Axboe 已提交
3806
	req_set_fail_links(req);
3807 3808 3809 3810 3811
	io_cqring_fill_event(req, -ECANCELED);
	io_put_req(req);
	return 0;
}

3812 3813
static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
		return -EINVAL;

	req->timeout.addr = READ_ONCE(sqe->addr);
	req->timeout.flags = READ_ONCE(sqe->timeout_flags);
	if (req->timeout.flags)
		return -EINVAL;

	return 0;
}

3828 3829 3830
/*
 * Remove or update an existing timeout command
 */
3831
static int io_timeout_remove(struct io_kiocb *req)
3832 3833
{
	struct io_ring_ctx *ctx = req->ctx;
3834
	int ret;
3835 3836

	spin_lock_irq(&ctx->completion_lock);
3837
	ret = io_timeout_cancel(ctx, req->timeout.addr);
3838

3839
	io_cqring_fill_event(req, ret);
3840 3841
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
3842
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
3843 3844
	if (ret < 0)
		req_set_fail_links(req);
3845
	io_put_req(req);
3846
	return 0;
J
Jens Axboe 已提交
3847 3848
}

3849
static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
3850
			   bool is_timeout_link)
J
Jens Axboe 已提交
3851
{
3852
	struct io_timeout_data *data;
3853
	unsigned flags;
J
Jens Axboe 已提交
3854

3855
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
3856
		return -EINVAL;
3857
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
3858
		return -EINVAL;
3859 3860
	if (sqe->off && is_timeout_link)
		return -EINVAL;
3861 3862
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
3863
		return -EINVAL;
3864

3865 3866
	req->timeout.count = READ_ONCE(sqe->off);

3867
	if (!req->io && io_alloc_async_ctx(req))
3868 3869 3870
		return -ENOMEM;

	data = &req->io->timeout;
3871 3872 3873 3874
	data->req = req;
	req->flags |= REQ_F_TIMEOUT;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
3875 3876
		return -EFAULT;

3877
	if (flags & IORING_TIMEOUT_ABS)
3878
		data->mode = HRTIMER_MODE_ABS;
3879
	else
3880
		data->mode = HRTIMER_MODE_REL;
3881

3882 3883 3884 3885
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}

3886
static int io_timeout(struct io_kiocb *req)
3887 3888 3889 3890 3891 3892 3893
{
	unsigned count;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_timeout_data *data;
	struct list_head *entry;
	unsigned span = 0;

3894
	data = &req->io->timeout;
3895

J
Jens Axboe 已提交
3896 3897
	/*
	 * sqe->off holds how many events that need to occur for this
3898 3899
	 * timeout event to be satisfied. If it isn't set, then this is
	 * a pure timeout request, sequence isn't used.
J
Jens Axboe 已提交
3900
	 */
3901
	count = req->timeout.count;
3902 3903 3904 3905 3906 3907
	if (!count) {
		req->flags |= REQ_F_TIMEOUT_NOSEQ;
		spin_lock_irq(&ctx->completion_lock);
		entry = ctx->timeout_list.prev;
		goto add;
	}
J
Jens Axboe 已提交
3908 3909

	req->sequence = ctx->cached_sq_head + count - 1;
3910
	data->seq_offset = count;
J
Jens Axboe 已提交
3911 3912 3913 3914 3915 3916 3917 3918

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	spin_lock_irq(&ctx->completion_lock);
	list_for_each_prev(entry, &ctx->timeout_list) {
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
3919 3920
		unsigned nxt_sq_head;
		long long tmp, tmp_nxt;
3921
		u32 nxt_offset = nxt->io->timeout.seq_offset;
J
Jens Axboe 已提交
3922

3923 3924 3925
		if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
			continue;

3926 3927 3928 3929 3930
		/*
		 * Since cached_sq_head + count - 1 can overflow, use type long
		 * long to store it.
		 */
		tmp = (long long)ctx->cached_sq_head + count - 1;
3931 3932
		nxt_sq_head = nxt->sequence - nxt_offset + 1;
		tmp_nxt = (long long)nxt_sq_head + nxt_offset - 1;
3933 3934 3935 3936 3937 3938

		/*
		 * cached_sq_head may overflow, and it will never overflow twice
		 * once there is some timeout req still be valid.
		 */
		if (ctx->cached_sq_head < nxt_sq_head)
3939
			tmp += UINT_MAX;
3940

3941
		if (tmp > tmp_nxt)
J
Jens Axboe 已提交
3942
			break;
3943 3944 3945 3946 3947 3948 3949

		/*
		 * Sequence of reqs after the insert one and itself should
		 * be adjusted because each timeout req consumes a slot.
		 */
		span++;
		nxt->sequence++;
J
Jens Axboe 已提交
3950
	}
3951
	req->sequence -= span;
3952
add:
J
Jens Axboe 已提交
3953
	list_add(&req->list, entry);
3954 3955
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
J
Jens Axboe 已提交
3956 3957 3958 3959
	spin_unlock_irq(&ctx->completion_lock);
	return 0;
}

3960 3961 3962 3963 3964 3965 3966
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

3967
static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;

	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

3985 3986 3987
	return ret;
}

3988 3989
static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
3990
				     struct io_kiocb **nxt, int success_ret)
3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
	if (ret != -ENOENT) {
		spin_lock_irqsave(&ctx->completion_lock, flags);
		goto done;
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr);
done:
4007 4008
	if (!ret)
		ret = success_ret;
4009 4010 4011 4012 4013
	io_cqring_fill_event(req, ret);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

J
Jens Axboe 已提交
4014 4015
	if (ret < 0)
		req_set_fail_links(req);
4016 4017 4018
	io_put_req_find_next(req, nxt);
}

4019 4020
static int io_async_cancel_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
4021
{
4022
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4023 4024 4025 4026 4027
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
	    sqe->cancel_flags)
		return -EINVAL;

4028 4029 4030 4031 4032 4033 4034 4035 4036
	req->cancel.addr = READ_ONCE(sqe->addr);
	return 0;
}

static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
J
Jens Axboe 已提交
4037 4038 4039
	return 0;
}

4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054
static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (sqe->flags || sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	req->files_update.offset = READ_ONCE(sqe->off);
	req->files_update.nr_args = READ_ONCE(sqe->len);
	if (!req->files_update.nr_args)
		return -EINVAL;
	req->files_update.arg = READ_ONCE(sqe->addr);
	return 0;
}

static int io_files_update(struct io_kiocb *req, bool force_nonblock)
4055 4056
{
	struct io_ring_ctx *ctx = req->ctx;
4057 4058
	struct io_uring_files_update up;
	int ret;
4059

4060
	if (force_nonblock)
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
J
Jens Axboe 已提交
4074 4075 4076
	return 0;
}

4077 4078
static int io_req_defer_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
4079
{
J
Jens Axboe 已提交
4080
	ssize_t ret = 0;
4081

4082 4083 4084 4085 4086 4087
	if (io_op_defs[req->opcode].file_table) {
		ret = io_grab_files(req);
		if (unlikely(ret))
			return ret;
	}

4088 4089
	io_req_work_grab_env(req, &io_op_defs[req->opcode]);

4090
	switch (req->opcode) {
J
Jens Axboe 已提交
4091 4092
	case IORING_OP_NOP:
		break;
4093 4094
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
4095
	case IORING_OP_READ:
4096
		ret = io_read_prep(req, sqe, true);
4097 4098 4099
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
4100
	case IORING_OP_WRITE:
4101
		ret = io_write_prep(req, sqe, true);
4102
		break;
4103
	case IORING_OP_POLL_ADD:
4104
		ret = io_poll_add_prep(req, sqe);
4105 4106
		break;
	case IORING_OP_POLL_REMOVE:
4107
		ret = io_poll_remove_prep(req, sqe);
4108
		break;
4109
	case IORING_OP_FSYNC:
4110
		ret = io_prep_fsync(req, sqe);
4111 4112
		break;
	case IORING_OP_SYNC_FILE_RANGE:
4113
		ret = io_prep_sfr(req, sqe);
4114
		break;
4115
	case IORING_OP_SENDMSG:
4116
	case IORING_OP_SEND:
4117
		ret = io_sendmsg_prep(req, sqe);
4118 4119
		break;
	case IORING_OP_RECVMSG:
4120
	case IORING_OP_RECV:
4121
		ret = io_recvmsg_prep(req, sqe);
4122
		break;
4123
	case IORING_OP_CONNECT:
4124
		ret = io_connect_prep(req, sqe);
4125
		break;
4126
	case IORING_OP_TIMEOUT:
4127
		ret = io_timeout_prep(req, sqe, false);
4128
		break;
4129
	case IORING_OP_TIMEOUT_REMOVE:
4130
		ret = io_timeout_remove_prep(req, sqe);
4131
		break;
4132
	case IORING_OP_ASYNC_CANCEL:
4133
		ret = io_async_cancel_prep(req, sqe);
4134
		break;
4135
	case IORING_OP_LINK_TIMEOUT:
4136
		ret = io_timeout_prep(req, sqe, true);
4137
		break;
4138
	case IORING_OP_ACCEPT:
4139
		ret = io_accept_prep(req, sqe);
4140
		break;
4141 4142 4143
	case IORING_OP_FALLOCATE:
		ret = io_fallocate_prep(req, sqe);
		break;
4144 4145 4146
	case IORING_OP_OPENAT:
		ret = io_openat_prep(req, sqe);
		break;
4147 4148 4149
	case IORING_OP_CLOSE:
		ret = io_close_prep(req, sqe);
		break;
4150 4151 4152
	case IORING_OP_FILES_UPDATE:
		ret = io_files_update_prep(req, sqe);
		break;
4153 4154 4155
	case IORING_OP_STATX:
		ret = io_statx_prep(req, sqe);
		break;
J
Jens Axboe 已提交
4156 4157 4158
	case IORING_OP_FADVISE:
		ret = io_fadvise_prep(req, sqe);
		break;
J
Jens Axboe 已提交
4159 4160 4161
	case IORING_OP_MADVISE:
		ret = io_madvise_prep(req, sqe);
		break;
4162 4163 4164
	case IORING_OP_OPENAT2:
		ret = io_openat2_prep(req, sqe);
		break;
4165 4166 4167
	case IORING_OP_EPOLL_CTL:
		ret = io_epoll_ctl_prep(req, sqe);
		break;
4168
	default:
J
Jens Axboe 已提交
4169 4170 4171
		printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
				req->opcode);
		ret = -EINVAL;
4172
		break;
4173 4174
	}

4175
	return ret;
4176 4177
}

4178
static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4179
{
4180
	struct io_ring_ctx *ctx = req->ctx;
4181
	int ret;
4182

B
Bob Liu 已提交
4183 4184
	/* Still need defer if there is pending req in defer list. */
	if (!req_need_defer(req) && list_empty(&ctx->defer_list))
4185 4186
		return 0;

4187
	if (!req->io && io_alloc_async_ctx(req))
4188 4189
		return -EAGAIN;

4190
	ret = io_req_defer_prep(req, sqe);
4191
	if (ret < 0)
4192 4193
		return ret;

4194
	spin_lock_irq(&ctx->completion_lock);
B
Bob Liu 已提交
4195
	if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
4196 4197 4198 4199
		spin_unlock_irq(&ctx->completion_lock);
		return 0;
	}

4200
	trace_io_uring_defer(ctx, req, req->user_data);
4201 4202 4203 4204 4205
	list_add_tail(&req->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

4206 4207
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
4208
{
4209
	struct io_ring_ctx *ctx = req->ctx;
4210
	int ret;
J
Jens Axboe 已提交
4211

4212
	switch (req->opcode) {
J
Jens Axboe 已提交
4213
	case IORING_OP_NOP:
4214
		ret = io_nop(req);
J
Jens Axboe 已提交
4215 4216
		break;
	case IORING_OP_READV:
4217
	case IORING_OP_READ_FIXED:
4218
	case IORING_OP_READ:
4219 4220 4221 4222 4223
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
4224
		ret = io_read(req, nxt, force_nonblock);
4225
		break;
4226
	case IORING_OP_WRITEV:
4227
	case IORING_OP_WRITE_FIXED:
4228
	case IORING_OP_WRITE:
4229 4230 4231 4232 4233
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
4234
		ret = io_write(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4235
		break;
C
Christoph Hellwig 已提交
4236
	case IORING_OP_FSYNC:
4237 4238 4239 4240 4241
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
4242
		ret = io_fsync(req, nxt, force_nonblock);
C
Christoph Hellwig 已提交
4243
		break;
4244
	case IORING_OP_POLL_ADD:
4245 4246 4247 4248 4249
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
4250
		ret = io_poll_add(req, nxt);
4251 4252
		break;
	case IORING_OP_POLL_REMOVE:
4253 4254 4255 4256 4257
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
4258
		ret = io_poll_remove(req);
4259
		break;
4260
	case IORING_OP_SYNC_FILE_RANGE:
4261 4262 4263 4264 4265
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
4266
		ret = io_sync_file_range(req, nxt, force_nonblock);
4267
		break;
J
Jens Axboe 已提交
4268
	case IORING_OP_SENDMSG:
4269
	case IORING_OP_SEND:
4270 4271 4272 4273 4274
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
4275 4276 4277 4278
		if (req->opcode == IORING_OP_SENDMSG)
			ret = io_sendmsg(req, nxt, force_nonblock);
		else
			ret = io_send(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4279
		break;
J
Jens Axboe 已提交
4280
	case IORING_OP_RECVMSG:
4281
	case IORING_OP_RECV:
4282 4283 4284 4285 4286
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
4287 4288 4289 4290
		if (req->opcode == IORING_OP_RECVMSG)
			ret = io_recvmsg(req, nxt, force_nonblock);
		else
			ret = io_recv(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4291
		break;
J
Jens Axboe 已提交
4292
	case IORING_OP_TIMEOUT:
4293 4294 4295 4296 4297
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
4298
		ret = io_timeout(req);
J
Jens Axboe 已提交
4299
		break;
4300
	case IORING_OP_TIMEOUT_REMOVE:
4301 4302 4303 4304 4305
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
4306
		ret = io_timeout_remove(req);
4307
		break;
4308
	case IORING_OP_ACCEPT:
4309 4310 4311 4312 4313
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
4314
		ret = io_accept(req, nxt, force_nonblock);
4315
		break;
4316
	case IORING_OP_CONNECT:
4317 4318 4319 4320 4321
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
4322
		ret = io_connect(req, nxt, force_nonblock);
4323
		break;
4324
	case IORING_OP_ASYNC_CANCEL:
4325 4326 4327 4328 4329
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
4330
		ret = io_async_cancel(req, nxt);
4331
		break;
4332 4333 4334 4335 4336 4337 4338 4339
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fallocate(req, nxt, force_nonblock);
		break;
4340 4341 4342 4343 4344 4345 4346 4347
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat(req, nxt, force_nonblock);
		break;
4348 4349 4350 4351 4352 4353 4354 4355
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_close(req, nxt, force_nonblock);
		break;
4356 4357 4358 4359 4360 4361 4362 4363
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_files_update(req, force_nonblock);
		break;
4364 4365 4366 4367 4368 4369 4370 4371
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_statx(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4372 4373 4374 4375 4376 4377 4378 4379
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fadvise(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4380 4381 4382 4383 4384 4385 4386 4387
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_madvise(req, nxt, force_nonblock);
		break;
4388 4389 4390 4391 4392 4393 4394 4395
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat2(req, nxt, force_nonblock);
		break;
4396 4397 4398 4399 4400 4401 4402 4403
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_epoll_ctl(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4404 4405 4406 4407 4408
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
4409 4410 4411 4412
	if (ret)
		return ret;

	if (ctx->flags & IORING_SETUP_IOPOLL) {
4413 4414
		const bool in_async = io_wq_current_is_worker();

J
Jens Axboe 已提交
4415
		if (req->result == -EAGAIN)
J
Jens Axboe 已提交
4416 4417
			return -EAGAIN;

4418 4419 4420 4421
		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

J
Jens Axboe 已提交
4422
		io_iopoll_req_issued(req);
4423 4424 4425

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
4426 4427 4428
	}

	return 0;
J
Jens Axboe 已提交
4429 4430
}

4431
static void io_wq_submit_work(struct io_wq_work **workptr)
J
Jens Axboe 已提交
4432
{
4433
	struct io_wq_work *work = *workptr;
J
Jens Axboe 已提交
4434
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
4435 4436
	struct io_kiocb *nxt = NULL;
	int ret = 0;
J
Jens Axboe 已提交
4437

4438 4439 4440
	/* if NO_CANCEL is set, we must still run the work */
	if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
				IO_WQ_WORK_CANCEL) {
4441
		ret = -ECANCELED;
4442
	}
4443

4444
	if (!ret) {
4445 4446
		req->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
		req->in_async = true;
4447
		do {
4448
			ret = io_issue_sqe(req, NULL, &nxt, false);
4449 4450 4451 4452 4453 4454 4455 4456 4457 4458
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
4459

4460
	/* drop submission reference */
4461
	io_put_req(req);
4462

4463
	if (ret) {
J
Jens Axboe 已提交
4464
		req_set_fail_links(req);
4465
		io_cqring_add_event(req, ret);
4466
		io_put_req(req);
4467
	}
J
Jens Axboe 已提交
4468

4469
	/* if a dependent link is ready, pass it back */
4470 4471
	if (!ret && nxt)
		io_wq_assign_next(workptr, nxt);
J
Jens Axboe 已提交
4472 4473
}

4474
static int io_req_needs_file(struct io_kiocb *req, int fd)
4475
{
4476
	if (!io_op_defs[req->opcode].needs_file)
4477
		return 0;
4478 4479 4480
	if (fd == -1 && io_op_defs[req->opcode].fd_non_neg)
		return 0;
	return 1;
J
Jens Axboe 已提交
4481 4482
}

4483 4484 4485 4486 4487
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

4488 4489
	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
	return table->files[index & IORING_FILE_TABLE_MASK];;
4490 4491
}

4492 4493
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4494
{
4495
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
4496
	unsigned flags;
4497
	int fd;
J
Jens Axboe 已提交
4498

4499 4500
	flags = READ_ONCE(sqe->flags);
	fd = READ_ONCE(sqe->fd);
J
Jens Axboe 已提交
4501

4502 4503
	if (!io_req_needs_file(req, fd))
		return 0;
J
Jens Axboe 已提交
4504 4505

	if (flags & IOSQE_FIXED_FILE) {
4506
		if (unlikely(!ctx->file_data ||
J
Jens Axboe 已提交
4507 4508
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
4509
		fd = array_index_nospec(fd, ctx->nr_user_files);
4510 4511
		req->file = io_file_from_index(ctx, fd);
		if (!req->file)
4512
			return -EBADF;
J
Jens Axboe 已提交
4513
		req->flags |= REQ_F_FIXED_FILE;
4514
		percpu_ref_get(&ctx->file_data->refs);
J
Jens Axboe 已提交
4515
	} else {
4516
		if (req->needs_fixed_file)
J
Jens Axboe 已提交
4517
			return -EBADF;
4518
		trace_io_uring_file_get(ctx, fd);
J
Jens Axboe 已提交
4519 4520 4521 4522 4523 4524 4525 4526
		req->file = io_file_get(state, fd);
		if (unlikely(!req->file))
			return -EBADF;
	}

	return 0;
}

4527
static int io_grab_files(struct io_kiocb *req)
4528 4529
{
	int ret = -EBADF;
4530
	struct io_ring_ctx *ctx = req->ctx;
4531

4532 4533
	if (req->work.files)
		return 0;
P
Pavel Begunkov 已提交
4534
	if (!ctx->ring_file)
4535 4536
		return -EBADF;

4537 4538 4539 4540 4541 4542 4543 4544
	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
P
Pavel Begunkov 已提交
4545
	if (fcheck(ctx->ring_fd) == ctx->ring_file) {
4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

4557
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
J
Jens Axboe 已提交
4558
{
4559 4560 4561
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
4562 4563 4564 4565 4566 4567 4568 4569 4570 4571
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
4572 4573 4574
	if (!list_empty(&req->link_list)) {
		prev = list_entry(req->link_list.prev, struct io_kiocb,
				  link_list);
4575
		if (refcount_inc_not_zero(&prev->refs)) {
4576
			list_del_init(&req->link_list);
4577 4578
			prev->flags &= ~REQ_F_LINK_TIMEOUT;
		} else
4579
			prev = NULL;
4580 4581 4582 4583 4584
	}

	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
J
Jens Axboe 已提交
4585
		req_set_fail_links(prev);
4586 4587
		io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
						-ETIME);
4588
		io_put_req(prev);
4589 4590 4591
	} else {
		io_cqring_add_event(req, -ETIME);
		io_put_req(req);
4592 4593 4594 4595
	}
	return HRTIMER_NORESTART;
}

4596
static void io_queue_linked_timeout(struct io_kiocb *req)
4597
{
4598
	struct io_ring_ctx *ctx = req->ctx;
4599

4600 4601 4602 4603 4604
	/*
	 * If the list is now empty, then our linked request finished before
	 * we got a chance to setup the timer
	 */
	spin_lock_irq(&ctx->completion_lock);
4605
	if (!list_empty(&req->link_list)) {
4606
		struct io_timeout_data *data = &req->io->timeout;
4607

4608 4609 4610
		data->timer.function = io_link_timeout_fn;
		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
				data->mode);
4611
	}
4612
	spin_unlock_irq(&ctx->completion_lock);
4613 4614

	/* drop submission reference */
4615 4616
	io_put_req(req);
}
4617

4618
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
4619 4620 4621 4622 4623 4624
{
	struct io_kiocb *nxt;

	if (!(req->flags & REQ_F_LINK))
		return NULL;

4625 4626
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
					link_list);
4627
	if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
4628
		return NULL;
4629

4630 4631
	req->flags |= REQ_F_LINK_TIMEOUT;
	return nxt;
4632 4633
}

4634
static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4635
{
4636
	struct io_kiocb *linked_timeout;
4637
	struct io_kiocb *nxt = NULL;
4638
	int ret;
J
Jens Axboe 已提交
4639

4640 4641 4642
again:
	linked_timeout = io_prep_linked_timeout(req);

4643
	ret = io_issue_sqe(req, sqe, &nxt, true);
4644 4645 4646 4647 4648 4649 4650

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
	    (req->flags & REQ_F_MUST_PUNT))) {
4651
punt:
4652
		if (io_op_defs[req->opcode].file_table) {
4653 4654 4655
			ret = io_grab_files(req);
			if (ret)
				goto err;
J
Jens Axboe 已提交
4656
		}
4657 4658 4659 4660 4661 4662

		/*
		 * Queued up for async execution, worker will release
		 * submit reference when the iocb is actually submitted.
		 */
		io_queue_async_work(req);
4663
		goto done_req;
J
Jens Axboe 已提交
4664
	}
4665

4666
err:
4667
	/* drop submission reference */
4668
	io_put_req(req);
4669

4670
	if (linked_timeout) {
4671
		if (!ret)
4672
			io_queue_linked_timeout(linked_timeout);
4673
		else
4674
			io_put_req(linked_timeout);
4675 4676
	}

4677
	/* and drop final reference, if we failed */
J
Jens Axboe 已提交
4678
	if (ret) {
4679
		io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
4680
		req_set_fail_links(req);
4681
		io_put_req(req);
J
Jens Axboe 已提交
4682
	}
4683 4684 4685 4686
done_req:
	if (nxt) {
		req = nxt;
		nxt = NULL;
4687 4688 4689

		if (req->flags & REQ_F_FORCE_ASYNC)
			goto punt;
4690 4691
		goto again;
	}
J
Jens Axboe 已提交
4692 4693
}

4694
static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4695 4696 4697
{
	int ret;

4698
	ret = io_req_defer(req, sqe);
4699 4700
	if (ret) {
		if (ret != -EIOCBQUEUED) {
4701
fail_req:
4702
			io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
4703
			req_set_fail_links(req);
4704
			io_double_put_req(req);
4705
		}
4706
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
4707 4708 4709
		ret = io_req_defer_prep(req, sqe);
		if (unlikely(ret < 0))
			goto fail_req;
J
Jens Axboe 已提交
4710 4711 4712 4713 4714 4715 4716
		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
4717
		__io_queue_sqe(req, sqe);
J
Jens Axboe 已提交
4718
	}
4719 4720
}

4721
static inline void io_queue_link_head(struct io_kiocb *req)
4722
{
4723
	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
4724 4725 4726
		io_cqring_add_event(req, -ECANCELED);
		io_double_put_req(req);
	} else
4727
		io_queue_sqe(req, NULL);
4728 4729
}

J
Jens Axboe 已提交
4730
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK|	\
J
Jens Axboe 已提交
4731
				IOSQE_IO_HARDLINK | IOSQE_ASYNC)
J
Jens Axboe 已提交
4732

4733 4734
static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			  struct io_submit_state *state, struct io_kiocb **link)
J
Jens Axboe 已提交
4735
{
4736
	const struct cred *old_creds = NULL;
4737
	struct io_ring_ctx *ctx = req->ctx;
4738
	unsigned int sqe_flags;
4739
	int ret, id;
J
Jens Axboe 已提交
4740

4741
	sqe_flags = READ_ONCE(sqe->flags);
J
Jens Axboe 已提交
4742 4743

	/* enforce forwards compatibility on users */
4744
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
J
Jens Axboe 已提交
4745
		ret = -EINVAL;
4746
		goto err_req;
J
Jens Axboe 已提交
4747 4748
	}

4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760
	id = READ_ONCE(sqe->personality);
	if (id) {
		const struct cred *personality_creds;

		personality_creds = idr_find(&ctx->personality_idr, id);
		if (unlikely(!personality_creds)) {
			ret = -EINVAL;
			goto err_req;
		}
		old_creds = override_creds(personality_creds);
	}

4761 4762 4763
	/* same numerical values with corresponding REQ_F_*, safe to copy */
	req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|
					IOSQE_ASYNC);
J
Jens Axboe 已提交
4764

4765
	ret = io_req_set_file(state, req, sqe);
J
Jens Axboe 已提交
4766 4767
	if (unlikely(ret)) {
err_req:
4768 4769
		io_cqring_add_event(req, ret);
		io_double_put_req(req);
4770 4771
		if (old_creds)
			revert_creds(old_creds);
4772
		return false;
J
Jens Axboe 已提交
4773 4774 4775 4776 4777 4778 4779 4780 4781 4782
	}

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
P
Pavel Begunkov 已提交
4783
		struct io_kiocb *head = *link;
J
Jens Axboe 已提交
4784

4785 4786 4787 4788 4789 4790 4791
		/*
		 * Taking sequential execution of a link, draining both sides
		 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
		 * requests in the link. So, it drains the head and the
		 * next after the link request. The last one is done via
		 * drain_next flag to persist the effect across calls.
		 */
4792 4793 4794 4795
		if (sqe_flags & IOSQE_IO_DRAIN) {
			head->flags |= REQ_F_IO_DRAIN;
			ctx->drain_next = 1;
		}
4796
		if (io_alloc_async_ctx(req)) {
J
Jens Axboe 已提交
4797 4798 4799 4800
			ret = -EAGAIN;
			goto err_req;
		}

4801
		ret = io_req_defer_prep(req, sqe);
4802
		if (ret) {
J
Jens Axboe 已提交
4803
			/* fail even hard links since we don't submit */
P
Pavel Begunkov 已提交
4804
			head->flags |= REQ_F_FAIL_LINK;
4805
			goto err_req;
4806
		}
P
Pavel Begunkov 已提交
4807 4808
		trace_io_uring_link(ctx, req, head);
		list_add_tail(&req->link_list, &head->link_list);
4809 4810 4811 4812 4813 4814

		/* last request of a link, enqueue the link */
		if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
			io_queue_link_head(head);
			*link = NULL;
		}
J
Jens Axboe 已提交
4815
	} else {
4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829
		if (unlikely(ctx->drain_next)) {
			req->flags |= REQ_F_IO_DRAIN;
			req->ctx->drain_next = 0;
		}
		if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
			req->flags |= REQ_F_LINK;
			INIT_LIST_HEAD(&req->link_list);
			ret = io_req_defer_prep(req, sqe);
			if (ret)
				req->flags |= REQ_F_FAIL_LINK;
			*link = req;
		} else {
			io_queue_sqe(req, sqe);
		}
J
Jens Axboe 已提交
4830
	}
4831

4832 4833
	if (old_creds)
		revert_creds(old_creds);
4834
	return true;
J
Jens Axboe 已提交
4835 4836
}

4837 4838 4839 4840 4841 4842
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
	blk_finish_plug(&state->plug);
4843
	io_file_put(state);
J
Jens Axboe 已提交
4844 4845 4846
	if (state->free_reqs)
		kmem_cache_free_bulk(req_cachep, state->free_reqs,
					&state->reqs[state->cur_req]);
4847 4848 4849 4850 4851 4852
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
4853
				  unsigned int max_ios)
4854 4855
{
	blk_start_plug(&state->plug);
J
Jens Axboe 已提交
4856
	state->free_reqs = 0;
4857 4858 4859 4860
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
4861 4862
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
4863
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
4864

4865 4866 4867 4868 4869 4870
	/*
	 * Ensure any loads from the SQEs are done at this point,
	 * since once we write the new head, the application could
	 * write new data to them.
	 */
	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
4871 4872 4873
}

/*
4874
 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
J
Jens Axboe 已提交
4875 4876 4877 4878 4879 4880
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
4881 4882
static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req,
			  const struct io_uring_sqe **sqe_ptr)
J
Jens Axboe 已提交
4883
{
4884
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
4885 4886 4887 4888 4889 4890 4891 4892 4893 4894
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
4895
	head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
4896
	if (likely(head < ctx->sq_entries)) {
4897 4898 4899 4900 4901 4902
		/*
		 * All io need record the previous position, if LINK vs DARIN,
		 * it can be used to mark the position of the first IO in the
		 * link list.
		 */
		req->sequence = ctx->cached_sq_head;
4903 4904 4905
		*sqe_ptr = &ctx->sq_sqes[head];
		req->opcode = READ_ONCE((*sqe_ptr)->opcode);
		req->user_data = READ_ONCE((*sqe_ptr)->user_data);
J
Jens Axboe 已提交
4906 4907 4908 4909 4910 4911
		ctx->cached_sq_head++;
		return true;
	}

	/* drop invalid entries */
	ctx->cached_sq_head++;
4912
	ctx->cached_sq_dropped++;
4913
	WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
J
Jens Axboe 已提交
4914 4915 4916
	return false;
}

4917
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
4918 4919
			  struct file *ring_file, int ring_fd,
			  struct mm_struct **mm, bool async)
J
Jens Axboe 已提交
4920 4921
{
	struct io_submit_state state, *statep = NULL;
J
Jens Axboe 已提交
4922 4923
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
4924
	bool mm_fault = false;
J
Jens Axboe 已提交
4925

4926
	/* if we have a backlog and couldn't flush it all, return BUSY */
4927 4928 4929 4930 4931
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}
J
Jens Axboe 已提交
4932

4933 4934
	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
4935

4936 4937
	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;
J
Jens Axboe 已提交
4938 4939

	if (nr > IO_PLUG_THRESHOLD) {
4940
		io_submit_state_start(&state, nr);
J
Jens Axboe 已提交
4941 4942 4943
		statep = &state;
	}

P
Pavel Begunkov 已提交
4944 4945 4946
	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

J
Jens Axboe 已提交
4947
	for (i = 0; i < nr; i++) {
4948
		const struct io_uring_sqe *sqe;
4949
		struct io_kiocb *req;
4950

4951 4952 4953 4954
		req = io_get_req(ctx, statep);
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
4955
			break;
4956
		}
4957
		if (!io_get_sqring(ctx, req, &sqe)) {
4958
			__io_req_do_free(req);
4959 4960
			break;
		}
4961

4962 4963 4964 4965 4966 4967
		/* will complete beyond this point, count as submitted */
		submitted++;

		if (unlikely(req->opcode >= IORING_OP_LAST)) {
			io_cqring_add_event(req, -EINVAL);
			io_double_put_req(req);
4968 4969
			break;
		}
4970

4971
		if (io_op_defs[req->opcode].needs_mm && !*mm) {
4972 4973 4974 4975 4976
			mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
			if (!mm_fault) {
				use_mm(ctx->sqo_mm);
				*mm = ctx->sqo_mm;
			}
J
Jens Axboe 已提交
4977 4978
		}

4979 4980 4981
		req->has_user = *mm != NULL;
		req->in_async = async;
		req->needs_fixed_file = async;
4982 4983
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
						true, async);
4984
		if (!io_submit_sqe(req, sqe, statep, &link))
4985
			break;
J
Jens Axboe 已提交
4986 4987
	}

4988 4989 4990 4991 4992
	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
J
Jens Axboe 已提交
4993
	if (link)
4994
		io_queue_link_head(link);
J
Jens Axboe 已提交
4995 4996 4997
	if (statep)
		io_submit_state_end(&state);

4998 4999 5000
	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

J
Jens Axboe 已提交
5001 5002 5003 5004 5005 5006 5007
	return submitted;
}

static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
	struct mm_struct *cur_mm = NULL;
5008
	const struct cred *old_cred;
J
Jens Axboe 已提交
5009 5010 5011 5012
	mm_segment_t old_fs;
	DEFINE_WAIT(wait);
	unsigned inflight;
	unsigned long timeout;
5013
	int ret;
J
Jens Axboe 已提交
5014

5015
	complete(&ctx->completions[1]);
5016

J
Jens Axboe 已提交
5017 5018
	old_fs = get_fs();
	set_fs(USER_DS);
5019
	old_cred = override_creds(ctx->creds);
J
Jens Axboe 已提交
5020

5021
	ret = timeout = inflight = 0;
5022
	while (!kthread_should_park()) {
5023
		unsigned int to_submit;
J
Jens Axboe 已提交
5024 5025 5026 5027 5028

		if (inflight) {
			unsigned nr_events = 0;

			if (ctx->flags & IORING_SETUP_IOPOLL) {
5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042
				/*
				 * inflight is the count of the maximum possible
				 * entries we submitted, but it can be smaller
				 * if we dropped some of them. If we don't have
				 * poll entries available, then we know that we
				 * have nothing left to poll for. Reset the
				 * inflight count to zero in that case.
				 */
				mutex_lock(&ctx->uring_lock);
				if (!list_empty(&ctx->poll_list))
					__io_iopoll_check(ctx, &nr_events, 0);
				else
					inflight = 0;
				mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055
			} else {
				/*
				 * Normal IO, just pretend everything completed.
				 * We don't have to poll completions for that.
				 */
				nr_events = inflight;
			}

			inflight -= nr_events;
			if (!inflight)
				timeout = jiffies + ctx->sq_thread_idle;
		}

5056
		to_submit = io_sqring_entries(ctx);
5057 5058 5059 5060 5061 5062

		/*
		 * If submit got -EBUSY, flag us as needing the application
		 * to enter the kernel to reap and flush events.
		 */
		if (!to_submit || ret == -EBUSY) {
J
Jens Axboe 已提交
5063 5064 5065
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
5066 5067 5068
			 * to sleep. The exception is if we got EBUSY doing
			 * more IO, we should wait for the application to
			 * reap events and wake us up.
J
Jens Axboe 已提交
5069
			 */
5070 5071
			if (inflight ||
			    (!time_after(jiffies, timeout) && ret != -EBUSY)) {
5072
				cond_resched();
J
Jens Axboe 已提交
5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091
				continue;
			}

			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
			if (cur_mm) {
				unuse_mm(cur_mm);
				mmput(cur_mm);
				cur_mm = NULL;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);

			/* Tell userspace we may need a wakeup call */
5092
			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
5093 5094
			/* make sure to read SQ tail after writing flags */
			smp_mb();
J
Jens Axboe 已提交
5095

5096
			to_submit = io_sqring_entries(ctx);
5097
			if (!to_submit || ret == -EBUSY) {
5098
				if (kthread_should_park()) {
J
Jens Axboe 已提交
5099 5100 5101 5102 5103 5104 5105 5106
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

5107
				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
5108 5109 5110 5111
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

5112
			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
5113 5114
		}

5115
		mutex_lock(&ctx->uring_lock);
5116
		ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
5117
		mutex_unlock(&ctx->uring_lock);
5118 5119
		if (ret > 0)
			inflight += ret;
J
Jens Axboe 已提交
5120 5121 5122 5123 5124 5125 5126
	}

	set_fs(old_fs);
	if (cur_mm) {
		unuse_mm(cur_mm);
		mmput(cur_mm);
	}
5127
	revert_creds(old_cred);
5128

5129
	kthread_parkme();
5130

J
Jens Axboe 已提交
5131 5132 5133
	return 0;
}

5134 5135 5136 5137 5138 5139 5140
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

5141
static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
5142 5143 5144 5145
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
5146
	 * Wake up if we have enough events, or if a timeout occurred since we
5147 5148 5149
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
5150
	return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
5151 5152 5153 5154 5155 5156 5157 5158 5159
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

5160 5161
	/* use noflush == true, as we can't safely rely on locking context */
	if (!io_should_wake(iowq, true))
5162 5163 5164 5165 5166
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
5167 5168 5169 5170 5171 5172 5173
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
5174 5175 5176 5177 5178 5179 5180 5181 5182
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
5183
	struct io_rings *rings = ctx->rings;
5184
	int ret = 0;
J
Jens Axboe 已提交
5185

5186
	if (io_cqring_events(ctx, false) >= min_events)
J
Jens Axboe 已提交
5187 5188 5189
		return 0;

	if (sig) {
5190 5191 5192
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
5193
						      sigsz);
5194 5195
		else
#endif
5196
			ret = set_user_sigmask(sig, sigsz);
5197

J
Jens Axboe 已提交
5198 5199 5200 5201
		if (ret)
			return ret;
	}

5202
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
5203
	trace_io_uring_cqring_wait(ctx, min_events);
5204 5205 5206
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
5207
		if (io_should_wake(&iowq, false))
5208 5209 5210
			break;
		schedule();
		if (signal_pending(current)) {
5211
			ret = -EINTR;
5212 5213 5214 5215 5216
			break;
		}
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

5217
	restore_saved_sigmask_unless(ret == -EINTR);
J
Jens Axboe 已提交
5218

5219
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
5220 5221
}

J
Jens Axboe 已提交
5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

5235 5236 5237 5238 5239 5240 5241
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
5242 5243 5244
#endif
}

5245 5246 5247 5248 5249 5250 5251 5252
static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}

J
Jens Axboe 已提交
5253 5254
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
5255
	struct fixed_file_data *data = ctx->file_data;
5256 5257
	unsigned nr_tables, i;

5258
	if (!data)
J
Jens Axboe 已提交
5259 5260
		return -ENXIO;

5261 5262
	/* protect against inflight atomic switch, which drops the ref */
	percpu_ref_get(&data->refs);
5263 5264
	/* wait for existing switches */
	flush_work(&data->ref_work);
5265 5266 5267
	percpu_ref_kill_and_confirm(&data->refs, io_file_ref_kill);
	wait_for_completion(&data->done);
	percpu_ref_put(&data->refs);
5268 5269
	/* flush potential new switch */
	flush_work(&data->ref_work);
5270 5271
	percpu_ref_exit(&data->refs);

J
Jens Axboe 已提交
5272
	__io_sqe_files_unregister(ctx);
5273 5274
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
5275 5276 5277 5278
		kfree(data->table[i].files);
	kfree(data->table);
	kfree(data);
	ctx->file_data = NULL;
J
Jens Axboe 已提交
5279 5280 5281 5282
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
5283 5284 5285
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
5286
		wait_for_completion(&ctx->completions[1]);
5287 5288 5289 5290 5291
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
5292
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
5293 5294 5295 5296 5297
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
5298 5299
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
5300 5301
	io_sq_thread_stop(ctx);

5302 5303 5304
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318
	}
}

#if defined(CONFIG_UNIX)
/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
5319
	int i, nr_files;
J
Jens Axboe 已提交
5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339

	if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
		unsigned long inflight = ctx->user->unix_inflight + nr;

		if (inflight > task_rlimit(current, RLIMIT_NOFILE))
			return -EMFILE;
	}

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

5340
	nr_files = 0;
J
Jens Axboe 已提交
5341 5342
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
5343 5344 5345
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
5346
			continue;
5347
		fpl->fp[nr_files] = get_file(file);
5348 5349
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
5350 5351
	}

5352 5353 5354 5355
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
5356
		skb->destructor = unix_destruct_scm;
5357 5358
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
5359

5360 5361 5362 5363 5364 5365
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
5396 5397 5398 5399
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

5412 5413 5414 5415 5416 5417
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
5418
		struct fixed_file_table *table = &ctx->file_data->table[i];
5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
5433
		struct fixed_file_table *table = &ctx->file_data->table[i];
5434 5435 5436 5437 5438
		kfree(table->files);
	}
	return 1;
}

5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507
static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
	fput(file);
#endif
}

struct io_file_put {
	struct llist_node llist;
	struct file *file;
	struct completion *done;
};

static void io_ring_file_ref_switch(struct work_struct *work)
5508
{
5509 5510 5511
	struct io_file_put *pfile, *tmp;
	struct fixed_file_data *data;
	struct llist_node *node;
5512

5513
	data = container_of(work, struct fixed_file_data, ref_work);
5514

5515 5516 5517 5518 5519 5520 5521 5522
	while ((node = llist_del_all(&data->put_llist)) != NULL) {
		llist_for_each_entry_safe(pfile, tmp, node, llist) {
			io_ring_file_put(data->ctx, pfile->file);
			if (pfile->done)
				complete(pfile->done);
			else
				kfree(pfile);
		}
5523 5524
	}

5525 5526 5527
	percpu_ref_get(&data->refs);
	percpu_ref_switch_to_percpu(&data->refs);
}
5528

5529 5530 5531 5532 5533 5534 5535 5536
static void io_file_data_ref_zero(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);

	/* we can't safely switch from inside this context, punt to wq */
	queue_work(system_wq, &data->ref_work);
5537 5538
}

J
Jens Axboe 已提交
5539 5540 5541 5542
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
5543
	unsigned nr_tables;
5544
	struct file *file;
J
Jens Axboe 已提交
5545 5546 5547
	int fd, ret = 0;
	unsigned i;

5548
	if (ctx->file_data)
J
Jens Axboe 已提交
5549 5550 5551 5552 5553 5554
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

5555 5556 5557 5558 5559 5560
	ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
	if (!ctx->file_data)
		return -ENOMEM;
	ctx->file_data->ctx = ctx;
	init_completion(&ctx->file_data->done);

5561
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
5562 5563
	ctx->file_data->table = kcalloc(nr_tables,
					sizeof(struct fixed_file_table),
5564
					GFP_KERNEL);
5565 5566 5567
	if (!ctx->file_data->table) {
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
5568
		return -ENOMEM;
5569 5570 5571 5572 5573 5574 5575
	}

	if (percpu_ref_init(&ctx->file_data->refs, io_file_data_ref_zero,
				PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
5576
		return -ENOMEM;
5577 5578 5579
	}
	ctx->file_data->put_llist.first = NULL;
	INIT_WORK(&ctx->file_data->ref_work, io_ring_file_ref_switch);
J
Jens Axboe 已提交
5580

5581
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
5582 5583 5584 5585
		percpu_ref_exit(&ctx->file_data->refs);
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
5586 5587 5588
		return -ENOMEM;
	}

5589
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
5590 5591 5592
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
5593 5594 5595
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
5596 5597 5598 5599 5600
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
5601

5602
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5603
		index = i & IORING_FILE_TABLE_MASK;
5604
		file = fget(fd);
J
Jens Axboe 已提交
5605 5606

		ret = -EBADF;
5607
		if (!file)
J
Jens Axboe 已提交
5608
			break;
5609

J
Jens Axboe 已提交
5610 5611 5612 5613 5614 5615 5616
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
5617 5618
		if (file->f_op == &io_uring_fops) {
			fput(file);
J
Jens Axboe 已提交
5619 5620 5621
			break;
		}
		ret = 0;
5622
		table->files[index] = file;
J
Jens Axboe 已提交
5623 5624 5625
	}

	if (ret) {
5626 5627 5628 5629 5630 5631
		for (i = 0; i < ctx->nr_user_files; i++) {
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
5632
			kfree(ctx->file_data->table[i].files);
J
Jens Axboe 已提交
5633

5634 5635 5636
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
	if (ret)
		io_sqe_files_unregister(ctx);

	return ret;
}

5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690
static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

5691
static void io_atomic_switch(struct percpu_ref *ref)
5692
{
5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	clear_bit(FFD_F_ATOMIC, &data->state);
}

static bool io_queue_file_removal(struct fixed_file_data *data,
				  struct file *file)
{
	struct io_file_put *pfile, pfile_stack;
	DECLARE_COMPLETION_ONSTACK(done);

	/*
	 * If we fail allocating the struct we need for doing async reomval
	 * of this file, just punt to sync and wait for it.
	 */
	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
	if (!pfile) {
		pfile = &pfile_stack;
		pfile->done = &done;
	}

	pfile->file = file;
	llist_add(&pfile->llist, &data->put_llist);

	if (pfile == &pfile_stack) {
		if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
			percpu_ref_put(&data->refs);
			percpu_ref_switch_to_atomic(&data->refs,
							io_atomic_switch);
		}
		wait_for_completion(&done);
		flush_work(&data->ref_work);
		return false;
	}

	return true;
}

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *up,
				 unsigned nr_args)
{
	struct fixed_file_data *data = ctx->file_data;
	bool ref_switch = false;
	struct file *file;
5739 5740 5741 5742
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;

5743
	if (check_add_overflow(up->offset, nr_args, &done))
5744 5745 5746 5747 5748
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

	done = 0;
5749
	fds = u64_to_user_ptr(up->fds);
5750
	while (nr_args) {
5751 5752 5753
		struct fixed_file_table *table;
		unsigned index;

5754 5755 5756 5757 5758
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
5759 5760
		i = array_index_nospec(up->offset, ctx->nr_user_files);
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5761 5762
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
5763
			file = io_file_from_index(ctx, index);
5764
			table->files[index] = NULL;
5765 5766
			if (io_queue_file_removal(data, file))
				ref_switch = true;
5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786
		}
		if (fd != -1) {
			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
5787
			table->files[index] = file;
5788 5789 5790 5791 5792 5793
			err = io_sqe_file_register(ctx, file, i);
			if (err)
				break;
		}
		nr_args--;
		done++;
5794 5795 5796 5797 5798 5799
		up->offset++;
	}

	if (ref_switch && !test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
		percpu_ref_put(&data->refs);
		percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
5800 5801 5802 5803
	}

	return done ? done : err;
}
5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819
static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}
5820

5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834
static void io_put_work(struct io_wq_work *work)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	io_put_req(req);
}

static void io_get_work(struct io_wq_work *work)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	refcount_inc(&req->refs);
}

5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881
static int io_init_wq_offload(struct io_ring_ctx *ctx,
			      struct io_uring_params *p)
{
	struct io_wq_data data;
	struct fd f;
	struct io_ring_ctx *ctx_attach;
	unsigned int concurrency;
	int ret = 0;

	data.user = ctx->user;
	data.get_work = io_get_work;
	data.put_work = io_put_work;

	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
		/* Do QD, or 4 * CPUS, whatever is smallest */
		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

		ctx->io_wq = io_wq_create(concurrency, &data);
		if (IS_ERR(ctx->io_wq)) {
			ret = PTR_ERR(ctx->io_wq);
			ctx->io_wq = NULL;
		}
		return ret;
	}

	f = fdget(p->wq_fd);
	if (!f.file)
		return -EBADF;

	if (f.file->f_op != &io_uring_fops) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx_attach = f.file->private_data;
	/* @io_wq is protected by holding the fd */
	if (!io_wq_get(ctx_attach->io_wq, &data)) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx->io_wq = ctx_attach->io_wq;
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
5882 5883
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
5884 5885 5886
{
	int ret;

J
Jens Axboe 已提交
5887
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
5888 5889 5890
	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

J
Jens Axboe 已提交
5891
	if (ctx->flags & IORING_SETUP_SQPOLL) {
5892 5893 5894 5895
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

5896 5897 5898 5899
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
5900
		if (p->flags & IORING_SETUP_SQ_AFF) {
5901
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
5902

5903
			ret = -EINVAL;
5904 5905
			if (cpu >= nr_cpu_ids)
				goto err;
5906
			if (!cpu_online(cpu))
5907 5908
				goto err;

J
Jens Axboe 已提交
5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

5928 5929
	ret = io_init_wq_offload(ctx, p);
	if (ret)
J
Jens Axboe 已提交
5930 5931 5932 5933
		goto err;

	return 0;
err:
5934
	io_finish_async(ctx);
J
Jens Axboe 已提交
5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964
	mmdrop(ctx->sqo_mm);
	ctx->sqo_mm = NULL;
	return ret;
}

static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

static void io_mem_free(void *ptr)
{
5965 5966 5967 5968
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
5969

5970
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	if (sq_offset)
		*sq_offset = off;

	return off;
}

J
Jens Axboe 已提交
6012 6013
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
6014
	size_t pages;
J
Jens Axboe 已提交
6015

6016 6017 6018 6019
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
6020

6021
	return pages;
J
Jens Axboe 已提交
6022 6023
}

6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
6035
			put_user_page(imu->bvec[j].bv_page);
6036 6037 6038

		if (ctx->account_mem)
			io_unaccount_mem(ctx->user, imu->nr_bvecs);
6039
		kvfree(imu->bvec);
6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

6063
		dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
6101
			goto err;
6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

		if (ctx->account_mem) {
			ret = io_account_mem(ctx->user, nr_pages);
			if (ret)
				goto err;
		}

		ret = 0;
		if (!pages || nr_pages > got_pages) {
			kfree(vmas);
			kfree(pages);
6131
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
6132
						GFP_KERNEL);
6133
			vmas = kvmalloc_array(nr_pages,
6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
				if (ctx->account_mem)
					io_unaccount_mem(ctx->user, nr_pages);
				goto err;
			}
			got_pages = nr_pages;
		}

6145
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
6146 6147 6148 6149 6150 6151 6152 6153 6154 6155
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
			goto err;
		}

		ret = 0;
		down_read(&current->mm->mmap_sem);
6156 6157 6158
		pret = get_user_pages(ubuf, nr_pages,
				      FOLL_WRITE | FOLL_LONGTERM,
				      pages, vmas);
6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
		up_read(&current->mm->mmap_sem);
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
6179 6180
			if (pret > 0)
				put_user_pages(pages, pret);
6181 6182
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
6183
			kvfree(imu->bvec);
6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
6206 6207
	kvfree(pages);
	kvfree(vmas);
6208 6209
	return 0;
err:
6210 6211
	kvfree(pages);
	kvfree(vmas);
6212 6213 6214 6215
	io_sqe_buffer_unregister(ctx);
	return ret;
}

6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

J
Jens Axboe 已提交
6248 6249
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
6250
	io_finish_async(ctx);
J
Jens Axboe 已提交
6251 6252
	if (ctx->sqo_mm)
		mmdrop(ctx->sqo_mm);
J
Jens Axboe 已提交
6253 6254

	io_iopoll_reap_events(ctx);
6255
	io_sqe_buffer_unregister(ctx);
J
Jens Axboe 已提交
6256
	io_sqe_files_unregister(ctx);
6257
	io_eventfd_unregister(ctx);
J
Jens Axboe 已提交
6258

J
Jens Axboe 已提交
6259
#if defined(CONFIG_UNIX)
6260 6261
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
6262
		sock_release(ctx->ring_sock);
6263
	}
J
Jens Axboe 已提交
6264 6265
#endif

6266
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
6267 6268 6269 6270 6271 6272 6273
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	if (ctx->account_mem)
		io_unaccount_mem(ctx->user,
				ring_pages(ctx->sq_entries, ctx->cq_entries));
	free_uid(ctx->user);
6274
	put_cred(ctx->creds);
6275
	kfree(ctx->completions);
6276
	kfree(ctx->cancel_hash);
6277
	kmem_cache_free(req_cachep, ctx->fallback_req);
J
Jens Axboe 已提交
6278 6279 6280 6281 6282 6283 6284 6285 6286
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
6287 6288 6289 6290
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
6291
	smp_rmb();
6292 6293
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
6294
		mask |= EPOLLOUT | EPOLLWRNORM;
6295
	if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
J
Jens Axboe 已提交
6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318
static int io_remove_personalities(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	const struct cred *cred;

	cred = idr_remove(&ctx->personality_idr, id);
	if (cred)
		put_cred(cred);
	return 0;
}

J
Jens Axboe 已提交
6319 6320 6321 6322 6323 6324
static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
6325
	io_kill_timeouts(ctx);
6326
	io_poll_remove_all(ctx);
6327 6328 6329 6330

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

J
Jens Axboe 已提交
6331
	io_iopoll_reap_events(ctx);
6332 6333 6334
	/* if we failed setting up the ctx, we might not have any rings */
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);
6335
	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
6336
	wait_for_completion(&ctx->completions[0]);
J
Jens Axboe 已提交
6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348
	io_ring_ctx_free(ctx);
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

6349 6350 6351 6352 6353 6354 6355
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
	struct io_kiocb *req;
	DEFINE_WAIT(wait);

	while (!list_empty_careful(&ctx->inflight_list)) {
6356
		struct io_kiocb *cancel_req = NULL;
6357 6358 6359

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
6360 6361 6362 6363 6364 6365 6366
			if (req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
6367
		}
6368
		if (cancel_req)
6369
			prepare_to_wait(&ctx->inflight_wait, &wait,
6370
						TASK_UNINTERRUPTIBLE);
6371 6372
		spin_unlock_irq(&ctx->inflight_lock);

6373 6374
		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
6375
			break;
6376 6377 6378

		io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
		io_put_req(cancel_req);
6379 6380
		schedule();
	}
6381
	finish_wait(&ctx->inflight_wait, &wait);
6382 6383 6384 6385 6386 6387 6388 6389 6390 6391
}

static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
	return 0;
}

6392 6393
static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
J
Jens Axboe 已提交
6394 6395
{
	struct io_ring_ctx *ctx = file->private_data;
6396
	loff_t offset = pgoff << PAGE_SHIFT;
J
Jens Axboe 已提交
6397 6398 6399 6400 6401
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
6402 6403
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
6404 6405 6406 6407 6408
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
6409
		return ERR_PTR(-EINVAL);
J
Jens Axboe 已提交
6410 6411 6412
	}

	page = virt_to_head_page(ptr);
6413
	if (sz > page_size(page))
6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429
		return ERR_PTR(-EINVAL);

	return ptr;
}

#ifdef CONFIG_MMU

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);
J
Jens Axboe 已提交
6430 6431 6432 6433 6434

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461
#else /* !CONFIG_MMU */

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
}

static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
{
	return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
}

static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len,
	unsigned long pgoff, unsigned long flags)
{
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, pgoff, len);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	return (unsigned long) ptr;
}

#endif /* !CONFIG_MMU */

J
Jens Axboe 已提交
6462 6463 6464 6465 6466 6467 6468 6469 6470
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

J
Jens Axboe 已提交
6471
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
6487 6488 6489 6490 6491
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
6492
	ret = 0;
J
Jens Axboe 已提交
6493
	if (ctx->flags & IORING_SETUP_SQPOLL) {
6494 6495
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false);
J
Jens Axboe 已提交
6496 6497 6498
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
6499
	} else if (to_submit) {
6500
		struct mm_struct *cur_mm;
J
Jens Axboe 已提交
6501 6502

		mutex_lock(&ctx->uring_lock);
6503 6504 6505 6506
		/* already have mm, so io_submit_sqes() won't try to grab it */
		cur_mm = ctx->sqo_mm;
		submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
					   &cur_mm, false);
J
Jens Axboe 已提交
6507
		mutex_unlock(&ctx->uring_lock);
6508 6509 6510

		if (submitted != to_submit)
			goto out;
J
Jens Axboe 已提交
6511 6512
	}
	if (flags & IORING_ENTER_GETEVENTS) {
J
Jens Axboe 已提交
6513 6514
		unsigned nr_events = 0;

J
Jens Axboe 已提交
6515 6516
		min_complete = min(min_complete, ctx->cq_entries);

J
Jens Axboe 已提交
6517 6518 6519 6520 6521
		if (ctx->flags & IORING_SETUP_IOPOLL) {
			ret = io_iopoll_check(ctx, &nr_events, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
6522 6523
	}

6524
out:
6525
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
6526 6527 6528 6529 6530
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604
static int io_uring_show_cred(int id, void *p, void *data)
{
	const struct cred *cred = p;
	struct seq_file *m = data;
	struct user_namespace *uns = seq_user_ns(m);
	struct group_info *gi;
	kernel_cap_t cap;
	unsigned __capi;
	int g;

	seq_printf(m, "%5d\n", id);
	seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
	seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
	seq_puts(m, "\n\tGroups:\t");
	gi = cred->group_info;
	for (g = 0; g < gi->ngroups; g++) {
		seq_put_decimal_ull(m, g ? " " : "",
					from_kgid_munged(uns, gi->gid[g]));
	}
	seq_puts(m, "\n\tCapEff:\t");
	cap = cred->cap_effective;
	CAP_FOR_EACH_U32(__capi)
		seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
	seq_putc(m, '\n');
	return 0;
}

static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	int i;

	mutex_lock(&ctx->uring_lock);
	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (!idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
	mutex_unlock(&ctx->uring_lock);
}

static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
{
	struct io_ring_ctx *ctx = f->private_data;

	if (percpu_ref_tryget(&ctx->refs)) {
		__io_uring_show_fdinfo(ctx, m);
		percpu_ref_put(&ctx->refs);
	}
}

J
Jens Axboe 已提交
6605 6606
static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
6607
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
6608
	.mmap		= io_uring_mmap,
6609 6610 6611 6612
#ifndef CONFIG_MMU
	.get_unmapped_area = io_uring_nommu_get_unmapped_area,
	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
#endif
J
Jens Axboe 已提交
6613 6614
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
6615
	.show_fdinfo	= io_uring_show_fdinfo,
J
Jens Axboe 已提交
6616 6617 6618 6619 6620
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
6621 6622
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
6623

6624 6625 6626 6627 6628 6629
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
6630 6631
		return -ENOMEM;

6632 6633 6634 6635 6636 6637 6638 6639 6640 6641
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
	ctx->sq_entries = rings->sq_ring_entries;
	ctx->cq_entries = rings->cq_ring_entries;
J
Jens Axboe 已提交
6642 6643

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
6644 6645 6646
	if (size == SIZE_MAX) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
6647
		return -EOVERFLOW;
6648
	}
J
Jens Axboe 已提交
6649 6650

	ctx->sq_sqes = io_mem_alloc(size);
6651 6652 6653
	if (!ctx->sq_sqes) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
6654
		return -ENOMEM;
6655
	}
J
Jens Axboe 已提交
6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

static int io_uring_create(unsigned entries, struct io_uring_params *p)
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
	bool account_mem;
	int ret;

6710
	if (!entries)
J
Jens Axboe 已提交
6711
		return -EINVAL;
6712 6713 6714 6715 6716
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}
J
Jens Axboe 已提交
6717 6718 6719 6720 6721

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
6722 6723 6724
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
6725 6726
	 */
	p->sq_entries = roundup_pow_of_two(entries);
6727 6728 6729 6730 6731 6732
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
6733
		if (p->cq_entries < p->sq_entries)
6734
			return -EINVAL;
6735 6736 6737 6738 6739
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
6740 6741 6742 6743
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767

	user = get_uid(current_user());
	account_mem = !capable(CAP_IPC_LOCK);

	if (account_mem) {
		ret = io_account_mem(user,
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
		if (account_mem)
			io_unaccount_mem(user, ring_pages(p->sq_entries,
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->account_mem = account_mem;
	ctx->user = user;
6768
	ctx->creds = get_current_cred();
J
Jens Axboe 已提交
6769 6770 6771 6772 6773

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
6774
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
6775 6776 6777 6778
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
6779 6780 6781 6782 6783 6784 6785
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
6786 6787

	memset(&p->cq_off, 0, sizeof(p->cq_off));
6788 6789 6790 6791 6792 6793
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
6794

6795 6796 6797 6798 6799 6800 6801 6802
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

6803
	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
6804 6805
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
			IORING_FEAT_CUR_PERSONALITY;
6806
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
J
Jens Axboe 已提交
6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	long ret;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
6831
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
6832
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
6833
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
J
Jens Axboe 已提交
6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851
		return -EINVAL;

	ret = io_uring_create(entries, &p);
	if (ret < 0)
		return ret;

	if (copy_to_user(params, &p, sizeof(p)))
		return -EFAULT;

	return ret;
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890
static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}

6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929
static int io_register_personality(struct io_ring_ctx *ctx)
{
	const struct cred *creds = get_current_cred();
	int id;

	id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
				USHRT_MAX, GFP_KERNEL);
	if (id < 0)
		put_cred(creds);
	return id;
}

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}

static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}

6930 6931
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
6932 6933
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
6934 6935 6936
{
	int ret;

6937 6938 6939 6940 6941 6942 6943 6944
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

6945
	if (io_register_op_must_quiesce(opcode)) {
6946
		percpu_ref_kill(&ctx->refs);
6947

6948 6949 6950 6951 6952 6953 6954 6955 6956
		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
6957
		ret = wait_for_completion_interruptible(&ctx->completions[0]);
6958
		mutex_lock(&ctx->uring_lock);
6959 6960 6961 6962 6963
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
6964
	}
6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
6976 6977 6978 6979 6980 6981 6982 6983 6984
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
6985 6986 6987
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
6988
	case IORING_REGISTER_EVENTFD:
6989
	case IORING_REGISTER_EVENTFD_ASYNC:
6990 6991 6992 6993
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
6994 6995 6996 6997 6998 6999
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
7000 7001 7002 7003 7004 7005 7006
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
7007 7008 7009 7010 7011 7012
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
7025 7026 7027 7028 7029
	default:
		ret = -EINVAL;
		break;
	}

7030
	if (io_register_op_must_quiesce(opcode)) {
7031 7032
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
7033 7034
out:
		reinit_completion(&ctx->completions[0]);
7035
	}
7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
7059 7060
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
7061 7062 7063 7064 7065
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
7066 7067
static int __init io_uring_init(void)
{
7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
	BUILD_BUG_SQE_ELEM(28, __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);

7101
	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
J
Jens Axboe 已提交
7102 7103 7104 7105
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);