io_uring.c 165.8 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46 47 48
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
#include <linux/refcount.h>
#include <linux/uio.h>
49
#include <linux/bits.h>
J
Jens Axboe 已提交
50 51 52 53 54 55 56 57 58 59

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/mmu_context.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
60
#include <linux/kthread.h>
J
Jens Axboe 已提交
61
#include <linux/blkdev.h>
62
#include <linux/bvec.h>
J
Jens Axboe 已提交
63 64 65
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
66
#include <net/scm.h>
J
Jens Axboe 已提交
67 68 69 70
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
71 72
#include <linux/sizes.h>
#include <linux/hugetlb.h>
73
#include <linux/highmem.h>
74 75
#include <linux/namei.h>
#include <linux/fsnotify.h>
J
Jens Axboe 已提交
76
#include <linux/fadvise.h>
77
#include <linux/eventpoll.h>
J
Jens Axboe 已提交
78

79 80 81
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
82 83 84
#include <uapi/linux/io_uring.h>

#include "internal.h"
85
#include "io-wq.h"
J
Jens Axboe 已提交
86

87
#define IORING_MAX_ENTRIES	32768
88
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
89 90 91 92 93 94 95 96

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
97 98 99 100 101 102

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
103
/*
104 105
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
106 107 108 109
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
110
struct io_rings {
S
Stefan Bühler 已提交
111 112 113 114
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
115 116 117
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
118
	 */
119
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
120
	/*
121
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
122 123
	 * ring_entries - 1)
	 */
124 125 126
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
127 128 129 130 131 132 133 134 135 136 137 138
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
139
	u32			sq_dropped;
S
Stefan Bühler 已提交
140 141 142 143 144 145 146 147 148
	/*
	 * Runtime flags
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
149
	u32			sq_flags;
S
Stefan Bühler 已提交
150 151 152
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
L
LimingWu 已提交
153
	 * there are not more requests pending than there is space in
S
Stefan Bühler 已提交
154 155 156 157 158 159 160 161 162
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
163
	u32			cq_overflow;
S
Stefan Bühler 已提交
164 165 166 167 168 169 170
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
171
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
172 173
};

174 175 176 177 178 179 180
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

181 182 183 184
struct fixed_file_table {
	struct file		**files;
};

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
enum {
	FFD_F_ATOMIC,
};

struct fixed_file_data {
	struct fixed_file_table		*table;
	struct io_ring_ctx		*ctx;

	struct percpu_ref		refs;
	struct llist_head		put_llist;
	unsigned long			state;
	struct work_struct		ref_work;
	struct completion		done;
};

J
Jens Axboe 已提交
200 201 202 203 204 205 206
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
207 208 209 210
		int			compat: 1;
		int			account_mem: 1;
		int			cq_overflow_flushed: 1;
		int			drain_next: 1;
211
		int			eventfd_async: 1;
J
Jens Axboe 已提交
212

213 214 215 216 217 218 219 220 221 222 223 224
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
225 226 227
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
228
		unsigned		sq_thread_idle;
229
		unsigned		cached_sq_dropped;
230
		atomic_t		cached_cq_overflow;
231
		unsigned long		sq_check_overflow;
232 233

		struct list_head	defer_list;
J
Jens Axboe 已提交
234
		struct list_head	timeout_list;
235
		struct list_head	cq_overflow_list;
236 237

		wait_queue_head_t	inflight_wait;
238
		struct io_uring_sqe	*sq_sqes;
J
Jens Axboe 已提交
239 240
	} ____cacheline_aligned_in_smp;

241 242
	struct io_rings	*rings;

J
Jens Axboe 已提交
243
	/* IO offload */
244
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
245
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
246
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
247
	wait_queue_head_t	sqo_wait;
248

J
Jens Axboe 已提交
249 250 251 252 253
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
254
	struct fixed_file_data	*file_data;
J
Jens Axboe 已提交
255
	unsigned		nr_user_files;
P
Pavel Begunkov 已提交
256 257
	int 			ring_fd;
	struct file 		*ring_file;
J
Jens Axboe 已提交
258

259 260 261 262
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
263 264
	struct user_struct	*user;

265
	const struct cred	*creds;
266

267 268 269
	/* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
	struct completion	*completions;

270 271 272
	/* if all else fails... */
	struct io_kiocb		*fallback_req;

273 274 275 276
#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif

277 278
	struct idr		personality_idr;

279 280 281 282 283
	struct {
		unsigned		cached_cq_tail;
		unsigned		cq_entries;
		unsigned		cq_mask;
		atomic_t		cq_timeouts;
284
		unsigned long		cq_check_overflow;
285 286 287 288
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
		struct eventfd_ctx	*cq_ev_fd;
	} ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
289 290 291 292 293 294 295 296

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
297 298
		struct llist_head	poll_llist;

J
Jens Axboe 已提交
299 300 301 302 303 304 305
		/*
		 * ->poll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct list_head	poll_list;
306 307
		struct hlist_head	*cancel_hash;
		unsigned		cancel_hash_bits;
308
		bool			poll_multi_file;
309 310 311

		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
312 313 314
	} ____cacheline_aligned_in_smp;
};

J
Jens Axboe 已提交
315 316 317 318
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
319 320
struct io_poll_iocb {
	struct file			*file;
321 322 323 324
	union {
		struct wait_queue_head	*head;
		u64			addr;
	};
325
	__poll_t			events;
J
Jens Axboe 已提交
326
	bool				done;
327
	bool				canceled;
328
	struct wait_queue_entry		wait;
329 330
};

331 332 333 334 335 336
struct io_close {
	struct file			*file;
	struct file			*put_file;
	int				fd;
};

337 338 339 340 341
struct io_timeout_data {
	struct io_kiocb			*req;
	struct hrtimer			timer;
	struct timespec64		ts;
	enum hrtimer_mode		mode;
342
	u32				seq_offset;
343 344
};

345 346 347 348 349 350 351 352 353 354 355 356
struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
};

struct io_sync {
	struct file			*file;
	loff_t				len;
	loff_t				off;
	int				flags;
357
	int				mode;
358 359
};

360 361 362 363 364
struct io_cancel {
	struct file			*file;
	u64				addr;
};

365 366 367 368
struct io_timeout {
	struct file			*file;
	u64				addr;
	int				flags;
369
	unsigned			count;
370 371
};

372 373 374 375 376 377 378
struct io_rw {
	/* NOTE: kiocb has the file as the first member, so don't do it here */
	struct kiocb			kiocb;
	u64				addr;
	u64				len;
};

379 380 381 382 383 384
struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
};

385 386
struct io_sr_msg {
	struct file			*file;
387 388 389 390
	union {
		struct user_msghdr __user *msg;
		void __user		*buf;
	};
391
	int				msg_flags;
392
	size_t				len;
393 394
};

395 396 397
struct io_open {
	struct file			*file;
	int				dfd;
398 399 400
	union {
		unsigned		mask;
	};
401
	struct filename			*filename;
402
	struct statx __user		*buffer;
403
	struct open_how			how;
404 405
};

406 407 408 409 410 411 412
struct io_files_update {
	struct file			*file;
	u64				arg;
	u32				nr_args;
	u32				offset;
};

J
Jens Axboe 已提交
413 414 415 416 417 418 419
struct io_fadvise {
	struct file			*file;
	u64				offset;
	u32				len;
	u32				advice;
};

J
Jens Axboe 已提交
420 421 422 423 424 425 426
struct io_madvise {
	struct file			*file;
	u64				addr;
	u32				len;
	u32				advice;
};

427 428 429 430 431 432 433 434
struct io_epoll {
	struct file			*file;
	int				epfd;
	int				op;
	int				fd;
	struct epoll_event		event;
};

435 436 437 438
struct io_async_connect {
	struct sockaddr_storage		address;
};

439 440 441 442 443 444 445
struct io_async_msghdr {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	struct sockaddr __user		*uaddr;
	struct msghdr			msg;
};

446 447 448 449 450 451 452
struct io_async_rw {
	struct iovec			fast_iov[UIO_FASTIOV];
	struct iovec			*iov;
	ssize_t				nr_segs;
	ssize_t				size;
};

453 454 455 456
struct io_async_open {
	struct filename			*filename;
};

457
struct io_async_ctx {
458 459
	union {
		struct io_async_rw	rw;
460
		struct io_async_msghdr	msg;
461
		struct io_async_connect	connect;
462
		struct io_timeout_data	timeout;
463
		struct io_async_open	open;
464
	};
465 466
};

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
enum {
	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,

	REQ_F_LINK_NEXT_BIT,
	REQ_F_FAIL_LINK_BIT,
	REQ_F_INFLIGHT_BIT,
	REQ_F_CUR_POS_BIT,
	REQ_F_NOWAIT_BIT,
	REQ_F_IOPOLL_COMPLETED_BIT,
	REQ_F_LINK_TIMEOUT_BIT,
	REQ_F_TIMEOUT_BIT,
	REQ_F_ISREG_BIT,
	REQ_F_MUST_PUNT_BIT,
	REQ_F_TIMEOUT_NOSEQ_BIT,
	REQ_F_COMP_LOCKED_BIT,
};

enum {
	/* ctx owns file */
	REQ_F_FIXED_FILE	= BIT(REQ_F_FIXED_FILE_BIT),
	/* drain existing IO first */
	REQ_F_IO_DRAIN		= BIT(REQ_F_IO_DRAIN_BIT),
	/* linked sqes */
	REQ_F_LINK		= BIT(REQ_F_LINK_BIT),
	/* doesn't sever on completion < 0 */
	REQ_F_HARDLINK		= BIT(REQ_F_HARDLINK_BIT),
	/* IOSQE_ASYNC */
	REQ_F_FORCE_ASYNC	= BIT(REQ_F_FORCE_ASYNC_BIT),

	/* already grabbed next link */
	REQ_F_LINK_NEXT		= BIT(REQ_F_LINK_NEXT_BIT),
	/* fail rest of links */
	REQ_F_FAIL_LINK		= BIT(REQ_F_FAIL_LINK_BIT),
	/* on inflight list */
	REQ_F_INFLIGHT		= BIT(REQ_F_INFLIGHT_BIT),
	/* read/write uses file position */
	REQ_F_CUR_POS		= BIT(REQ_F_CUR_POS_BIT),
	/* must not punt to workers */
	REQ_F_NOWAIT		= BIT(REQ_F_NOWAIT_BIT),
	/* polled IO has completed */
	REQ_F_IOPOLL_COMPLETED	= BIT(REQ_F_IOPOLL_COMPLETED_BIT),
	/* has linked timeout */
	REQ_F_LINK_TIMEOUT	= BIT(REQ_F_LINK_TIMEOUT_BIT),
	/* timeout request */
	REQ_F_TIMEOUT		= BIT(REQ_F_TIMEOUT_BIT),
	/* regular file */
	REQ_F_ISREG		= BIT(REQ_F_ISREG_BIT),
	/* must be punted even for NONBLOCK */
	REQ_F_MUST_PUNT		= BIT(REQ_F_MUST_PUNT_BIT),
	/* no timeout sequence */
	REQ_F_TIMEOUT_NOSEQ	= BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
	/* completion under lock */
	REQ_F_COMP_LOCKED	= BIT(REQ_F_COMP_LOCKED_BIT),
};

J
Jens Axboe 已提交
526 527 528 529 530 531
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
532
struct io_kiocb {
533
	union {
J
Jens Axboe 已提交
534
		struct file		*file;
535
		struct io_rw		rw;
536
		struct io_poll_iocb	poll;
537 538
		struct io_accept	accept;
		struct io_sync		sync;
539
		struct io_cancel	cancel;
540
		struct io_timeout	timeout;
541
		struct io_connect	connect;
542
		struct io_sr_msg	sr_msg;
543
		struct io_open		open;
544
		struct io_close		close;
545
		struct io_files_update	files_update;
J
Jens Axboe 已提交
546
		struct io_fadvise	fadvise;
J
Jens Axboe 已提交
547
		struct io_madvise	madvise;
548
		struct io_epoll		epoll;
549
	};
J
Jens Axboe 已提交
550

551
	struct io_async_ctx		*io;
P
Pavel Begunkov 已提交
552 553 554 555
	/*
	 * llist_node is only used for poll deferred completions
	 */
	struct llist_node		llist_node;
556 557 558
	bool				has_user;
	bool				in_async;
	bool				needs_fixed_file;
559
	u8				opcode;
J
Jens Axboe 已提交
560 561

	struct io_ring_ctx	*ctx;
562 563
	union {
		struct list_head	list;
564
		struct hlist_node	hash_node;
565
	};
J
Jens Axboe 已提交
566
	struct list_head	link_list;
J
Jens Axboe 已提交
567
	unsigned int		flags;
J
Jens Axboe 已提交
568
	refcount_t		refs;
J
Jens Axboe 已提交
569
	u64			user_data;
J
Jens Axboe 已提交
570
	u32			result;
571
	u32			sequence;
J
Jens Axboe 已提交
572

573 574
	struct list_head	inflight_entry;

575
	struct io_wq_work	work;
J
Jens Axboe 已提交
576 577 578
};

#define IO_PLUG_THRESHOLD		2
J
Jens Axboe 已提交
579
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
580

581 582 583
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
584 585 586 587
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
588
	unsigned int		free_reqs;
J
Jens Axboe 已提交
589

590 591 592 593 594 595 596 597 598 599
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		used_refs;
	unsigned int		ios_left;
};

600 601 602 603 604 605 606 607 608 609 610 611 612
struct io_op_def {
	/* needs req->io allocated for deferral/async */
	unsigned		async_ctx : 1;
	/* needs current->mm setup, does mm access */
	unsigned		needs_mm : 1;
	/* needs req->file assigned */
	unsigned		needs_file : 1;
	/* needs req->file assigned IFF fd is >= 0 */
	unsigned		fd_non_neg : 1;
	/* hash wq insertion if file is a regular file */
	unsigned		hash_reg_file : 1;
	/* unbound wq insertion if file is a non-regular file */
	unsigned		unbound_nonreg_file : 1;
613 614
	/* opcode is not supported by this kernel */
	unsigned		not_supported : 1;
615 616
	/* needs file table */
	unsigned		file_table : 1;
617 618 619
};

static const struct io_op_def io_op_defs[] = {
620 621
	[IORING_OP_NOP] = {},
	[IORING_OP_READV] = {
622 623 624 625 626
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
627
	[IORING_OP_WRITEV] = {
628 629 630 631 632 633
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
634
	[IORING_OP_FSYNC] = {
635 636
		.needs_file		= 1,
	},
637
	[IORING_OP_READ_FIXED] = {
638 639 640
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
641
	[IORING_OP_WRITE_FIXED] = {
642 643 644 645
		.needs_file		= 1,
		.hash_reg_file		= 1,
		.unbound_nonreg_file	= 1,
	},
646
	[IORING_OP_POLL_ADD] = {
647 648 649
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
650 651
	[IORING_OP_POLL_REMOVE] = {},
	[IORING_OP_SYNC_FILE_RANGE] = {
652 653
		.needs_file		= 1,
	},
654
	[IORING_OP_SENDMSG] = {
655 656 657 658 659
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
660
	[IORING_OP_RECVMSG] = {
661 662 663 664 665
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
666
	[IORING_OP_TIMEOUT] = {
667 668 669
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
670 671
	[IORING_OP_TIMEOUT_REMOVE] = {},
	[IORING_OP_ACCEPT] = {
672 673 674
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
675
		.file_table		= 1,
676
	},
677 678
	[IORING_OP_ASYNC_CANCEL] = {},
	[IORING_OP_LINK_TIMEOUT] = {
679 680 681
		.async_ctx		= 1,
		.needs_mm		= 1,
	},
682
	[IORING_OP_CONNECT] = {
683 684 685 686 687
		.async_ctx		= 1,
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
688
	[IORING_OP_FALLOCATE] = {
689 690
		.needs_file		= 1,
	},
691
	[IORING_OP_OPENAT] = {
692 693
		.needs_file		= 1,
		.fd_non_neg		= 1,
694
		.file_table		= 1,
695
	},
696
	[IORING_OP_CLOSE] = {
697
		.needs_file		= 1,
698
		.file_table		= 1,
699
	},
700
	[IORING_OP_FILES_UPDATE] = {
701
		.needs_mm		= 1,
702
		.file_table		= 1,
703
	},
704
	[IORING_OP_STATX] = {
705 706 707 708
		.needs_mm		= 1,
		.needs_file		= 1,
		.fd_non_neg		= 1,
	},
709
	[IORING_OP_READ] = {
710 711 712 713
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
714
	[IORING_OP_WRITE] = {
715 716 717 718
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
719
	[IORING_OP_FADVISE] = {
J
Jens Axboe 已提交
720 721
		.needs_file		= 1,
	},
722
	[IORING_OP_MADVISE] = {
J
Jens Axboe 已提交
723 724
		.needs_mm		= 1,
	},
725
	[IORING_OP_SEND] = {
726 727 728 729
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
730
	[IORING_OP_RECV] = {
731 732 733 734
		.needs_mm		= 1,
		.needs_file		= 1,
		.unbound_nonreg_file	= 1,
	},
735
	[IORING_OP_OPENAT2] = {
736 737
		.needs_file		= 1,
		.fd_non_neg		= 1,
738
		.file_table		= 1,
739
	},
740 741 742 743
	[IORING_OP_EPOLL_CTL] = {
		.unbound_nonreg_file	= 1,
		.file_table		= 1,
	},
744 745
};

746
static void io_wq_submit_work(struct io_wq_work **workptr);
747
static void io_cqring_fill_event(struct io_kiocb *req, long res);
748
static void io_put_req(struct io_kiocb *req);
749
static void __io_double_put_req(struct io_kiocb *req);
750 751
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
static void io_queue_linked_timeout(struct io_kiocb *req);
752 753 754
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *ip,
				 unsigned nr_args);
755
static int io_grab_files(struct io_kiocb *req);
756

J
Jens Axboe 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

778
	complete(&ctx->completions[0]);
J
Jens Axboe 已提交
779 780 781 782 783
}

static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;
784
	int hash_bits;
J
Jens Axboe 已提交
785 786 787 788 789

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

790 791 792 793
	ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
	if (!ctx->fallback_req)
		goto err;

794 795 796 797
	ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
	if (!ctx->completions)
		goto err;

798 799 800 801 802 803 804 805 806 807 808 809 810 811 812
	/*
	 * Use 5 bits less than the max cq entries, that should give us around
	 * 32 entries per hash list if totally full and uniformly spread.
	 */
	hash_bits = ilog2(p->cq_entries);
	hash_bits -= 5;
	if (hash_bits <= 0)
		hash_bits = 1;
	ctx->cancel_hash_bits = hash_bits;
	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
					GFP_KERNEL);
	if (!ctx->cancel_hash)
		goto err;
	__hash_init(ctx->cancel_hash, 1U << hash_bits);

813 814
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
815
		goto err;
J
Jens Axboe 已提交
816 817 818

	ctx->flags = p->flags;
	init_waitqueue_head(&ctx->cq_wait);
819
	INIT_LIST_HEAD(&ctx->cq_overflow_list);
820 821
	init_completion(&ctx->completions[0]);
	init_completion(&ctx->completions[1]);
822
	idr_init(&ctx->personality_idr);
J
Jens Axboe 已提交
823 824 825
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
826
	init_llist_head(&ctx->poll_llist);
J
Jens Axboe 已提交
827
	INIT_LIST_HEAD(&ctx->poll_list);
828
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
829
	INIT_LIST_HEAD(&ctx->timeout_list);
830 831 832
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
J
Jens Axboe 已提交
833
	return ctx;
834
err:
835 836
	if (ctx->fallback_req)
		kmem_cache_free(req_cachep, ctx->fallback_req);
837
	kfree(ctx->completions);
838
	kfree(ctx->cancel_hash);
839 840
	kfree(ctx);
	return NULL;
J
Jens Axboe 已提交
841 842
}

B
Bob Liu 已提交
843
static inline bool __req_need_defer(struct io_kiocb *req)
844
{
845 846
	struct io_ring_ctx *ctx = req->ctx;

847 848
	return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
					+ atomic_read(&ctx->cached_cq_overflow);
849 850
}

B
Bob Liu 已提交
851
static inline bool req_need_defer(struct io_kiocb *req)
852
{
853
	if (unlikely(req->flags & REQ_F_IO_DRAIN))
B
Bob Liu 已提交
854
		return __req_need_defer(req);
855

B
Bob Liu 已提交
856
	return false;
857 858
}

859
static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
860 861 862
{
	struct io_kiocb *req;

863
	req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
B
Bob Liu 已提交
864
	if (req && !req_need_defer(req)) {
865 866 867 868 869 870 871
		list_del_init(&req->list);
		return req;
	}

	return NULL;
}

J
Jens Axboe 已提交
872 873
static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
{
874 875 876
	struct io_kiocb *req;

	req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
877 878 879
	if (req) {
		if (req->flags & REQ_F_TIMEOUT_NOSEQ)
			return NULL;
B
Bob Liu 已提交
880
		if (!__req_need_defer(req)) {
881 882 883
			list_del_init(&req->list);
			return req;
		}
884 885 886
	}

	return NULL;
J
Jens Axboe 已提交
887 888
}

889
static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
890
{
891
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
892

893 894
	/* order cqe stores with ring update */
	smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
895

896 897 898
	if (wq_has_sleeper(&ctx->cq_wait)) {
		wake_up_interruptible(&ctx->cq_wait);
		kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
J
Jens Axboe 已提交
899 900 901
	}
}

902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
static inline void io_req_work_grab_env(struct io_kiocb *req,
					const struct io_op_def *def)
{
	if (!req->work.mm && def->needs_mm) {
		mmgrab(current->mm);
		req->work.mm = current->mm;
	}
	if (!req->work.creds)
		req->work.creds = get_current_cred();
}

static inline void io_req_work_drop_env(struct io_kiocb *req)
{
	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
}

925 926
static inline bool io_prep_async_work(struct io_kiocb *req,
				      struct io_kiocb **link)
927
{
928
	const struct io_op_def *def = &io_op_defs[req->opcode];
929
	bool do_hashed = false;
930

931 932
	if (req->flags & REQ_F_ISREG) {
		if (def->hash_reg_file)
933
			do_hashed = true;
934 935
	} else {
		if (def->unbound_nonreg_file)
936
			req->work.flags |= IO_WQ_WORK_UNBOUND;
937
	}
938 939

	io_req_work_grab_env(req, def);
940

941
	*link = io_prep_linked_timeout(req);
942 943 944
	return do_hashed;
}

945
static inline void io_queue_async_work(struct io_kiocb *req)
946
{
947
	struct io_ring_ctx *ctx = req->ctx;
948 949 950 951
	struct io_kiocb *link;
	bool do_hashed;

	do_hashed = io_prep_async_work(req, &link);
952 953 954 955 956 957 958 959 960

	trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
					req->flags);
	if (!do_hashed) {
		io_wq_enqueue(ctx->io_wq, &req->work);
	} else {
		io_wq_enqueue_hashed(ctx->io_wq, &req->work,
					file_inode(req->file));
	}
961 962 963

	if (link)
		io_queue_linked_timeout(link);
964 965
}

J
Jens Axboe 已提交
966 967 968 969
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

970
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
J
Jens Axboe 已提交
971 972
	if (ret != -1) {
		atomic_inc(&req->ctx->cq_timeouts);
973
		list_del_init(&req->list);
974
		io_cqring_fill_event(req, 0);
975
		io_put_req(req);
J
Jens Axboe 已提交
976 977 978 979 980 981 982 983 984 985 986 987 988
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

989 990 991 992
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

J
Jens Axboe 已提交
993 994 995
	while ((req = io_get_timeout_req(ctx)) != NULL)
		io_kill_timeout(req);

996 997
	__io_commit_cqring(ctx);

998
	while ((req = io_get_deferred_req(ctx)) != NULL)
999
		io_queue_async_work(req);
1000 1001
}

J
Jens Axboe 已提交
1002 1003
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
1004
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
1005 1006 1007
	unsigned tail;

	tail = ctx->cached_cq_tail;
1008 1009 1010 1011 1012
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
1013
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
1014 1015 1016
		return NULL;

	ctx->cached_cq_tail++;
1017
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
1018 1019
}

1020 1021
static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
{
1022 1023
	if (!ctx->cq_ev_fd)
		return false;
1024 1025 1026 1027 1028
	if (!ctx->eventfd_async)
		return true;
	return io_wq_current_is_worker() || in_interrupt();
}

1029
static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
1030 1031 1032 1033 1034
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
1035
	if (trigger_ev)
1036 1037 1038
		eventfd_signal(ctx->cq_ev_fd, 1);
}

1039 1040 1041 1042 1043
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
{
	__io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
}

1044 1045
/* Returns true if there are no backlogged entries after the flush */
static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1046 1047 1048 1049 1050 1051 1052 1053 1054
{
	struct io_rings *rings = ctx->rings;
	struct io_uring_cqe *cqe;
	struct io_kiocb *req;
	unsigned long flags;
	LIST_HEAD(list);

	if (!force) {
		if (list_empty_careful(&ctx->cq_overflow_list))
1055
			return true;
1056 1057
		if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
		    rings->cq_ring_entries))
1058
			return false;
1059 1060 1061 1062 1063 1064
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/* if force is set, the ring is going away. always drop after that */
	if (force)
1065
		ctx->cq_overflow_flushed = 1;
1066

1067
	cqe = NULL;
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	while (!list_empty(&ctx->cq_overflow_list)) {
		cqe = io_get_cqring(ctx);
		if (!cqe && !force)
			break;

		req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
						list);
		list_move(&req->list, &list);
		if (cqe) {
			WRITE_ONCE(cqe->user_data, req->user_data);
			WRITE_ONCE(cqe->res, req->result);
			WRITE_ONCE(cqe->flags, 0);
		} else {
			WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
		}
	}

	io_commit_cqring(ctx);
1087 1088 1089 1090
	if (cqe) {
		clear_bit(0, &ctx->sq_check_overflow);
		clear_bit(0, &ctx->cq_check_overflow);
	}
1091 1092 1093 1094 1095 1096
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	while (!list_empty(&list)) {
		req = list_first_entry(&list, struct io_kiocb, list);
		list_del(&req->list);
1097
		io_put_req(req);
1098
	}
1099 1100

	return cqe != NULL;
1101 1102
}

1103
static void io_cqring_fill_event(struct io_kiocb *req, long res)
J
Jens Axboe 已提交
1104
{
1105
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1106 1107
	struct io_uring_cqe *cqe;

1108
	trace_io_uring_complete(ctx, req->user_data, res);
1109

J
Jens Axboe 已提交
1110 1111 1112 1113 1114 1115
	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
1116
	if (likely(cqe)) {
1117
		WRITE_ONCE(cqe->user_data, req->user_data);
J
Jens Axboe 已提交
1118
		WRITE_ONCE(cqe->res, res);
1119
		WRITE_ONCE(cqe->flags, 0);
1120
	} else if (ctx->cq_overflow_flushed) {
1121 1122
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
1123
	} else {
1124 1125 1126 1127
		if (list_empty(&ctx->cq_overflow_list)) {
			set_bit(0, &ctx->sq_check_overflow);
			set_bit(0, &ctx->cq_check_overflow);
		}
1128 1129 1130
		refcount_inc(&req->refs);
		req->result = res;
		list_add_tail(&req->list, &ctx->cq_overflow_list);
J
Jens Axboe 已提交
1131 1132 1133
	}
}

1134
static void io_cqring_add_event(struct io_kiocb *req, long res)
J
Jens Axboe 已提交
1135
{
1136
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1137 1138 1139
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
1140
	io_cqring_fill_event(req, res);
J
Jens Axboe 已提交
1141 1142 1143
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
1144
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1145 1146
}

1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
static inline bool io_is_fallback_req(struct io_kiocb *req)
{
	return req == (struct io_kiocb *)
			((unsigned long) req->ctx->fallback_req & ~1UL);
}

static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	req = ctx->fallback_req;
	if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
		return req;

	return NULL;
}

J
Jens Axboe 已提交
1164 1165
static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
				   struct io_submit_state *state)
J
Jens Axboe 已提交
1166
{
1167
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
1168 1169
	struct io_kiocb *req;

J
Jens Axboe 已提交
1170
	if (!state) {
1171
		req = kmem_cache_alloc(req_cachep, gfp);
J
Jens Axboe 已提交
1172
		if (unlikely(!req))
1173
			goto fallback;
J
Jens Axboe 已提交
1174 1175 1176 1177 1178
	} else if (!state->free_reqs) {
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
1179 1180 1181 1182 1183 1184 1185 1186 1187
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
1188
				goto fallback;
1189 1190
			ret = 1;
		}
J
Jens Axboe 已提交
1191
		state->free_reqs = ret - 1;
1192
		req = state->reqs[ret - 1];
J
Jens Axboe 已提交
1193 1194
	} else {
		state->free_reqs--;
1195
		req = state->reqs[state->free_reqs];
J
Jens Axboe 已提交
1196 1197
	}

1198
got_it:
1199
	req->io = NULL;
1200
	req->file = NULL;
J
Jens Axboe 已提交
1201 1202
	req->ctx = ctx;
	req->flags = 0;
1203 1204
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
J
Jens Axboe 已提交
1205
	req->result = 0;
1206
	INIT_IO_WORK(&req->work, io_wq_submit_work);
J
Jens Axboe 已提交
1207
	return req;
1208 1209 1210 1211
fallback:
	req = io_get_fallback_req(ctx);
	if (req)
		goto got_it;
1212
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
1213 1214 1215
	return NULL;
}

1216 1217 1218 1219 1220 1221 1222 1223
static void __io_req_do_free(struct io_kiocb *req)
{
	if (likely(!io_is_fallback_req(req)))
		kmem_cache_free(req_cachep, req);
	else
		clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
}

1224
static void __io_req_aux_free(struct io_kiocb *req)
J
Jens Axboe 已提交
1225
{
1226 1227
	struct io_ring_ctx *ctx = req->ctx;

1228
	kfree(req->io);
1229 1230 1231 1232 1233 1234
	if (req->file) {
		if (req->flags & REQ_F_FIXED_FILE)
			percpu_ref_put(&ctx->file_data->refs);
		else
			fput(req->file);
	}
1235 1236

	io_req_work_drop_env(req);
1237 1238 1239 1240 1241 1242
}

static void __io_free_req(struct io_kiocb *req)
{
	__io_req_aux_free(req);

1243
	if (req->flags & REQ_F_INFLIGHT) {
1244
		struct io_ring_ctx *ctx = req->ctx;
1245 1246 1247 1248 1249 1250 1251 1252
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
	}
1253 1254 1255

	percpu_ref_put(&req->ctx->refs);
	__io_req_do_free(req);
1256 1257
}

1258 1259 1260 1261 1262 1263 1264 1265
struct req_batch {
	void *reqs[IO_IOPOLL_BATCH];
	int to_free;
	int need_iter;
};

static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
{
1266 1267
	int fixed_refs = rb->to_free;

1268 1269 1270 1271 1272 1273
	if (!rb->to_free)
		return;
	if (rb->need_iter) {
		int i, inflight = 0;
		unsigned long flags;

1274
		fixed_refs = 0;
1275 1276 1277
		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

1278
			if (req->flags & REQ_F_FIXED_FILE) {
1279
				req->file = NULL;
1280 1281
				fixed_refs++;
			}
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
			if (req->flags & REQ_F_INFLIGHT)
				inflight++;
			__io_req_aux_free(req);
		}
		if (!inflight)
			goto do_free;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		for (i = 0; i < rb->to_free; i++) {
			struct io_kiocb *req = rb->reqs[i];

1293
			if (req->flags & REQ_F_INFLIGHT) {
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305
				list_del(&req->inflight_entry);
				if (!--inflight)
					break;
			}
		}
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);

		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
	}
do_free:
	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
1306 1307
	if (fixed_refs)
		percpu_ref_put_many(&ctx->file_data->refs, fixed_refs);
1308 1309 1310 1311
	percpu_ref_put_many(&ctx->refs, rb->to_free);
	rb->to_free = rb->need_iter = 0;
}

1312
static bool io_link_cancel_timeout(struct io_kiocb *req)
1313
{
1314
	struct io_ring_ctx *ctx = req->ctx;
1315 1316
	int ret;

1317
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1318
	if (ret != -1) {
1319
		io_cqring_fill_event(req, -ECANCELED);
1320 1321
		io_commit_cqring(ctx);
		req->flags &= ~REQ_F_LINK;
1322
		io_put_req(req);
1323 1324 1325 1326 1327 1328
		return true;
	}

	return false;
}

1329
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
J
Jens Axboe 已提交
1330
{
1331 1332
	struct io_ring_ctx *ctx = req->ctx;
	bool wake_ev = false;
J
Jens Axboe 已提交
1333

1334 1335 1336 1337
	/* Already got next link */
	if (req->flags & REQ_F_LINK_NEXT)
		return;

J
Jens Axboe 已提交
1338 1339 1340 1341 1342
	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
1343 1344 1345
	while (!list_empty(&req->link_list)) {
		struct io_kiocb *nxt = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
1346

1347 1348 1349
		if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
			     (nxt->flags & REQ_F_TIMEOUT))) {
			list_del_init(&nxt->link_list);
1350 1351 1352 1353
			wake_ev |= io_link_cancel_timeout(nxt);
			req->flags &= ~REQ_F_LINK_TIMEOUT;
			continue;
		}
J
Jens Axboe 已提交
1354

1355 1356 1357
		list_del_init(&req->link_list);
		if (!list_empty(&nxt->link_list))
			nxt->flags |= REQ_F_LINK;
1358
		*nxtptr = nxt;
1359
		break;
J
Jens Axboe 已提交
1360
	}
1361

1362
	req->flags |= REQ_F_LINK_NEXT;
1363 1364
	if (wake_ev)
		io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1365 1366 1367 1368 1369 1370 1371
}

/*
 * Called if REQ_F_LINK is set, and we fail the head request
 */
static void io_fail_links(struct io_kiocb *req)
{
1372 1373 1374 1375
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
J
Jens Axboe 已提交
1376 1377

	while (!list_empty(&req->link_list)) {
1378 1379
		struct io_kiocb *link = list_first_entry(&req->link_list,
						struct io_kiocb, link_list);
J
Jens Axboe 已提交
1380

1381
		list_del_init(&link->link_list);
1382
		trace_io_uring_fail_link(req, link);
1383 1384

		if ((req->flags & REQ_F_LINK_TIMEOUT) &&
1385
		    link->opcode == IORING_OP_LINK_TIMEOUT) {
1386
			io_link_cancel_timeout(link);
1387
		} else {
1388
			io_cqring_fill_event(link, -ECANCELED);
1389
			__io_double_put_req(link);
1390
		}
1391
		req->flags &= ~REQ_F_LINK_TIMEOUT;
J
Jens Axboe 已提交
1392
	}
1393 1394 1395 1396

	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
1397 1398
}

1399
static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
J
Jens Axboe 已提交
1400
{
1401
	if (likely(!(req->flags & REQ_F_LINK)))
1402 1403
		return;

J
Jens Axboe 已提交
1404 1405 1406 1407 1408 1409
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
1410 1411
	if (req->flags & REQ_F_FAIL_LINK) {
		io_fail_links(req);
1412 1413
	} else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
			REQ_F_LINK_TIMEOUT) {
1414 1415 1416 1417 1418 1419
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;

		/*
		 * If this is a timeout link, we could be racing with the
		 * timeout timer. Grab the completion lock for this case to
1420
		 * protect against that.
1421 1422 1423 1424 1425 1426
		 */
		spin_lock_irqsave(&ctx->completion_lock, flags);
		io_req_link_next(req, nxt);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
		io_req_link_next(req, nxt);
J
Jens Axboe 已提交
1427
	}
1428
}
J
Jens Axboe 已提交
1429

1430 1431
static void io_free_req(struct io_kiocb *req)
{
1432 1433 1434
	struct io_kiocb *nxt = NULL;

	io_req_find_next(req, &nxt);
1435
	__io_free_req(req);
1436 1437 1438

	if (nxt)
		io_queue_async_work(nxt);
1439 1440
}

1441 1442 1443 1444
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
1445
__attribute__((nonnull))
1446
static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
1447
{
1448
	io_req_find_next(req, nxtptr);
1449

1450
	if (refcount_dec_and_test(&req->refs))
1451
		__io_free_req(req);
J
Jens Axboe 已提交
1452 1453
}

1454 1455 1456
static void io_put_req(struct io_kiocb *req)
{
	if (refcount_dec_and_test(&req->refs))
1457
		io_free_req(req);
1458 1459
}

1460 1461 1462 1463 1464
/*
 * Must only be used if we don't need to care about links, usually from
 * within the completion handling itself.
 */
static void __io_double_put_req(struct io_kiocb *req)
1465 1466 1467 1468 1469 1470
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		__io_free_req(req);
}

1471 1472 1473 1474 1475 1476 1477
static void io_double_put_req(struct io_kiocb *req)
{
	/* drop both submit and complete references */
	if (refcount_sub_and_test(2, &req->refs))
		io_free_req(req);
}

1478
static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
1479
{
1480 1481
	struct io_rings *rings = ctx->rings;

1482 1483 1484 1485 1486 1487 1488 1489
	if (test_bit(0, &ctx->cq_check_overflow)) {
		/*
		 * noflush == true is from the waitqueue handler, just ensure
		 * we wake up the task, and the next invocation will flush the
		 * entries. We cannot safely to it from here.
		 */
		if (noflush && !list_empty(&ctx->cq_overflow_list))
			return -1U;
1490

1491 1492
		io_cqring_overflow_flush(ctx, false);
	}
1493

1494 1495
	/* See comment at the top of this file */
	smp_rmb();
1496
	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
1497 1498
}

1499 1500 1501 1502 1503 1504 1505 1506
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

1507
static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
1508
{
1509 1510
	if ((req->flags & REQ_F_LINK) || io_is_fallback_req(req))
		return false;
1511

1512 1513 1514 1515 1516 1517 1518
	if (!(req->flags & REQ_F_FIXED_FILE) || req->io)
		rb->need_iter++;

	rb->reqs[rb->to_free++] = req;
	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		io_free_req_many(req->ctx, rb);
	return true;
1519 1520
}

J
Jens Axboe 已提交
1521 1522 1523 1524 1525 1526
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
1527
	struct req_batch rb;
J
Jens Axboe 已提交
1528 1529
	struct io_kiocb *req;

1530
	rb.to_free = rb.need_iter = 0;
J
Jens Axboe 已提交
1531 1532 1533 1534
	while (!list_empty(done)) {
		req = list_first_entry(done, struct io_kiocb, list);
		list_del(&req->list);

1535
		io_cqring_fill_event(req, req->result);
J
Jens Axboe 已提交
1536 1537
		(*nr_events)++;

1538 1539 1540
		if (refcount_dec_and_test(&req->refs) &&
		    !io_req_multi_free(&rb, req))
			io_free_req(req);
J
Jens Axboe 已提交
1541 1542
	}

J
Jens Axboe 已提交
1543
	io_commit_cqring(ctx);
1544 1545
	if (ctx->flags & IORING_SETUP_SQPOLL)
		io_cqring_ev_posted(ctx);
1546
	io_free_req_many(ctx, &rb);
J
Jens Axboe 已提交
1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
}

static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
1565
		struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594

		/*
		 * Move completed entries to our local list. If we find a
		 * request that requires polling, break out and complete
		 * the done list first, if we have entries there.
		 */
		if (req->flags & REQ_F_IOPOLL_COMPLETED) {
			list_move_tail(&req->list, &done);
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
1595
 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
J
Jens Axboe 已提交
1596 1597 1598 1599 1600 1601
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
1602
	while (!list_empty(&ctx->poll_list) && !need_resched()) {
J
Jens Axboe 已提交
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
	while (!list_empty(&ctx->poll_list)) {
		unsigned int nr_events = 0;

		io_iopoll_getevents(ctx, &nr_events, 1);
1629 1630 1631 1632 1633 1634

		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
		 */
		cond_resched();
J
Jens Axboe 已提交
1635 1636 1637 1638
	}
	mutex_unlock(&ctx->uring_lock);
}

1639 1640
static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			   long min)
J
Jens Axboe 已提交
1641
{
1642
	int iters = 0, ret = 0;
1643

1644 1645 1646 1647 1648 1649
	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
J
Jens Axboe 已提交
1650 1651 1652
	do {
		int tmin = 0;

1653 1654 1655 1656 1657
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
1658
		if (io_cqring_events(ctx, false))
1659 1660
			break;

1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
			mutex_lock(&ctx->uring_lock);
		}

J
Jens Axboe 已提交
1676 1677 1678 1679 1680 1681 1682 1683 1684
		if (*nr_events < min)
			tmin = min - *nr_events;

		ret = io_iopoll_getevents(ctx, nr_events, tmin);
		if (ret <= 0)
			break;
		ret = 0;
	} while (min && !*nr_events && !need_resched());

1685
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
1686 1687 1688
	return ret;
}

1689
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
1690
{
1691 1692 1693 1694 1695 1696
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
1697

1698
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
1699
	}
1700
	file_end_write(req->file);
J
Jens Axboe 已提交
1701 1702
}

J
Jens Axboe 已提交
1703 1704 1705 1706 1707 1708
static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}

1709
static void io_complete_rw_common(struct kiocb *kiocb, long res)
J
Jens Axboe 已提交
1710
{
1711
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
1712

1713 1714
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1715

J
Jens Axboe 已提交
1716 1717
	if (res != req->result)
		req_set_fail_links(req);
1718
	io_cqring_add_event(req, res);
1719 1720 1721 1722
}

static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
1723
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1724 1725

	io_complete_rw_common(kiocb, res);
1726
	io_put_req(req);
1727 1728 1729 1730
}

static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
{
1731
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1732
	struct io_kiocb *nxt = NULL;
1733 1734

	io_complete_rw_common(kiocb, res);
1735 1736 1737
	io_put_req_find_next(req, &nxt);

	return nxt;
J
Jens Axboe 已提交
1738 1739
}

J
Jens Axboe 已提交
1740 1741
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
1742
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
J
Jens Axboe 已提交
1743

1744 1745
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1746

J
Jens Axboe 已提交
1747 1748
	if (res != req->result)
		req_set_fail_links(req);
J
Jens Axboe 已提交
1749
	req->result = res;
J
Jens Axboe 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775
	if (res != -EAGAIN)
		req->flags |= REQ_F_IOPOLL_COMPLETED;
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->poll_list)) {
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

		list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
						list);
1776
		if (list_req->file != req->file)
J
Jens Axboe 已提交
1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
	if (req->flags & REQ_F_IOPOLL_COMPLETED)
		list_add(&req->list, &ctx->poll_list);
	else
		list_add_tail(&req->list, &ctx->poll_list);
1788 1789 1790 1791

	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
	    wq_has_sleeper(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
J
Jens Axboe 已提交
1792 1793
}

1794
static void io_file_put(struct io_submit_state *state)
1795
{
1796
	if (state->file) {
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
		int diff = state->has_refs - state->used_refs;

		if (diff)
			fput_many(state->file, diff);
		state->file = NULL;
	}
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
static struct file *io_file_get(struct io_submit_state *state, int fd)
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
			state->used_refs++;
			state->ios_left--;
			return state->file;
		}
1821
		io_file_put(state);
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->has_refs = state->ios_left;
	state->used_refs = 1;
	state->ios_left--;
	return state->file;
}

J
Jens Axboe 已提交
1834 1835 1836 1837 1838 1839 1840 1841 1842
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
static bool io_file_supports_async(struct file *file)
{
	umode_t mode = file_inode(file)->i_mode;

1843
	if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
J
Jens Axboe 已提交
1844 1845 1846 1847 1848 1849 1850
		return true;
	if (S_ISREG(mode) && file->f_op != &io_uring_fops)
		return true;

	return false;
}

1851 1852
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		      bool force_nonblock)
J
Jens Axboe 已提交
1853
{
J
Jens Axboe 已提交
1854
	struct io_ring_ctx *ctx = req->ctx;
1855
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
1856 1857
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
1858

1859 1860 1861
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

J
Jens Axboe 已提交
1862
	kiocb->ki_pos = READ_ONCE(sqe->off);
1863 1864 1865 1866
	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
		req->flags |= REQ_F_CUR_POS;
		kiocb->ki_pos = req->file->f_pos;
	}
J
Jens Axboe 已提交
1867 1868 1869 1870 1871 1872 1873
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
1874
			return ret;
J
Jens Axboe 已提交
1875 1876 1877 1878 1879 1880 1881

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
J
Jens Axboe 已提交
1882
		return ret;
1883 1884

	/* don't allow async punt if RWF_NOWAIT was requested */
1885 1886
	if ((kiocb->ki_flags & IOCB_NOWAIT) ||
	    (req->file->f_flags & O_NONBLOCK))
1887 1888 1889
		req->flags |= REQ_F_NOWAIT;

	if (force_nonblock)
J
Jens Axboe 已提交
1890
		kiocb->ki_flags |= IOCB_NOWAIT;
1891

J
Jens Axboe 已提交
1892 1893 1894
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
1895
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
1896

J
Jens Axboe 已提交
1897 1898
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
1899
		req->result = 0;
J
Jens Axboe 已提交
1900
	} else {
J
Jens Axboe 已提交
1901 1902
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
1903 1904
		kiocb->ki_complete = io_complete_rw;
	}
1905

1906 1907
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
1908 1909
	/* we own ->private, reuse it for the buffer index */
	req->rw.kiocb.private = (void *) (unsigned long)
1910
					READ_ONCE(sqe->buf_index);
J
Jens Axboe 已提交
1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

1935 1936 1937
static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
		       bool in_async)
{
1938 1939 1940 1941
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

	if (req->flags & REQ_F_CUR_POS)
		req->file->f_pos = kiocb->ki_pos;
1942
	if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
1943 1944 1945 1946 1947
		*nxt = __io_complete_rw(kiocb, ret);
	else
		io_rw_done(kiocb, ret);
}

1948
static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
1949
			       struct iov_iter *iter)
1950
{
1951 1952
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
1953 1954 1955 1956 1957 1958 1959 1960 1961
	struct io_mapped_ubuf *imu;
	unsigned index, buf_index;
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

1962
	buf_index = (unsigned long) req->rw.kiocb.private;
1963 1964 1965 1966 1967
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
1968
	buf_addr = req->rw.addr;
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
2014
			iter->count -= bvec->bv_len + offset;
2015 2016 2017 2018
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

2019
	return len;
2020 2021
}

2022 2023
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
			       struct iovec **iovec, struct iov_iter *iter)
J
Jens Axboe 已提交
2024
{
2025 2026
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
2027 2028
	u8 opcode;

2029
	opcode = req->opcode;
2030
	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2031
		*iovec = NULL;
2032
		return io_import_fixed(req, rw, iter);
2033
	}
J
Jens Axboe 已提交
2034

2035 2036 2037 2038
	/* buffer index only valid with fixed read/write */
	if (req->rw.kiocb.private)
		return -EINVAL;

2039 2040 2041 2042 2043 2044 2045
	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
		ssize_t ret;
		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
		return ret;
	}

2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
	if (req->io) {
		struct io_async_rw *iorw = &req->io->rw;

		*iovec = iorw->iov;
		iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
		if (iorw->iov == iorw->fast_iov)
			*iovec = NULL;
		return iorw->size;
	}

2056
	if (!req->has_user)
J
Jens Axboe 已提交
2057 2058 2059
		return -EFAULT;

#ifdef CONFIG_COMPAT
2060
	if (req->ctx->compat)
J
Jens Axboe 已提交
2061 2062 2063 2064 2065 2066 2067
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
/*
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
 */
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
2088
		struct iovec iovec;
2089 2090
		ssize_t nr;

2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

2101 2102 2103 2104 2105 2106 2107 2108
		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, &kiocb->ki_pos);
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, &kiocb->ki_pos);
		}

2109 2110 2111
		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

2126
static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
			  struct iovec *iovec, struct iovec *fast_iov,
			  struct iov_iter *iter)
{
	req->io->rw.nr_segs = iter->nr_segs;
	req->io->rw.size = io_size;
	req->io->rw.iov = iovec;
	if (!req->io->rw.iov) {
		req->io->rw.iov = req->io->rw.fast_iov;
		memcpy(req->io->rw.iov, fast_iov,
			sizeof(struct iovec) * iter->nr_segs);
	}
}

2140
static int io_alloc_async_ctx(struct io_kiocb *req)
2141
{
2142 2143
	if (!io_op_defs[req->opcode].async_ctx)
		return 0;
2144
	req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
2145
	return req->io == NULL;
2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162
}

static void io_rw_async(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct iovec *iov = NULL;

	if (req->io->rw.iov != req->io->rw.fast_iov)
		iov = req->io->rw.iov;
	io_wq_submit_work(workptr);
	kfree(iov);
}

static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
			     struct iovec *iovec, struct iovec *fast_iov,
			     struct iov_iter *iter)
{
2163
	if (!io_op_defs[req->opcode].async_ctx)
2164
		return 0;
2165 2166 2167
	if (!req->io) {
		if (io_alloc_async_ctx(req))
			return -ENOMEM;
2168

2169 2170
		io_req_map_rw(req, io_size, iovec, fast_iov, iter);
	}
2171 2172
	req->work.func = io_rw_async;
	return 0;
2173 2174
}

2175 2176
static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
2177
{
2178 2179
	struct io_async_ctx *io;
	struct iov_iter iter;
2180 2181
	ssize_t ret;

2182 2183 2184
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2185

2186 2187
	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;
2188

2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201
	if (!req->io)
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
	ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2202 2203
}

2204 2205
static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
J
Jens Axboe 已提交
2206 2207
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2208
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2209
	struct iov_iter iter;
2210
	size_t iov_count;
2211
	ssize_t io_size, ret;
J
Jens Axboe 已提交
2212

2213
	ret = io_import_iovec(READ, req, &iovec, &iter);
2214 2215
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2216

2217 2218
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2219
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2220

2221
	req->result = 0;
2222
	io_size = ret;
J
Jens Axboe 已提交
2223
	if (req->flags & REQ_F_LINK)
2224 2225 2226 2227 2228 2229
		req->result = io_size;

	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
2230
	if (force_nonblock && !io_file_supports_async(req->file)) {
2231 2232 2233
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}
J
Jens Axboe 已提交
2234

2235
	iov_count = iov_iter_count(&iter);
2236
	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2237 2238 2239
	if (!ret) {
		ssize_t ret2;

2240 2241
		if (req->file->f_op->read_iter)
			ret2 = call_read_iter(req->file, kiocb, &iter);
2242
		else
2243
			ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
2244

2245
		/* Catch -EAGAIN return for forced non-blocking submission */
2246
		if (!force_nonblock || ret2 != -EAGAIN) {
2247
			kiocb_done(kiocb, ret2, nxt, req->in_async);
2248 2249
		} else {
copy_iov:
2250
			ret = io_setup_async_rw(req, io_size, iovec,
2251 2252 2253 2254 2255
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
J
Jens Axboe 已提交
2256
	}
2257
out_free:
2258 2259
	if (!io_wq_current_is_worker())
		kfree(iovec);
J
Jens Axboe 已提交
2260 2261 2262
	return ret;
}

2263 2264
static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 bool force_nonblock)
2265
{
2266 2267
	struct io_async_ctx *io;
	struct iov_iter iter;
2268 2269
	ssize_t ret;

2270 2271 2272
	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;
2273

2274 2275
	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
		return -EBADF;
2276

2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289
	if (!req->io)
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
	ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
2290 2291
}

2292 2293
static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
J
Jens Axboe 已提交
2294 2295
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2296
	struct kiocb *kiocb = &req->rw.kiocb;
J
Jens Axboe 已提交
2297
	struct iov_iter iter;
2298
	size_t iov_count;
2299
	ssize_t ret, io_size;
J
Jens Axboe 已提交
2300

2301
	ret = io_import_iovec(WRITE, req, &iovec, &iter);
2302 2303
	if (ret < 0)
		return ret;
J
Jens Axboe 已提交
2304

2305 2306
	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
2307
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2308

2309
	req->result = 0;
2310
	io_size = ret;
J
Jens Axboe 已提交
2311
	if (req->flags & REQ_F_LINK)
2312
		req->result = io_size;
J
Jens Axboe 已提交
2313

2314 2315 2316 2317 2318 2319 2320 2321
	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
	if (force_nonblock && !io_file_supports_async(req->file)) {
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}
2322

2323 2324 2325
	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
2326
		goto copy_iov;
2327

2328
	iov_count = iov_iter_count(&iter);
2329
	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
2330
	if (!ret) {
2331 2332
		ssize_t ret2;

J
Jens Axboe 已提交
2333 2334 2335 2336 2337 2338 2339
		/*
		 * Open-code file_start_write here to grab freeze protection,
		 * which will be released by another thread in
		 * io_complete_rw().  Fool lockdep by telling it the lock got
		 * released so that it doesn't complain about the held lock when
		 * we return to userspace.
		 */
2340
		if (req->flags & REQ_F_ISREG) {
2341
			__sb_start_write(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
2342
						SB_FREEZE_WRITE, true);
2343
			__sb_writers_release(file_inode(req->file)->i_sb,
J
Jens Axboe 已提交
2344 2345 2346
						SB_FREEZE_WRITE);
		}
		kiocb->ki_flags |= IOCB_WRITE;
2347

2348 2349
		if (req->file->f_op->write_iter)
			ret2 = call_write_iter(req->file, kiocb, &iter);
2350
		else
2351
			ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
2352
		if (!force_nonblock || ret2 != -EAGAIN) {
2353
			kiocb_done(kiocb, ret2, nxt, req->in_async);
2354 2355
		} else {
copy_iov:
2356
			ret = io_setup_async_rw(req, io_size, iovec,
2357 2358 2359 2360 2361
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
J
Jens Axboe 已提交
2362
	}
2363
out_free:
2364 2365
	if (!io_wq_current_is_worker())
		kfree(iovec);
J
Jens Axboe 已提交
2366 2367 2368 2369 2370 2371
	return ret;
}

/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
2372
static int io_nop(struct io_kiocb *req)
J
Jens Axboe 已提交
2373 2374 2375
{
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
2376 2377 2378
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

2379
	io_cqring_add_event(req, 0);
2380
	io_put_req(req);
J
Jens Axboe 已提交
2381 2382 2383
	return 0;
}

2384
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
C
Christoph Hellwig 已提交
2385
{
J
Jens Axboe 已提交
2386
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
2387

J
Jens Axboe 已提交
2388 2389
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
2390

J
Jens Axboe 已提交
2391
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
2392
		return -EINVAL;
2393
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
2394 2395
		return -EINVAL;

2396 2397 2398 2399 2400 2401
	req->sync.flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
C
Christoph Hellwig 已提交
2402 2403 2404
	return 0;
}

2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416
static bool io_req_cancelled(struct io_kiocb *req)
{
	if (req->work.flags & IO_WQ_WORK_CANCEL) {
		req_set_fail_links(req);
		io_cqring_add_event(req, -ECANCELED);
		io_put_req(req);
		return true;
	}

	return false;
}

2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
static void io_link_work_cb(struct io_wq_work **workptr)
{
	struct io_wq_work *work = *workptr;
	struct io_kiocb *link = work->data;

	io_queue_linked_timeout(link);
	work->func = io_wq_submit_work;
}

static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
{
	struct io_kiocb *link;

	io_prep_async_work(nxt, &link);
	*workptr = &nxt->work;
	if (link) {
		nxt->work.flags |= IO_WQ_WORK_CB;
		nxt->work.func = io_link_work_cb;
		nxt->work.data = link;
	}
}

2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
static void io_fsync_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	loff_t end = req->sync.off + req->sync.len;
	struct io_kiocb *nxt = NULL;
	int ret;

	if (io_req_cancelled(req))
		return;

2449
	ret = vfs_fsync_range(req->file, req->sync.off,
2450 2451 2452 2453 2454 2455 2456
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
2457
		io_wq_assign_next(workptr, nxt);
2458 2459
}

2460 2461
static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
C
Christoph Hellwig 已提交
2462
{
2463
	struct io_wq_work *work, *old_work;
C
Christoph Hellwig 已提交
2464 2465

	/* fsync always requires a blocking context */
2466 2467 2468
	if (force_nonblock) {
		io_put_req(req);
		req->work.func = io_fsync_finish;
C
Christoph Hellwig 已提交
2469
		return -EAGAIN;
2470
	}
C
Christoph Hellwig 已提交
2471

2472 2473 2474 2475
	work = old_work = &req->work;
	io_fsync_finish(&work);
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);
C
Christoph Hellwig 已提交
2476 2477 2478
	return 0;
}

2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
static void io_fallocate_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;
	int ret;

	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
				req->sync.len);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
		io_wq_assign_next(workptr, nxt);
}

static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
	return 0;
}

static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
			bool force_nonblock)
{
	struct io_wq_work *work, *old_work;

	/* fallocate always requiring blocking context */
	if (force_nonblock) {
		io_put_req(req);
		req->work.func = io_fallocate_finish;
		return -EAGAIN;
	}

	work = old_work = &req->work;
	io_fallocate_finish(&work);
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);

	return 0;
}

2527 2528
static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
2529
	const char __user *fname;
2530 2531 2532 2533 2534 2535
	int ret;

	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->open.dfd = READ_ONCE(sqe->fd);
2536
	req->open.how.mode = READ_ONCE(sqe->len);
2537
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2538
	req->open.how.flags = READ_ONCE(sqe->open_flags);
2539

2540
	req->open.filename = getname(fname);
2541 2542 2543 2544 2545 2546 2547 2548 2549
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}

	return 0;
}

2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587
static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct open_how __user *how;
	const char __user *fname;
	size_t len;
	int ret;

	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->open.dfd = READ_ONCE(sqe->fd);
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	len = READ_ONCE(sqe->len);

	if (len < OPEN_HOW_SIZE_VER0)
		return -EINVAL;

	ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
					len);
	if (ret)
		return ret;

	if (!(req->open.how.flags & O_PATH) && force_o_largefile())
		req->open.how.flags |= O_LARGEFILE;

	req->open.filename = getname(fname);
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}

	return 0;
}

static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
2588 2589 2590 2591 2592
{
	struct open_flags op;
	struct file *file;
	int ret;

2593
	if (force_nonblock)
2594 2595
		return -EAGAIN;

2596
	ret = build_open_flags(&req->open.how, &op);
2597 2598 2599
	if (ret)
		goto err;

2600
	ret = get_unused_fd_flags(req->open.how.flags);
2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620
	if (ret < 0)
		goto err;

	file = do_filp_open(req->open.dfd, req->open.filename, &op);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
	} else {
		fsnotify_open(file);
		fd_install(ret, file);
	}
err:
	putname(req->open.filename);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2621 2622 2623 2624 2625 2626 2627
static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
	req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
	return io_openat2(req, nxt, force_nonblock);
}

2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673
static int io_epoll_ctl_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_EPOLL)
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->epoll.epfd = READ_ONCE(sqe->fd);
	req->epoll.op = READ_ONCE(sqe->len);
	req->epoll.fd = READ_ONCE(sqe->off);

	if (ep_op_has_event(req->epoll.op)) {
		struct epoll_event __user *ev;

		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
		if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
			return -EFAULT;
	}

	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,
			bool force_nonblock)
{
#if defined(CONFIG_EPOLL)
	struct io_epoll *ie = &req->epoll;
	int ret;

	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709
static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	if (sqe->ioprio || sqe->buf_index || sqe->off)
		return -EINVAL;

	req->madvise.addr = READ_ONCE(sqe->addr);
	req->madvise.len = READ_ONCE(sqe->len);
	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

J
Jens Axboe 已提交
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726
static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr)
		return -EINVAL;

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}

static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
	struct io_fadvise *fa = &req->fadvise;
	int ret;

2727 2728 2729 2730 2731 2732 2733 2734 2735 2736
	if (force_nonblock) {
		switch (fa->advice) {
		case POSIX_FADV_NORMAL:
		case POSIX_FADV_RANDOM:
		case POSIX_FADV_SEQUENTIAL:
			break;
		default:
			return -EAGAIN;
		}
	}
J
Jens Axboe 已提交
2737 2738 2739 2740 2741 2742 2743 2744 2745

	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2746 2747
static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
2748
	const char __user *fname;
2749 2750 2751 2752 2753 2754 2755 2756
	unsigned lookup_flags;
	int ret;

	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;

	req->open.dfd = READ_ONCE(sqe->fd);
	req->open.mask = READ_ONCE(sqe->len);
2757
	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2758
	req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
2759
	req->open.how.flags = READ_ONCE(sqe->statx_flags);
2760

2761
	if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags))
2762 2763
		return -EINVAL;

2764
	req->open.filename = getname_flags(fname, lookup_flags, NULL);
2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
	if (IS_ERR(req->open.filename)) {
		ret = PTR_ERR(req->open.filename);
		req->open.filename = NULL;
		return ret;
	}

	return 0;
}

static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	struct io_open *ctx = &req->open;
	unsigned lookup_flags;
	struct path path;
	struct kstat stat;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

2786
	if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
		return -EINVAL;

retry:
	/* filename_lookup() drops it, keep a reference */
	ctx->filename->refcnt++;

	ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
				NULL);
	if (ret)
		goto err;

2798
	ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
	path_put(&path);
	if (retry_estale(ret, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
	if (!ret)
		ret = cp_statx(&stat, ctx->buffer);
err:
	putname(ctx->filename);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}

2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830
static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	/*
	 * If we queue this for async, it must not be cancellable. That would
	 * leave the 'file' in an undeterminate state.
	 */
	req->work.flags |= IO_WQ_WORK_NO_CANCEL;

	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
	    sqe->rw_flags || sqe->buf_index)
		return -EINVAL;
	if (sqe->flags & IOSQE_FIXED_FILE)
		return -EINVAL;

	req->close.fd = READ_ONCE(sqe->fd);
	if (req->file->f_op == &io_uring_fops ||
P
Pavel Begunkov 已提交
2831
	    req->close.fd == req->ctx->ring_fd)
2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
		return -EBADF;

	return 0;
}

static void io_close_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;

	/* Invoked with files, we need to do the close */
	if (req->work.files) {
		int ret;

		ret = filp_close(req->close.put_file, req->work.files);
2847
		if (ret < 0)
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869
			req_set_fail_links(req);
		io_cqring_add_event(req, ret);
	}

	fput(req->close.put_file);

	io_put_req_find_next(req, &nxt);
	if (nxt)
		io_wq_assign_next(workptr, nxt);
}

static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	int ret;

	req->close.put_file = NULL;
	ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
	if (ret < 0)
		return ret;

	/* if the file has a flush method, be safe and punt to async */
2870
	if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
		goto eagain;

	/*
	 * No ->flush(), safely close from here and just punt the
	 * fput() to async context.
	 */
	ret = filp_close(req->close.put_file, current->files);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);

	if (io_wq_current_is_worker()) {
		struct io_wq_work *old_work, *work;

		old_work = work = &req->work;
		io_close_finish(&work);
		if (work && work != old_work)
			*nxt = container_of(work, struct io_kiocb, work);
		return 0;
	}

eagain:
	req->work.func = io_close_finish;
2895 2896 2897 2898 2899 2900 2901
	/*
	 * Do manual async queue here to avoid grabbing files - we don't
	 * need the files, and it'll cause io_close_finish() to close
	 * the file again and cause a double CQE entry for this request
	 */
	io_queue_async_work(req);
	return 0;
2902 2903
}

2904
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915
{
	struct io_ring_ctx *ctx = req->ctx;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930
	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->len);
	req->sync.flags = READ_ONCE(sqe->sync_range_flags);
	return 0;
}

static void io_sync_file_range_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;
	int ret;

	if (io_req_cancelled(req))
		return;

2931
	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
2932 2933 2934 2935 2936 2937
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
2938
		io_wq_assign_next(workptr, nxt);
2939 2940
}

2941
static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
2942 2943
			      bool force_nonblock)
{
2944
	struct io_wq_work *work, *old_work;
2945 2946

	/* sync_file_range always requires a blocking context */
2947 2948 2949
	if (force_nonblock) {
		io_put_req(req);
		req->work.func = io_sync_file_range_finish;
2950
		return -EAGAIN;
2951
	}
2952

2953 2954 2955 2956
	work = old_work = &req->work;
	io_sync_file_range_finish(&work);
	if (work && work != old_work)
		*nxt = container_of(work, struct io_kiocb, work);
2957 2958 2959
	return 0;
}

2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
#if defined(CONFIG_NET)
static void io_sendrecv_async(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct iovec *iov = NULL;

	if (req->io->rw.iov != req->io->rw.fast_iov)
		iov = req->io->msg.iov;
	io_wq_submit_work(workptr);
	kfree(iov);
}
#endif

2973
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2974
{
J
Jens Axboe 已提交
2975
#if defined(CONFIG_NET)
2976
	struct io_sr_msg *sr = &req->sr_msg;
2977
	struct io_async_ctx *io = req->io;
2978

2979 2980
	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
2981
	sr->len = READ_ONCE(sqe->len);
2982

2983
	if (!io || req->opcode == IORING_OP_SEND)
2984 2985
		return 0;

2986
	io->msg.iov = io->msg.fast_iov;
2987
	return sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
2988
					&io->msg.iov);
2989
#else
2990
	return -EOPNOTSUPP;
2991 2992 2993
#endif
}

2994 2995
static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
J
Jens Axboe 已提交
2996
{
2997
#if defined(CONFIG_NET)
2998
	struct io_async_msghdr *kmsg = NULL;
J
Jens Axboe 已提交
2999 3000 3001 3002 3003 3004 3005 3006
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
3007
		struct io_async_ctx io;
3008
		struct sockaddr_storage addr;
J
Jens Axboe 已提交
3009 3010
		unsigned flags;

3011
		if (req->io) {
3012 3013 3014 3015 3016 3017
			kmsg = &req->io->msg;
			kmsg->msg.msg_name = &addr;
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
3018
		} else {
3019 3020
			struct io_sr_msg *sr = &req->sr_msg;

3021 3022
			kmsg = &io.msg;
			kmsg->msg.msg_name = &addr;
3023 3024 3025 3026

			io.msg.iov = io.msg.fast_iov;
			ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.iov);
3027
			if (ret)
3028
				return ret;
3029
		}
J
Jens Axboe 已提交
3030

3031 3032 3033 3034 3035 3036
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3037
		ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3038
		if (force_nonblock && ret == -EAGAIN) {
3039 3040 3041 3042 3043 3044
			if (req->io)
				return -EAGAIN;
			if (io_alloc_async_ctx(req))
				return -ENOMEM;
			memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
			req->work.func = io_sendrecv_async;
3045
			return -EAGAIN;
3046
		}
3047 3048
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
J
Jens Axboe 已提交
3049 3050
	}

3051
	if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
3052
		kfree(kmsg->iov);
3053
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3054 3055
	if (ret < 0)
		req_set_fail_links(req);
3056
	io_put_req_find_next(req, nxt);
J
Jens Axboe 已提交
3057
	return 0;
3058 3059
#else
	return -EOPNOTSUPP;
J
Jens Axboe 已提交
3060
#endif
3061
}
J
Jens Axboe 已提交
3062

3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095
static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

		ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
						&msg.msg_iter);
		if (ret)
			return ret;

		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3096 3097
		msg.msg_flags = flags;
		ret = sock_sendmsg(sock, &msg);
3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3114 3115
static int io_recvmsg_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
3116 3117
{
#if defined(CONFIG_NET)
3118
	struct io_sr_msg *sr = &req->sr_msg;
3119 3120 3121 3122
	struct io_async_ctx *io = req->io;

	sr->msg_flags = READ_ONCE(sqe->msg_flags);
	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3123
	sr->len = READ_ONCE(sqe->len);
3124

3125
	if (!io || req->opcode == IORING_OP_RECV)
3126
		return 0;
3127

3128
	io->msg.iov = io->msg.fast_iov;
3129
	return recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3130
					&io->msg.uaddr, &io->msg.iov);
J
Jens Axboe 已提交
3131
#else
3132
	return -EOPNOTSUPP;
J
Jens Axboe 已提交
3133 3134 3135
#endif
}

3136 3137
static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
J
Jens Axboe 已提交
3138 3139
{
#if defined(CONFIG_NET)
3140
	struct io_async_msghdr *kmsg = NULL;
3141 3142 3143 3144 3145 3146 3147 3148
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
3149
		struct io_async_ctx io;
3150 3151 3152 3153
		struct sockaddr_storage addr;
		unsigned flags;

		if (req->io) {
3154 3155 3156 3157 3158 3159
			kmsg = &req->io->msg;
			kmsg->msg.msg_name = &addr;
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
3160
		} else {
3161 3162
			struct io_sr_msg *sr = &req->sr_msg;

3163 3164
			kmsg = &io.msg;
			kmsg->msg.msg_name = &addr;
3165 3166 3167 3168 3169

			io.msg.iov = io.msg.fast_iov;
			ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.uaddr,
					&io.msg.iov);
3170
			if (ret)
3171
				return ret;
3172 3173
		}

3174 3175 3176 3177 3178 3179 3180 3181
		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

		ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
						kmsg->uaddr, flags);
3182
		if (force_nonblock && ret == -EAGAIN) {
3183 3184 3185 3186 3187 3188
			if (req->io)
				return -EAGAIN;
			if (io_alloc_async_ctx(req))
				return -ENOMEM;
			memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
			req->work.func = io_sendrecv_async;
3189
			return -EAGAIN;
3190 3191 3192 3193 3194
		}
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

3195
	if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
3196
		kfree(kmsg->iov);
3197
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3198 3199
	if (ret < 0)
		req_set_fail_links(req);
3200 3201
	io_put_req_find_next(req, nxt);
	return 0;
J
Jens Axboe 已提交
3202 3203 3204 3205 3206
#else
	return -EOPNOTSUPP;
#endif
}

3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241
static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_sr_msg *sr = &req->sr_msg;
		struct msghdr msg;
		struct iovec iov;
		unsigned flags;

		ret = import_single_range(READ, sr->buf, sr->len, &iov,
						&msg.msg_iter);
		if (ret)
			return ret;

		msg.msg_name = NULL;
		msg.msg_control = NULL;
		msg.msg_controllen = 0;
		msg.msg_namelen = 0;
		msg.msg_iocb = NULL;
		msg.msg_flags = 0;

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

3242
		ret = sock_recvmsg(sock, &msg, flags);
3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259
		if (force_nonblock && ret == -EAGAIN)
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


3260
static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3261 3262
{
#if defined(CONFIG_NET)
3263 3264
	struct io_accept *accept = &req->accept;

3265 3266
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
3267
	if (sqe->ioprio || sqe->len || sqe->buf_index)
3268 3269
		return -EINVAL;

3270 3271
	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3272 3273 3274 3275 3276 3277
	accept->flags = READ_ONCE(sqe->accept_flags);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}
3278

3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
#if defined(CONFIG_NET)
static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
		       bool force_nonblock)
{
	struct io_accept *accept = &req->accept;
	unsigned file_flags;
	int ret;

	file_flags = force_nonblock ? O_NONBLOCK : 0;
	ret = __sys_accept4_file(req->file, file_flags, accept->addr,
					accept->addr_len, accept->flags);
	if (ret == -EAGAIN && force_nonblock)
3291
		return -EAGAIN;
3292 3293
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
J
Jens Axboe 已提交
3294 3295
	if (ret < 0)
		req_set_fail_links(req);
3296
	io_cqring_add_event(req, ret);
3297
	io_put_req_find_next(req, nxt);
3298
	return 0;
3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
}

static void io_accept_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;

	if (io_req_cancelled(req))
		return;
	__io_accept(req, &nxt, false);
	if (nxt)
3310
		io_wq_assign_next(workptr, nxt);
3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
}
#endif

static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
#if defined(CONFIG_NET)
	int ret;

	ret = __io_accept(req, nxt, force_nonblock);
	if (ret == -EAGAIN && force_nonblock) {
		req->work.func = io_accept_finish;
		io_put_req(req);
		return -EAGAIN;
	}
	return 0;
3327 3328 3329 3330 3331
#else
	return -EOPNOTSUPP;
#endif
}

3332
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3333 3334
{
#if defined(CONFIG_NET)
3335 3336
	struct io_connect *conn = &req->connect;
	struct io_async_ctx *io = req->io;
3337

3338 3339 3340 3341 3342
	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

3343 3344 3345 3346 3347 3348 3349
	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	conn->addr_len =  READ_ONCE(sqe->addr2);

	if (!io)
		return 0;

	return move_addr_to_kernel(conn->addr, conn->addr_len,
3350
					&io->connect.address);
3351
#else
3352
	return -EOPNOTSUPP;
3353 3354 3355
#endif
}

3356 3357
static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
3358 3359
{
#if defined(CONFIG_NET)
3360
	struct io_async_ctx __io, *io;
3361
	unsigned file_flags;
3362
	int ret;
3363

3364 3365 3366
	if (req->io) {
		io = req->io;
	} else {
3367 3368 3369
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
3370 3371 3372 3373 3374
		if (ret)
			goto out;
		io = &__io;
	}

3375 3376 3377 3378
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
3379
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
3380 3381 3382
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
3383 3384 3385
			ret = -ENOMEM;
			goto out;
		}
3386
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
3387
		return -EAGAIN;
3388
	}
3389 3390
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
3391
out:
J
Jens Axboe 已提交
3392 3393
	if (ret < 0)
		req_set_fail_links(req);
3394 3395 3396 3397 3398 3399 3400 3401
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

3402 3403 3404 3405 3406 3407
static void io_poll_remove_one(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = &req->poll;

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
3408 3409
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
3410
		io_queue_async_work(req);
3411 3412
	}
	spin_unlock(&poll->head->lock);
3413
	hash_del(&req->hash_node);
3414 3415 3416 3417
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
3418
	struct hlist_node *tmp;
3419
	struct io_kiocb *req;
3420
	int i;
3421 3422

	spin_lock_irq(&ctx->completion_lock);
3423 3424 3425 3426 3427 3428
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node)
			io_poll_remove_one(req);
3429 3430 3431 3432
	}
	spin_unlock_irq(&ctx->completion_lock);
}

3433 3434
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
3435
	struct hlist_head *list;
3436 3437
	struct io_kiocb *req;

3438 3439 3440
	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	hlist_for_each_entry(req, list, hash_node) {
		if (sqe_addr == req->user_data) {
3441 3442 3443
			io_poll_remove_one(req);
			return 0;
		}
3444 3445 3446 3447 3448
	}

	return -ENOENT;
}

3449 3450
static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	req->poll.addr = READ_ONCE(sqe->addr);
	return 0;
}

3462 3463 3464 3465
/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
3466
static int io_poll_remove(struct io_kiocb *req)
3467 3468
{
	struct io_ring_ctx *ctx = req->ctx;
3469
	u64 addr;
3470
	int ret;
3471

3472
	addr = req->poll.addr;
3473
	spin_lock_irq(&ctx->completion_lock);
3474
	ret = io_poll_cancel(ctx, addr);
3475 3476
	spin_unlock_irq(&ctx->completion_lock);

3477
	io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
3478 3479
	if (ret < 0)
		req_set_fail_links(req);
3480
	io_put_req(req);
3481 3482 3483
	return 0;
}

3484
static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
3485
{
3486 3487
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
3488
	req->poll.done = true;
3489 3490 3491 3492
	if (error)
		io_cqring_fill_event(req, error);
	else
		io_cqring_fill_event(req, mangle_poll(mask));
J
Jens Axboe 已提交
3493
	io_commit_cqring(ctx);
3494 3495
}

3496
static void io_poll_complete_work(struct io_wq_work **workptr)
3497
{
3498
	struct io_wq_work *work = *workptr;
3499 3500 3501 3502
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_poll_iocb *poll = &req->poll;
	struct poll_table_struct pt = { ._key = poll->events };
	struct io_ring_ctx *ctx = req->ctx;
3503
	struct io_kiocb *nxt = NULL;
3504
	__poll_t mask = 0;
3505
	int ret = 0;
3506

3507
	if (work->flags & IO_WQ_WORK_CANCEL) {
3508
		WRITE_ONCE(poll->canceled, true);
3509 3510 3511 3512
		ret = -ECANCELED;
	} else if (READ_ONCE(poll->canceled)) {
		ret = -ECANCELED;
	}
3513

3514
	if (ret != -ECANCELED)
3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
		mask = vfs_poll(poll->file, &pt) & poll->events;

	/*
	 * Note that ->ki_cancel callers also delete iocb from active_reqs after
	 * calling ->ki_cancel.  We need the ctx_lock roundtrip here to
	 * synchronize with them.  In the cancellation case the list_del_init
	 * itself is not actually needed, but harmless so we keep it in to
	 * avoid further branches in the fast path.
	 */
	spin_lock_irq(&ctx->completion_lock);
3525
	if (!mask && ret != -ECANCELED) {
3526
		add_wait_queue(poll->head, &poll->wait);
3527 3528 3529
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}
3530
	hash_del(&req->hash_node);
3531
	io_poll_complete(req, mask, ret);
3532 3533
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
3534
	io_cqring_ev_posted(ctx);
3535

J
Jens Axboe 已提交
3536 3537
	if (ret < 0)
		req_set_fail_links(req);
3538
	io_put_req_find_next(req, &nxt);
3539
	if (nxt)
3540
		io_wq_assign_next(workptr, nxt);
3541 3542
}

3543 3544 3545
static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
{
	struct io_kiocb *req, *tmp;
3546
	struct req_batch rb;
3547

3548
	rb.to_free = rb.need_iter = 0;
3549 3550 3551 3552 3553
	spin_lock_irq(&ctx->completion_lock);
	llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
		hash_del(&req->hash_node);
		io_poll_complete(req, req->result, 0);

3554 3555 3556 3557
		if (refcount_dec_and_test(&req->refs) &&
		    !io_req_multi_free(&rb, req)) {
			req->flags |= REQ_F_COMP_LOCKED;
			io_free_req(req);
3558 3559 3560 3561 3562
		}
	}
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
3563
	io_free_req_many(ctx, &rb);
3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
}

static void io_poll_flush(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct llist_node *nodes;

	nodes = llist_del_all(&req->ctx->poll_llist);
	if (nodes)
		__io_poll_flush(req->ctx, nodes);
}

3576 3577 3578 3579 3580 3581 3582 3583
static void io_poll_trigger_evfd(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);

	eventfd_signal(req->ctx->cq_ev_fd, 1);
	io_put_req(req);
}

3584 3585 3586
static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
3587
	struct io_poll_iocb *poll = wait->private;
3588 3589 3590 3591 3592
	struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
	struct io_ring_ctx *ctx = req->ctx;
	__poll_t mask = key_to_poll(key);

	/* for instances that support it check for an event match first: */
J
Jens Axboe 已提交
3593 3594
	if (mask && !(mask & poll->events))
		return 0;
3595

3596
	list_del_init(&poll->wait.entry);
3597

3598 3599 3600 3601 3602 3603
	/*
	 * Run completion inline if we can. We're using trylock here because
	 * we are violating the completion_lock -> poll wq lock ordering.
	 * If we have a link timeout we're going to need the completion_lock
	 * for finalizing the request, mark us as having grabbed that already.
	 */
3604 3605
	if (mask) {
		unsigned long flags;
3606

3607 3608
		if (llist_empty(&ctx->poll_llist) &&
		    spin_trylock_irqsave(&ctx->completion_lock, flags)) {
3609 3610
			bool trigger_ev;

3611 3612 3613
			hash_del(&req->hash_node);
			io_poll_complete(req, mask, 0);

3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624
			trigger_ev = io_should_trigger_evfd(ctx);
			if (trigger_ev && eventfd_signal_count()) {
				trigger_ev = false;
				req->work.func = io_poll_trigger_evfd;
			} else {
				req->flags |= REQ_F_COMP_LOCKED;
				io_put_req(req);
				req = NULL;
			}
			spin_unlock_irqrestore(&ctx->completion_lock, flags);
			__io_cqring_ev_posted(ctx, trigger_ev);
3625 3626 3627 3628 3629 3630 3631 3632 3633
		} else {
			req->result = mask;
			req->llist_node.next = NULL;
			/* if the list wasn't empty, we're done */
			if (!llist_add(&req->llist_node, &ctx->poll_llist))
				req = NULL;
			else
				req->work.func = io_poll_flush;
		}
3634
	}
3635 3636
	if (req)
		io_queue_async_work(req);
3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658

	return 1;
}

struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

	if (unlikely(pt->req->poll.head)) {
		pt->error = -EINVAL;
		return;
	}

	pt->error = 0;
	pt->req->poll.head = head;
3659
	add_wait_queue(head, &pt->req->poll.wait);
3660 3661
}

3662 3663 3664
static void io_poll_req_insert(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
3665 3666 3667 3668
	struct hlist_head *list;

	list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	hlist_add_head(&req->hash_node, list);
3669 3670
}

3671
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3672 3673 3674 3675 3676 3677 3678 3679
{
	struct io_poll_iocb *poll = &req->poll;
	u16 events;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
3680 3681
	if (!poll->file)
		return -EBADF;
3682 3683 3684

	events = READ_ONCE(sqe->poll_events);
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696
	return 0;
}

static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	bool cancel = false;
	__poll_t mask;

	INIT_IO_WORK(&req->work, io_poll_complete_work);
3697
	INIT_HLIST_NODE(&req->hash_node);
3698 3699

	poll->head = NULL;
J
Jens Axboe 已提交
3700
	poll->done = false;
3701 3702 3703 3704 3705 3706 3707 3708
	poll->canceled = false;

	ipt.pt._qproc = io_poll_queue_proc;
	ipt.pt._key = poll->events;
	ipt.req = req;
	ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */

	/* initialized the list so that we can do list_empty checks */
3709 3710 3711
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, io_poll_wake);
	poll->wait.private = poll;
3712

3713 3714
	INIT_LIST_HEAD(&req->list);

3715 3716 3717
	mask = vfs_poll(poll->file, &ipt.pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
3718 3719
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
3720
		if (unlikely(list_empty(&poll->wait.entry))) {
J
Jens Axboe 已提交
3721 3722 3723 3724 3725 3726
			if (ipt.error)
				cancel = true;
			ipt.error = 0;
			mask = 0;
		}
		if (mask || ipt.error)
3727
			list_del_init(&poll->wait.entry);
J
Jens Axboe 已提交
3728 3729 3730
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
3731
			io_poll_req_insert(req);
J
Jens Axboe 已提交
3732 3733 3734
		spin_unlock(&poll->head->lock);
	}
	if (mask) { /* no async, we'd stolen it */
3735
		ipt.error = 0;
3736
		io_poll_complete(req, mask, 0);
3737 3738 3739
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
3740 3741
	if (mask) {
		io_cqring_ev_posted(ctx);
3742
		io_put_req_find_next(req, nxt);
3743
	}
J
Jens Axboe 已提交
3744
	return ipt.error;
3745 3746
}

J
Jens Axboe 已提交
3747 3748
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
3749 3750 3751 3752
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
3753 3754 3755 3756 3757
	unsigned long flags;

	atomic_inc(&ctx->cq_timeouts);

	spin_lock_irqsave(&ctx->completion_lock, flags);
3758
	/*
3759 3760
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
3761
	 */
3762
	if (!list_empty(&req->list)) {
3763
		struct io_kiocb *prev;
J
Jens Axboe 已提交
3764

3765 3766
		/*
		 * Adjust the reqs sequence before the current one because it
3767
		 * will consume a slot in the cq_ring and the cq_tail
3768 3769 3770 3771 3772 3773 3774 3775
		 * pointer will be increased, otherwise other timeout reqs may
		 * return in advance without waiting for enough wait_nr.
		 */
		prev = req;
		list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
			prev->sequence++;
		list_del_init(&req->list);
	}
3776

3777
	io_cqring_fill_event(req, -ETIME);
3778
	io_commit_cqring(ctx);
J
Jens Axboe 已提交
3779 3780
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

3781
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
3782
	req_set_fail_links(req);
3783
	io_put_req(req);
3784 3785 3786
	return HRTIMER_NORESTART;
}

3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802
static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
{
	struct io_kiocb *req;
	int ret = -ENOENT;

	list_for_each_entry(req, &ctx->timeout_list, list) {
		if (user_data == req->user_data) {
			list_del_init(&req->list);
			ret = 0;
			break;
		}
	}

	if (ret == -ENOENT)
		return ret;

3803
	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
3804 3805 3806
	if (ret == -1)
		return -EALREADY;

J
Jens Axboe 已提交
3807
	req_set_fail_links(req);
3808 3809 3810 3811 3812
	io_cqring_fill_event(req, -ECANCELED);
	io_put_req(req);
	return 0;
}

3813 3814
static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
		return -EINVAL;

	req->timeout.addr = READ_ONCE(sqe->addr);
	req->timeout.flags = READ_ONCE(sqe->timeout_flags);
	if (req->timeout.flags)
		return -EINVAL;

	return 0;
}

3829 3830 3831
/*
 * Remove or update an existing timeout command
 */
3832
static int io_timeout_remove(struct io_kiocb *req)
3833 3834
{
	struct io_ring_ctx *ctx = req->ctx;
3835
	int ret;
3836 3837

	spin_lock_irq(&ctx->completion_lock);
3838
	ret = io_timeout_cancel(ctx, req->timeout.addr);
3839

3840
	io_cqring_fill_event(req, ret);
3841 3842
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
3843
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
3844 3845
	if (ret < 0)
		req_set_fail_links(req);
3846
	io_put_req(req);
3847
	return 0;
J
Jens Axboe 已提交
3848 3849
}

3850
static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
3851
			   bool is_timeout_link)
J
Jens Axboe 已提交
3852
{
3853
	struct io_timeout_data *data;
3854
	unsigned flags;
J
Jens Axboe 已提交
3855

3856
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
3857
		return -EINVAL;
3858
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
3859
		return -EINVAL;
3860 3861
	if (sqe->off && is_timeout_link)
		return -EINVAL;
3862 3863
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
3864
		return -EINVAL;
3865

3866 3867
	req->timeout.count = READ_ONCE(sqe->off);

3868
	if (!req->io && io_alloc_async_ctx(req))
3869 3870 3871
		return -ENOMEM;

	data = &req->io->timeout;
3872 3873 3874 3875
	data->req = req;
	req->flags |= REQ_F_TIMEOUT;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
3876 3877
		return -EFAULT;

3878
	if (flags & IORING_TIMEOUT_ABS)
3879
		data->mode = HRTIMER_MODE_ABS;
3880
	else
3881
		data->mode = HRTIMER_MODE_REL;
3882

3883 3884 3885
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}
3886

3887
static int io_timeout(struct io_kiocb *req)
3888 3889 3890 3891 3892 3893 3894
{
	unsigned count;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_timeout_data *data;
	struct list_head *entry;
	unsigned span = 0;

3895
	data = &req->io->timeout;
3896

J
Jens Axboe 已提交
3897 3898
	/*
	 * sqe->off holds how many events that need to occur for this
3899 3900
	 * timeout event to be satisfied. If it isn't set, then this is
	 * a pure timeout request, sequence isn't used.
J
Jens Axboe 已提交
3901
	 */
3902
	count = req->timeout.count;
3903 3904 3905 3906 3907 3908
	if (!count) {
		req->flags |= REQ_F_TIMEOUT_NOSEQ;
		spin_lock_irq(&ctx->completion_lock);
		entry = ctx->timeout_list.prev;
		goto add;
	}
J
Jens Axboe 已提交
3909 3910

	req->sequence = ctx->cached_sq_head + count - 1;
3911
	data->seq_offset = count;
J
Jens Axboe 已提交
3912 3913 3914 3915 3916 3917 3918 3919

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	spin_lock_irq(&ctx->completion_lock);
	list_for_each_prev(entry, &ctx->timeout_list) {
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
3920 3921
		unsigned nxt_sq_head;
		long long tmp, tmp_nxt;
3922
		u32 nxt_offset = nxt->io->timeout.seq_offset;
J
Jens Axboe 已提交
3923

3924 3925 3926
		if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
			continue;

3927 3928 3929 3930 3931
		/*
		 * Since cached_sq_head + count - 1 can overflow, use type long
		 * long to store it.
		 */
		tmp = (long long)ctx->cached_sq_head + count - 1;
3932 3933
		nxt_sq_head = nxt->sequence - nxt_offset + 1;
		tmp_nxt = (long long)nxt_sq_head + nxt_offset - 1;
3934 3935 3936 3937 3938 3939

		/*
		 * cached_sq_head may overflow, and it will never overflow twice
		 * once there is some timeout req still be valid.
		 */
		if (ctx->cached_sq_head < nxt_sq_head)
3940
			tmp += UINT_MAX;
3941

3942
		if (tmp > tmp_nxt)
J
Jens Axboe 已提交
3943
			break;
3944 3945 3946 3947 3948 3949 3950

		/*
		 * Sequence of reqs after the insert one and itself should
		 * be adjusted because each timeout req consumes a slot.
		 */
		span++;
		nxt->sequence++;
J
Jens Axboe 已提交
3951
	}
3952
	req->sequence -= span;
3953
add:
J
Jens Axboe 已提交
3954
	list_add(&req->list, entry);
3955 3956
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
3957
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
3958 3959 3960
	return 0;
}

3961 3962 3963 3964 3965 3966 3967
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

3968
static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
{
	enum io_wq_cancel cancel_ret;
	int ret = 0;

	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

3986 3987 3988
	return ret;
}

3989 3990
static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
3991
				     struct io_kiocb **nxt, int success_ret)
3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
	if (ret != -ENOENT) {
		spin_lock_irqsave(&ctx->completion_lock, flags);
		goto done;
	}

	spin_lock_irqsave(&ctx->completion_lock, flags);
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr);
done:
4008 4009
	if (!ret)
		ret = success_ret;
4010 4011 4012 4013 4014
	io_cqring_fill_event(req, ret);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

J
Jens Axboe 已提交
4015 4016
	if (ret < 0)
		req_set_fail_links(req);
4017 4018 4019
	io_put_req_find_next(req, nxt);
}

4020 4021
static int io_async_cancel_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
4022
{
4023
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4024 4025 4026 4027 4028
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
	    sqe->cancel_flags)
		return -EINVAL;

4029 4030 4031 4032 4033 4034 4035 4036 4037
	req->cancel.addr = READ_ONCE(sqe->addr);
	return 0;
}

static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
4038 4039 4040
	return 0;
}

4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060
static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (sqe->flags || sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	req->files_update.offset = READ_ONCE(sqe->off);
	req->files_update.nr_args = READ_ONCE(sqe->len);
	if (!req->files_update.nr_args)
		return -EINVAL;
	req->files_update.arg = READ_ONCE(sqe->addr);
	return 0;
}

static int io_files_update(struct io_kiocb *req, bool force_nonblock)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_uring_files_update up;
	int ret;

4061
	if (force_nonblock)
4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req(req);
	return 0;
}

4078 4079
static int io_req_defer_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
4080
{
4081
	ssize_t ret = 0;
4082

4083 4084 4085 4086 4087 4088
	if (io_op_defs[req->opcode].file_table) {
		ret = io_grab_files(req);
		if (unlikely(ret))
			return ret;
	}

4089 4090
	io_req_work_grab_env(req, &io_op_defs[req->opcode]);

4091
	switch (req->opcode) {
4092 4093
	case IORING_OP_NOP:
		break;
4094 4095
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
4096
	case IORING_OP_READ:
4097
		ret = io_read_prep(req, sqe, true);
4098 4099 4100
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
4101
	case IORING_OP_WRITE:
4102
		ret = io_write_prep(req, sqe, true);
4103
		break;
4104
	case IORING_OP_POLL_ADD:
4105
		ret = io_poll_add_prep(req, sqe);
4106 4107
		break;
	case IORING_OP_POLL_REMOVE:
4108
		ret = io_poll_remove_prep(req, sqe);
4109
		break;
4110
	case IORING_OP_FSYNC:
4111
		ret = io_prep_fsync(req, sqe);
4112 4113
		break;
	case IORING_OP_SYNC_FILE_RANGE:
4114
		ret = io_prep_sfr(req, sqe);
4115
		break;
4116
	case IORING_OP_SENDMSG:
4117
	case IORING_OP_SEND:
4118
		ret = io_sendmsg_prep(req, sqe);
4119 4120
		break;
	case IORING_OP_RECVMSG:
4121
	case IORING_OP_RECV:
4122
		ret = io_recvmsg_prep(req, sqe);
4123
		break;
4124
	case IORING_OP_CONNECT:
4125
		ret = io_connect_prep(req, sqe);
4126
		break;
4127
	case IORING_OP_TIMEOUT:
4128
		ret = io_timeout_prep(req, sqe, false);
4129
		break;
4130
	case IORING_OP_TIMEOUT_REMOVE:
4131
		ret = io_timeout_remove_prep(req, sqe);
4132
		break;
4133
	case IORING_OP_ASYNC_CANCEL:
4134
		ret = io_async_cancel_prep(req, sqe);
4135
		break;
4136
	case IORING_OP_LINK_TIMEOUT:
4137
		ret = io_timeout_prep(req, sqe, true);
4138
		break;
4139
	case IORING_OP_ACCEPT:
4140
		ret = io_accept_prep(req, sqe);
4141
		break;
4142 4143 4144
	case IORING_OP_FALLOCATE:
		ret = io_fallocate_prep(req, sqe);
		break;
4145 4146 4147
	case IORING_OP_OPENAT:
		ret = io_openat_prep(req, sqe);
		break;
4148 4149 4150
	case IORING_OP_CLOSE:
		ret = io_close_prep(req, sqe);
		break;
4151 4152 4153
	case IORING_OP_FILES_UPDATE:
		ret = io_files_update_prep(req, sqe);
		break;
4154 4155 4156
	case IORING_OP_STATX:
		ret = io_statx_prep(req, sqe);
		break;
J
Jens Axboe 已提交
4157 4158 4159
	case IORING_OP_FADVISE:
		ret = io_fadvise_prep(req, sqe);
		break;
J
Jens Axboe 已提交
4160 4161 4162
	case IORING_OP_MADVISE:
		ret = io_madvise_prep(req, sqe);
		break;
4163 4164 4165
	case IORING_OP_OPENAT2:
		ret = io_openat2_prep(req, sqe);
		break;
4166 4167 4168
	case IORING_OP_EPOLL_CTL:
		ret = io_epoll_ctl_prep(req, sqe);
		break;
4169
	default:
4170 4171 4172
		printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
				req->opcode);
		ret = -EINVAL;
4173
		break;
4174 4175
	}

4176
	return ret;
4177 4178
}

4179
static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4180
{
4181
	struct io_ring_ctx *ctx = req->ctx;
4182
	int ret;
4183

B
Bob Liu 已提交
4184 4185
	/* Still need defer if there is pending req in defer list. */
	if (!req_need_defer(req) && list_empty(&ctx->defer_list))
4186 4187
		return 0;

4188
	if (!req->io && io_alloc_async_ctx(req))
4189 4190
		return -EAGAIN;

4191
	ret = io_req_defer_prep(req, sqe);
4192
	if (ret < 0)
4193 4194
		return ret;

4195
	spin_lock_irq(&ctx->completion_lock);
B
Bob Liu 已提交
4196
	if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
4197 4198 4199 4200
		spin_unlock_irq(&ctx->completion_lock);
		return 0;
	}

4201
	trace_io_uring_defer(ctx, req, req->user_data);
4202 4203 4204 4205 4206
	list_add_tail(&req->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

4207 4208
static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
4209
{
4210
	struct io_ring_ctx *ctx = req->ctx;
4211
	int ret;
J
Jens Axboe 已提交
4212

4213
	switch (req->opcode) {
J
Jens Axboe 已提交
4214
	case IORING_OP_NOP:
4215
		ret = io_nop(req);
J
Jens Axboe 已提交
4216 4217
		break;
	case IORING_OP_READV:
4218
	case IORING_OP_READ_FIXED:
4219
	case IORING_OP_READ:
4220 4221 4222 4223 4224
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
4225
		ret = io_read(req, nxt, force_nonblock);
4226
		break;
4227
	case IORING_OP_WRITEV:
4228
	case IORING_OP_WRITE_FIXED:
4229
	case IORING_OP_WRITE:
4230 4231 4232 4233 4234
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
4235
		ret = io_write(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4236
		break;
C
Christoph Hellwig 已提交
4237
	case IORING_OP_FSYNC:
4238 4239 4240 4241 4242
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
4243
		ret = io_fsync(req, nxt, force_nonblock);
C
Christoph Hellwig 已提交
4244
		break;
4245
	case IORING_OP_POLL_ADD:
4246 4247 4248 4249 4250
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
4251
		ret = io_poll_add(req, nxt);
4252 4253
		break;
	case IORING_OP_POLL_REMOVE:
4254 4255 4256 4257 4258
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
4259
		ret = io_poll_remove(req);
4260
		break;
4261
	case IORING_OP_SYNC_FILE_RANGE:
4262 4263 4264 4265 4266
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
4267
		ret = io_sync_file_range(req, nxt, force_nonblock);
4268
		break;
J
Jens Axboe 已提交
4269
	case IORING_OP_SENDMSG:
4270
	case IORING_OP_SEND:
4271 4272 4273 4274 4275
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
4276 4277 4278 4279
		if (req->opcode == IORING_OP_SENDMSG)
			ret = io_sendmsg(req, nxt, force_nonblock);
		else
			ret = io_send(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4280
		break;
J
Jens Axboe 已提交
4281
	case IORING_OP_RECVMSG:
4282
	case IORING_OP_RECV:
4283 4284 4285 4286 4287
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
4288 4289 4290 4291
		if (req->opcode == IORING_OP_RECVMSG)
			ret = io_recvmsg(req, nxt, force_nonblock);
		else
			ret = io_recv(req, nxt, force_nonblock);
J
Jens Axboe 已提交
4292
		break;
J
Jens Axboe 已提交
4293
	case IORING_OP_TIMEOUT:
4294 4295 4296 4297 4298
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
4299
		ret = io_timeout(req);
J
Jens Axboe 已提交
4300
		break;
4301
	case IORING_OP_TIMEOUT_REMOVE:
4302 4303 4304 4305 4306
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
4307
		ret = io_timeout_remove(req);
4308
		break;
4309
	case IORING_OP_ACCEPT:
4310 4311 4312 4313 4314
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
4315
		ret = io_accept(req, nxt, force_nonblock);
4316
		break;
4317
	case IORING_OP_CONNECT:
4318 4319 4320 4321 4322
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
4323
		ret = io_connect(req, nxt, force_nonblock);
4324
		break;
4325
	case IORING_OP_ASYNC_CANCEL:
4326 4327 4328 4329 4330
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
4331
		ret = io_async_cancel(req, nxt);
4332
		break;
4333 4334 4335 4336 4337 4338 4339 4340
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fallocate(req, nxt, force_nonblock);
		break;
4341 4342 4343 4344 4345 4346 4347 4348
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat(req, nxt, force_nonblock);
		break;
4349 4350 4351 4352 4353 4354 4355 4356
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_close(req, nxt, force_nonblock);
		break;
4357 4358 4359 4360 4361 4362 4363 4364
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_files_update(req, force_nonblock);
		break;
4365 4366 4367 4368 4369 4370 4371 4372
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_statx(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4373 4374 4375 4376 4377 4378 4379 4380
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fadvise(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4381 4382 4383 4384 4385 4386 4387 4388
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_madvise(req, nxt, force_nonblock);
		break;
4389 4390 4391 4392 4393 4394 4395 4396
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat2(req, nxt, force_nonblock);
		break;
4397 4398 4399 4400 4401 4402 4403 4404
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_epoll_ctl(req, nxt, force_nonblock);
		break;
J
Jens Axboe 已提交
4405 4406 4407 4408 4409
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
4410 4411 4412 4413
	if (ret)
		return ret;

	if (ctx->flags & IORING_SETUP_IOPOLL) {
4414 4415
		const bool in_async = io_wq_current_is_worker();

J
Jens Axboe 已提交
4416
		if (req->result == -EAGAIN)
J
Jens Axboe 已提交
4417 4418
			return -EAGAIN;

4419 4420 4421 4422
		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

J
Jens Axboe 已提交
4423
		io_iopoll_req_issued(req);
4424 4425 4426

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
4427 4428 4429
	}

	return 0;
J
Jens Axboe 已提交
4430 4431
}

4432
static void io_wq_submit_work(struct io_wq_work **workptr)
J
Jens Axboe 已提交
4433
{
4434
	struct io_wq_work *work = *workptr;
J
Jens Axboe 已提交
4435
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
4436 4437
	struct io_kiocb *nxt = NULL;
	int ret = 0;
J
Jens Axboe 已提交
4438

4439 4440 4441
	/* if NO_CANCEL is set, we must still run the work */
	if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
				IO_WQ_WORK_CANCEL) {
4442
		ret = -ECANCELED;
4443
	}
4444

4445
	if (!ret) {
4446 4447
		req->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
		req->in_async = true;
4448
		do {
4449
			ret = io_issue_sqe(req, NULL, &nxt, false);
4450 4451 4452 4453 4454 4455 4456 4457 4458 4459
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
4460

4461
	/* drop submission reference */
4462
	io_put_req(req);
4463

4464
	if (ret) {
J
Jens Axboe 已提交
4465
		req_set_fail_links(req);
4466
		io_cqring_add_event(req, ret);
4467
		io_put_req(req);
J
Jens Axboe 已提交
4468
	}
4469

4470
	/* if a dependent link is ready, pass it back */
4471 4472
	if (!ret && nxt)
		io_wq_assign_next(workptr, nxt);
J
Jens Axboe 已提交
4473 4474
}

4475
static int io_req_needs_file(struct io_kiocb *req, int fd)
J
Jens Axboe 已提交
4476
{
4477
	if (!io_op_defs[req->opcode].needs_file)
4478
		return 0;
4479 4480 4481
	if (fd == -1 && io_op_defs[req->opcode].fd_non_neg)
		return 0;
	return 1;
J
Jens Axboe 已提交
4482 4483
}

4484 4485 4486 4487 4488
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

4489 4490
	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
	return table->files[index & IORING_FILE_TABLE_MASK];;
4491 4492
}

4493 4494
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4495
{
4496
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
4497
	unsigned flags;
4498
	int fd;
J
Jens Axboe 已提交
4499

4500 4501
	flags = READ_ONCE(sqe->flags);
	fd = READ_ONCE(sqe->fd);
J
Jens Axboe 已提交
4502

4503 4504
	if (!io_req_needs_file(req, fd))
		return 0;
J
Jens Axboe 已提交
4505 4506

	if (flags & IOSQE_FIXED_FILE) {
4507
		if (unlikely(!ctx->file_data ||
J
Jens Axboe 已提交
4508 4509
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
4510
		fd = array_index_nospec(fd, ctx->nr_user_files);
4511 4512
		req->file = io_file_from_index(ctx, fd);
		if (!req->file)
4513
			return -EBADF;
J
Jens Axboe 已提交
4514
		req->flags |= REQ_F_FIXED_FILE;
4515
		percpu_ref_get(&ctx->file_data->refs);
J
Jens Axboe 已提交
4516
	} else {
4517
		if (req->needs_fixed_file)
J
Jens Axboe 已提交
4518
			return -EBADF;
4519
		trace_io_uring_file_get(ctx, fd);
J
Jens Axboe 已提交
4520 4521 4522 4523 4524 4525 4526 4527
		req->file = io_file_get(state, fd);
		if (unlikely(!req->file))
			return -EBADF;
	}

	return 0;
}

4528
static int io_grab_files(struct io_kiocb *req)
4529 4530
{
	int ret = -EBADF;
4531
	struct io_ring_ctx *ctx = req->ctx;
4532

4533 4534
	if (req->work.files)
		return 0;
P
Pavel Begunkov 已提交
4535
	if (!ctx->ring_file)
4536 4537
		return -EBADF;

4538 4539 4540 4541 4542 4543 4544 4545
	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
P
Pavel Begunkov 已提交
4546
	if (fcheck(ctx->ring_fd) == ctx->ring_file) {
4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

4558 4559
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
{
4560 4561 4562
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
4563 4564 4565 4566 4567 4568 4569 4570 4571 4572
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
4573 4574 4575
	if (!list_empty(&req->link_list)) {
		prev = list_entry(req->link_list.prev, struct io_kiocb,
				  link_list);
4576
		if (refcount_inc_not_zero(&prev->refs)) {
4577
			list_del_init(&req->link_list);
4578 4579
			prev->flags &= ~REQ_F_LINK_TIMEOUT;
		} else
4580
			prev = NULL;
4581 4582 4583 4584 4585
	}

	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
J
Jens Axboe 已提交
4586
		req_set_fail_links(prev);
4587 4588
		io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
						-ETIME);
4589
		io_put_req(prev);
4590 4591 4592
	} else {
		io_cqring_add_event(req, -ETIME);
		io_put_req(req);
4593 4594 4595 4596
	}
	return HRTIMER_NORESTART;
}

4597
static void io_queue_linked_timeout(struct io_kiocb *req)
4598
{
4599
	struct io_ring_ctx *ctx = req->ctx;
4600

4601 4602 4603 4604 4605
	/*
	 * If the list is now empty, then our linked request finished before
	 * we got a chance to setup the timer
	 */
	spin_lock_irq(&ctx->completion_lock);
4606
	if (!list_empty(&req->link_list)) {
4607
		struct io_timeout_data *data = &req->io->timeout;
4608

4609 4610 4611
		data->timer.function = io_link_timeout_fn;
		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
				data->mode);
4612
	}
4613
	spin_unlock_irq(&ctx->completion_lock);
4614 4615

	/* drop submission reference */
4616 4617
	io_put_req(req);
}
4618

4619
static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
4620 4621 4622 4623 4624 4625
{
	struct io_kiocb *nxt;

	if (!(req->flags & REQ_F_LINK))
		return NULL;

4626 4627
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
					link_list);
4628
	if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
4629
		return NULL;
4630

4631 4632
	req->flags |= REQ_F_LINK_TIMEOUT;
	return nxt;
4633 4634
}

4635
static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
J
Jens Axboe 已提交
4636
{
4637
	struct io_kiocb *linked_timeout;
4638
	struct io_kiocb *nxt = NULL;
4639
	int ret;
J
Jens Axboe 已提交
4640

4641 4642 4643
again:
	linked_timeout = io_prep_linked_timeout(req);

4644
	ret = io_issue_sqe(req, sqe, &nxt, true);
4645 4646 4647 4648 4649 4650 4651

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
	    (req->flags & REQ_F_MUST_PUNT))) {
4652
punt:
4653
		if (io_op_defs[req->opcode].file_table) {
4654 4655 4656
			ret = io_grab_files(req);
			if (ret)
				goto err;
J
Jens Axboe 已提交
4657
		}
4658 4659 4660 4661 4662 4663

		/*
		 * Queued up for async execution, worker will release
		 * submit reference when the iocb is actually submitted.
		 */
		io_queue_async_work(req);
4664
		goto done_req;
J
Jens Axboe 已提交
4665
	}
4666

4667
err:
4668
	/* drop submission reference */
4669
	io_put_req(req);
4670

4671
	if (linked_timeout) {
4672
		if (!ret)
4673
			io_queue_linked_timeout(linked_timeout);
4674
		else
4675
			io_put_req(linked_timeout);
4676 4677
	}

4678
	/* and drop final reference, if we failed */
J
Jens Axboe 已提交
4679
	if (ret) {
4680
		io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
4681
		req_set_fail_links(req);
4682
		io_put_req(req);
J
Jens Axboe 已提交
4683
	}
4684 4685 4686 4687
done_req:
	if (nxt) {
		req = nxt;
		nxt = NULL;
4688 4689 4690

		if (req->flags & REQ_F_FORCE_ASYNC)
			goto punt;
4691 4692
		goto again;
	}
J
Jens Axboe 已提交
4693 4694
}

4695
static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4696 4697 4698
{
	int ret;

4699
	ret = io_req_defer(req, sqe);
4700 4701
	if (ret) {
		if (ret != -EIOCBQUEUED) {
4702
fail_req:
4703
			io_cqring_add_event(req, ret);
J
Jens Axboe 已提交
4704
			req_set_fail_links(req);
4705
			io_double_put_req(req);
4706
		}
4707
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
4708 4709 4710
		ret = io_req_defer_prep(req, sqe);
		if (unlikely(ret < 0))
			goto fail_req;
J
Jens Axboe 已提交
4711 4712 4713 4714 4715 4716 4717
		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
4718
		__io_queue_sqe(req, sqe);
J
Jens Axboe 已提交
4719
	}
4720 4721
}

4722
static inline void io_queue_link_head(struct io_kiocb *req)
4723
{
4724
	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
4725 4726 4727
		io_cqring_add_event(req, -ECANCELED);
		io_double_put_req(req);
	} else
4728
		io_queue_sqe(req, NULL);
4729 4730
}

J
Jens Axboe 已提交
4731
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK|	\
J
Jens Axboe 已提交
4732
				IOSQE_IO_HARDLINK | IOSQE_ASYNC)
J
Jens Axboe 已提交
4733

4734 4735
static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			  struct io_submit_state *state, struct io_kiocb **link)
J
Jens Axboe 已提交
4736
{
4737
	const struct cred *old_creds = NULL;
4738
	struct io_ring_ctx *ctx = req->ctx;
4739
	unsigned int sqe_flags;
4740
	int ret, id;
J
Jens Axboe 已提交
4741

4742 4743
	sqe_flags = READ_ONCE(sqe->flags);

J
Jens Axboe 已提交
4744
	/* enforce forwards compatibility on users */
4745
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
J
Jens Axboe 已提交
4746
		ret = -EINVAL;
4747
		goto err_req;
J
Jens Axboe 已提交
4748
	}
4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761

	id = READ_ONCE(sqe->personality);
	if (id) {
		const struct cred *personality_creds;

		personality_creds = idr_find(&ctx->personality_idr, id);
		if (unlikely(!personality_creds)) {
			ret = -EINVAL;
			goto err_req;
		}
		old_creds = override_creds(personality_creds);
	}

4762 4763 4764
	/* same numerical values with corresponding REQ_F_*, safe to copy */
	req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|
					IOSQE_ASYNC);
J
Jens Axboe 已提交
4765

4766
	ret = io_req_set_file(state, req, sqe);
J
Jens Axboe 已提交
4767 4768
	if (unlikely(ret)) {
err_req:
4769 4770
		io_cqring_add_event(req, ret);
		io_double_put_req(req);
4771 4772
		if (old_creds)
			revert_creds(old_creds);
4773
		return false;
J
Jens Axboe 已提交
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783
	}

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
P
Pavel Begunkov 已提交
4784
		struct io_kiocb *head = *link;
J
Jens Axboe 已提交
4785

4786 4787 4788 4789 4790 4791 4792
		/*
		 * Taking sequential execution of a link, draining both sides
		 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
		 * requests in the link. So, it drains the head and the
		 * next after the link request. The last one is done via
		 * drain_next flag to persist the effect across calls.
		 */
4793 4794 4795 4796
		if (sqe_flags & IOSQE_IO_DRAIN) {
			head->flags |= REQ_F_IO_DRAIN;
			ctx->drain_next = 1;
		}
4797
		if (io_alloc_async_ctx(req)) {
J
Jens Axboe 已提交
4798 4799 4800 4801
			ret = -EAGAIN;
			goto err_req;
		}

4802
		ret = io_req_defer_prep(req, sqe);
4803
		if (ret) {
J
Jens Axboe 已提交
4804
			/* fail even hard links since we don't submit */
P
Pavel Begunkov 已提交
4805
			head->flags |= REQ_F_FAIL_LINK;
4806
			goto err_req;
4807
		}
P
Pavel Begunkov 已提交
4808 4809
		trace_io_uring_link(ctx, req, head);
		list_add_tail(&req->link_list, &head->link_list);
4810 4811 4812 4813 4814 4815

		/* last request of a link, enqueue the link */
		if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
			io_queue_link_head(head);
			*link = NULL;
		}
J
Jens Axboe 已提交
4816
	} else {
4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830
		if (unlikely(ctx->drain_next)) {
			req->flags |= REQ_F_IO_DRAIN;
			req->ctx->drain_next = 0;
		}
		if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
			req->flags |= REQ_F_LINK;
			INIT_LIST_HEAD(&req->link_list);
			ret = io_req_defer_prep(req, sqe);
			if (ret)
				req->flags |= REQ_F_FAIL_LINK;
			*link = req;
		} else {
			io_queue_sqe(req, sqe);
		}
J
Jens Axboe 已提交
4831
	}
4832

4833 4834
	if (old_creds)
		revert_creds(old_creds);
4835
	return true;
J
Jens Axboe 已提交
4836 4837
}

4838 4839 4840 4841 4842 4843
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
	blk_finish_plug(&state->plug);
4844
	io_file_put(state);
J
Jens Axboe 已提交
4845
	if (state->free_reqs)
4846
		kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
4847 4848 4849 4850 4851 4852
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
4853
				  unsigned int max_ios)
4854 4855
{
	blk_start_plug(&state->plug);
J
Jens Axboe 已提交
4856
	state->free_reqs = 0;
4857 4858 4859 4860
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
4861 4862
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
4863
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
4864

4865 4866 4867 4868 4869 4870
	/*
	 * Ensure any loads from the SQEs are done at this point,
	 * since once we write the new head, the application could
	 * write new data to them.
	 */
	smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
4871 4872 4873
}

/*
4874
 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
J
Jens Axboe 已提交
4875 4876 4877 4878 4879 4880
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
4881 4882
static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req,
			  const struct io_uring_sqe **sqe_ptr)
J
Jens Axboe 已提交
4883
{
4884
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
4885 4886 4887 4888 4889 4890 4891 4892 4893 4894
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
4895
	head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
4896
	if (likely(head < ctx->sq_entries)) {
4897 4898 4899 4900 4901 4902
		/*
		 * All io need record the previous position, if LINK vs DARIN,
		 * it can be used to mark the position of the first IO in the
		 * link list.
		 */
		req->sequence = ctx->cached_sq_head;
4903 4904 4905
		*sqe_ptr = &ctx->sq_sqes[head];
		req->opcode = READ_ONCE((*sqe_ptr)->opcode);
		req->user_data = READ_ONCE((*sqe_ptr)->user_data);
J
Jens Axboe 已提交
4906 4907 4908 4909 4910 4911
		ctx->cached_sq_head++;
		return true;
	}

	/* drop invalid entries */
	ctx->cached_sq_head++;
4912
	ctx->cached_sq_dropped++;
4913
	WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
J
Jens Axboe 已提交
4914 4915 4916
	return false;
}

4917
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
4918 4919
			  struct file *ring_file, int ring_fd,
			  struct mm_struct **mm, bool async)
J
Jens Axboe 已提交
4920 4921
{
	struct io_submit_state state, *statep = NULL;
J
Jens Axboe 已提交
4922 4923
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
4924
	bool mm_fault = false;
J
Jens Axboe 已提交
4925

4926
	/* if we have a backlog and couldn't flush it all, return BUSY */
4927 4928 4929 4930 4931
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}
4932

4933 4934
	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
4935

4936 4937 4938
	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;

J
Jens Axboe 已提交
4939
	if (nr > IO_PLUG_THRESHOLD) {
4940
		io_submit_state_start(&state, nr);
J
Jens Axboe 已提交
4941 4942 4943
		statep = &state;
	}

P
Pavel Begunkov 已提交
4944 4945 4946
	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

J
Jens Axboe 已提交
4947
	for (i = 0; i < nr; i++) {
4948
		const struct io_uring_sqe *sqe;
4949
		struct io_kiocb *req;
4950

4951 4952 4953 4954
		req = io_get_req(ctx, statep);
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
4955
			break;
4956
		}
4957
		if (!io_get_sqring(ctx, req, &sqe)) {
4958
			__io_req_do_free(req);
4959 4960
			break;
		}
4961

4962 4963 4964 4965 4966 4967 4968 4969 4970 4971
		/* will complete beyond this point, count as submitted */
		submitted++;

		if (unlikely(req->opcode >= IORING_OP_LAST)) {
			io_cqring_add_event(req, -EINVAL);
			io_double_put_req(req);
			break;
		}

		if (io_op_defs[req->opcode].needs_mm && !*mm) {
4972 4973 4974 4975 4976 4977 4978
			mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
			if (!mm_fault) {
				use_mm(ctx->sqo_mm);
				*mm = ctx->sqo_mm;
			}
		}

4979 4980 4981
		req->has_user = *mm != NULL;
		req->in_async = async;
		req->needs_fixed_file = async;
4982 4983
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
						true, async);
4984
		if (!io_submit_sqe(req, sqe, statep, &link))
4985
			break;
J
Jens Axboe 已提交
4986 4987
	}

4988 4989 4990 4991 4992
	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
J
Jens Axboe 已提交
4993
	if (link)
4994
		io_queue_link_head(link);
J
Jens Axboe 已提交
4995 4996 4997
	if (statep)
		io_submit_state_end(&state);

4998 4999 5000
	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

J
Jens Axboe 已提交
5001 5002 5003 5004 5005 5006 5007
	return submitted;
}

static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
	struct mm_struct *cur_mm = NULL;
5008
	const struct cred *old_cred;
J
Jens Axboe 已提交
5009 5010 5011
	mm_segment_t old_fs;
	DEFINE_WAIT(wait);
	unsigned long timeout;
5012
	int ret = 0;
J
Jens Axboe 已提交
5013

5014
	complete(&ctx->completions[1]);
5015

J
Jens Axboe 已提交
5016 5017
	old_fs = get_fs();
	set_fs(USER_DS);
5018
	old_cred = override_creds(ctx->creds);
J
Jens Axboe 已提交
5019

5020
	timeout = jiffies + ctx->sq_thread_idle;
5021
	while (!kthread_should_park()) {
5022
		unsigned int to_submit;
J
Jens Axboe 已提交
5023

5024
		if (!list_empty(&ctx->poll_list)) {
J
Jens Axboe 已提交
5025 5026
			unsigned nr_events = 0;

5027 5028 5029 5030
			mutex_lock(&ctx->uring_lock);
			if (!list_empty(&ctx->poll_list))
				io_iopoll_getevents(ctx, &nr_events, 0);
			else
J
Jens Axboe 已提交
5031
				timeout = jiffies + ctx->sq_thread_idle;
5032
			mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
5033 5034
		}

5035
		to_submit = io_sqring_entries(ctx);
5036 5037 5038 5039 5040 5041

		/*
		 * If submit got -EBUSY, flag us as needing the application
		 * to enter the kernel to reap and flush events.
		 */
		if (!to_submit || ret == -EBUSY) {
J
Jens Axboe 已提交
5042 5043 5044
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
5045 5046 5047
			 * to sleep. The exception is if we got EBUSY doing
			 * more IO, we should wait for the application to
			 * reap events and wake us up.
J
Jens Axboe 已提交
5048
			 */
5049
			if (!list_empty(&ctx->poll_list) ||
5050
			    (!time_after(jiffies, timeout) && ret != -EBUSY)) {
5051
				cond_resched();
J
Jens Axboe 已提交
5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068
				continue;
			}

			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
			if (cur_mm) {
				unuse_mm(cur_mm);
				mmput(cur_mm);
				cur_mm = NULL;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);
5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080
			/*
			 * While doing polled IO, before going to sleep, we need
			 * to check if there are new reqs added to poll_list, it
			 * is because reqs may have been punted to io worker and
			 * will be added to poll_list later, hence check the
			 * poll_list again.
			 */
			if ((ctx->flags & IORING_SETUP_IOPOLL) &&
			    !list_empty_careful(&ctx->poll_list)) {
				finish_wait(&ctx->sqo_wait, &wait);
				continue;
			}
J
Jens Axboe 已提交
5081 5082

			/* Tell userspace we may need a wakeup call */
5083
			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
5084 5085
			/* make sure to read SQ tail after writing flags */
			smp_mb();
J
Jens Axboe 已提交
5086

5087
			to_submit = io_sqring_entries(ctx);
5088
			if (!to_submit || ret == -EBUSY) {
5089
				if (kthread_should_park()) {
J
Jens Axboe 已提交
5090 5091 5092 5093 5094 5095 5096 5097
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

5098
				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
5099 5100 5101 5102
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

5103
			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
5104 5105
		}

5106
		mutex_lock(&ctx->uring_lock);
5107
		ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
5108
		mutex_unlock(&ctx->uring_lock);
5109
		timeout = jiffies + ctx->sq_thread_idle;
J
Jens Axboe 已提交
5110 5111 5112 5113 5114 5115 5116
	}

	set_fs(old_fs);
	if (cur_mm) {
		unuse_mm(cur_mm);
		mmput(cur_mm);
	}
5117
	revert_creds(old_cred);
5118

5119
	kthread_parkme();
5120

J
Jens Axboe 已提交
5121 5122 5123
	return 0;
}

5124 5125 5126 5127 5128 5129 5130
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

5131
static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
5132 5133 5134 5135
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
5136
	 * Wake up if we have enough events, or if a timeout occurred since we
5137 5138 5139
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
5140
	return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
5141 5142 5143 5144 5145 5146 5147 5148 5149
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

5150 5151
	/* use noflush == true, as we can't safely rely on locking context */
	if (!io_should_wake(iowq, true))
5152 5153 5154 5155 5156
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
5157 5158 5159 5160 5161 5162 5163
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
5164 5165 5166 5167 5168 5169 5170 5171 5172
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
5173
	struct io_rings *rings = ctx->rings;
5174
	int ret = 0;
J
Jens Axboe 已提交
5175

5176
	if (io_cqring_events(ctx, false) >= min_events)
J
Jens Axboe 已提交
5177 5178 5179
		return 0;

	if (sig) {
5180 5181 5182
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
5183
						      sigsz);
5184 5185
		else
#endif
5186
			ret = set_user_sigmask(sig, sigsz);
5187

J
Jens Axboe 已提交
5188 5189 5190 5191
		if (ret)
			return ret;
	}

5192
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
5193
	trace_io_uring_cqring_wait(ctx, min_events);
5194 5195 5196
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
5197
		if (io_should_wake(&iowq, false))
5198 5199 5200
			break;
		schedule();
		if (signal_pending(current)) {
5201
			ret = -EINTR;
5202 5203 5204 5205 5206
			break;
		}
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

5207
	restore_saved_sigmask_unless(ret == -EINTR);
J
Jens Axboe 已提交
5208

5209
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
5210 5211
}

J
Jens Axboe 已提交
5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

5225 5226 5227 5228 5229 5230 5231
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
5232 5233 5234
#endif
}

5235 5236 5237 5238 5239 5240 5241 5242
static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}

J
Jens Axboe 已提交
5243 5244
static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
5245
	struct fixed_file_data *data = ctx->file_data;
5246 5247
	unsigned nr_tables, i;

5248
	if (!data)
J
Jens Axboe 已提交
5249 5250
		return -ENXIO;

5251 5252
	/* protect against inflight atomic switch, which drops the ref */
	percpu_ref_get(&data->refs);
5253 5254
	/* wait for existing switches */
	flush_work(&data->ref_work);
5255 5256 5257
	percpu_ref_kill_and_confirm(&data->refs, io_file_ref_kill);
	wait_for_completion(&data->done);
	percpu_ref_put(&data->refs);
5258 5259
	/* flush potential new switch */
	flush_work(&data->ref_work);
5260 5261
	percpu_ref_exit(&data->refs);

J
Jens Axboe 已提交
5262
	__io_sqe_files_unregister(ctx);
5263 5264
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
5265 5266 5267 5268
		kfree(data->table[i].files);
	kfree(data->table);
	kfree(data);
	ctx->file_data = NULL;
J
Jens Axboe 已提交
5269 5270 5271 5272
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
5273 5274 5275
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
5276
		wait_for_completion(&ctx->completions[1]);
5277 5278 5279 5280 5281
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
5282
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
5283 5284 5285 5286 5287
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
5288 5289
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
5290 5291
	io_sq_thread_stop(ctx);

5292 5293 5294
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308
	}
}

#if defined(CONFIG_UNIX)
/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
5309
	int i, nr_files;
J
Jens Axboe 已提交
5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329

	if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
		unsigned long inflight = ctx->user->unix_inflight + nr;

		if (inflight > task_rlimit(current, RLIMIT_NOFILE))
			return -EMFILE;
	}

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

5330
	nr_files = 0;
J
Jens Axboe 已提交
5331 5332
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
5333 5334 5335
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
5336
			continue;
5337
		fpl->fp[nr_files] = get_file(file);
5338 5339
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
5340 5341
	}

5342 5343 5344 5345
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
5346
		skb->destructor = unix_destruct_scm;
5347 5348
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
5349

5350 5351 5352 5353 5354 5355
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
5386 5387 5388 5389
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

5402 5403 5404 5405 5406 5407
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
5408
		struct fixed_file_table *table = &ctx->file_data->table[i];
5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
5423
		struct fixed_file_table *table = &ctx->file_data->table[i];
5424 5425 5426 5427 5428
		kfree(table->files);
	}
	return 1;
}

5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528
static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
	fput(file);
#endif
}

struct io_file_put {
	struct llist_node llist;
	struct file *file;
	struct completion *done;
};

static void io_ring_file_ref_switch(struct work_struct *work)
{
	struct io_file_put *pfile, *tmp;
	struct fixed_file_data *data;
	struct llist_node *node;

	data = container_of(work, struct fixed_file_data, ref_work);

	while ((node = llist_del_all(&data->put_llist)) != NULL) {
		llist_for_each_entry_safe(pfile, tmp, node, llist) {
			io_ring_file_put(data->ctx, pfile->file);
			if (pfile->done)
				complete(pfile->done);
			else
				kfree(pfile);
		}
	}

	percpu_ref_get(&data->refs);
	percpu_ref_switch_to_percpu(&data->refs);
}

static void io_file_data_ref_zero(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);

	/* we can't safely switch from inside this context, punt to wq */
	queue_work(system_wq, &data->ref_work);
}

J
Jens Axboe 已提交
5529 5530 5531 5532
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
5533
	unsigned nr_tables;
5534
	struct file *file;
J
Jens Axboe 已提交
5535 5536 5537
	int fd, ret = 0;
	unsigned i;

5538
	if (ctx->file_data)
J
Jens Axboe 已提交
5539 5540 5541 5542 5543 5544
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

5545 5546 5547 5548 5549 5550
	ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
	if (!ctx->file_data)
		return -ENOMEM;
	ctx->file_data->ctx = ctx;
	init_completion(&ctx->file_data->done);

5551
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
5552 5553
	ctx->file_data->table = kcalloc(nr_tables,
					sizeof(struct fixed_file_table),
5554
					GFP_KERNEL);
5555 5556 5557
	if (!ctx->file_data->table) {
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
5558
		return -ENOMEM;
5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569
	}

	if (percpu_ref_init(&ctx->file_data->refs, io_file_data_ref_zero,
				PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
		return -ENOMEM;
	}
	ctx->file_data->put_llist.first = NULL;
	INIT_WORK(&ctx->file_data->ref_work, io_ring_file_ref_switch);
J
Jens Axboe 已提交
5570

5571
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
5572 5573 5574 5575
		percpu_ref_exit(&ctx->file_data->refs);
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
5576 5577 5578
		return -ENOMEM;
	}

5579
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
5580 5581 5582
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
5583 5584 5585
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
5586 5587 5588 5589 5590
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
5591

5592
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5593
		index = i & IORING_FILE_TABLE_MASK;
5594
		file = fget(fd);
J
Jens Axboe 已提交
5595 5596

		ret = -EBADF;
5597
		if (!file)
J
Jens Axboe 已提交
5598
			break;
5599

J
Jens Axboe 已提交
5600 5601 5602 5603 5604 5605 5606
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
5607 5608
		if (file->f_op == &io_uring_fops) {
			fput(file);
J
Jens Axboe 已提交
5609 5610 5611
			break;
		}
		ret = 0;
5612
		table->files[index] = file;
J
Jens Axboe 已提交
5613 5614 5615
	}

	if (ret) {
5616 5617 5618 5619 5620 5621
		for (i = 0; i < ctx->nr_user_files; i++) {
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
5622
			kfree(ctx->file_data->table[i].files);
5623

5624 5625 5626
		kfree(ctx->file_data->table);
		kfree(ctx->file_data);
		ctx->file_data = NULL;
J
Jens Axboe 已提交
5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
	if (ret)
		io_sqe_files_unregister(ctx);

	return ret;
}

5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680
static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

5681
static void io_atomic_switch(struct percpu_ref *ref)
5682
{
5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	clear_bit(FFD_F_ATOMIC, &data->state);
}

static bool io_queue_file_removal(struct fixed_file_data *data,
				  struct file *file)
{
	struct io_file_put *pfile, pfile_stack;
	DECLARE_COMPLETION_ONSTACK(done);

	/*
	 * If we fail allocating the struct we need for doing async reomval
	 * of this file, just punt to sync and wait for it.
	 */
	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
	if (!pfile) {
		pfile = &pfile_stack;
		pfile->done = &done;
	}

	pfile->file = file;
	llist_add(&pfile->llist, &data->put_llist);

	if (pfile == &pfile_stack) {
		if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
			percpu_ref_put(&data->refs);
			percpu_ref_switch_to_atomic(&data->refs,
							io_atomic_switch);
		}
		wait_for_completion(&done);
		flush_work(&data->ref_work);
		return false;
	}

	return true;
}

static int __io_sqe_files_update(struct io_ring_ctx *ctx,
				 struct io_uring_files_update *up,
				 unsigned nr_args)
{
	struct fixed_file_data *data = ctx->file_data;
	bool ref_switch = false;
	struct file *file;
5729 5730 5731 5732
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;

5733
	if (check_add_overflow(up->offset, nr_args, &done))
5734 5735 5736 5737 5738
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

	done = 0;
5739
	fds = u64_to_user_ptr(up->fds);
5740
	while (nr_args) {
5741 5742 5743
		struct fixed_file_table *table;
		unsigned index;

5744 5745 5746 5747 5748
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
5749 5750
		i = array_index_nospec(up->offset, ctx->nr_user_files);
		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5751 5752
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
5753
			file = io_file_from_index(ctx, index);
5754
			table->files[index] = NULL;
5755 5756
			if (io_queue_file_removal(data, file))
				ref_switch = true;
5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776
		}
		if (fd != -1) {
			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
5777
			table->files[index] = file;
5778 5779 5780 5781 5782 5783
			err = io_sqe_file_register(ctx, file, i);
			if (err)
				break;
		}
		nr_args--;
		done++;
5784 5785 5786 5787 5788 5789
		up->offset++;
	}

	if (ref_switch && !test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
		percpu_ref_put(&data->refs);
		percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
5790 5791 5792 5793
	}

	return done ? done : err;
}
5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809
static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}
5810

5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824
static void io_put_work(struct io_wq_work *work)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	io_put_req(req);
}

static void io_get_work(struct io_wq_work *work)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	refcount_inc(&req->refs);
}

5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871
static int io_init_wq_offload(struct io_ring_ctx *ctx,
			      struct io_uring_params *p)
{
	struct io_wq_data data;
	struct fd f;
	struct io_ring_ctx *ctx_attach;
	unsigned int concurrency;
	int ret = 0;

	data.user = ctx->user;
	data.get_work = io_get_work;
	data.put_work = io_put_work;

	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
		/* Do QD, or 4 * CPUS, whatever is smallest */
		concurrency = min(ctx->sq_entries, 4 * num_online_cpus());

		ctx->io_wq = io_wq_create(concurrency, &data);
		if (IS_ERR(ctx->io_wq)) {
			ret = PTR_ERR(ctx->io_wq);
			ctx->io_wq = NULL;
		}
		return ret;
	}

	f = fdget(p->wq_fd);
	if (!f.file)
		return -EBADF;

	if (f.file->f_op != &io_uring_fops) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx_attach = f.file->private_data;
	/* @io_wq is protected by holding the fd */
	if (!io_wq_get(ctx_attach->io_wq, &data)) {
		ret = -EINVAL;
		goto out_fput;
	}

	ctx->io_wq = ctx_attach->io_wq;
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
5872 5873
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
5874 5875 5876
{
	int ret;

J
Jens Axboe 已提交
5877
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
5878 5879 5880
	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

J
Jens Axboe 已提交
5881
	if (ctx->flags & IORING_SETUP_SQPOLL) {
5882 5883 5884 5885
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

5886 5887 5888 5889
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
5890
		if (p->flags & IORING_SETUP_SQ_AFF) {
5891
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
5892

5893
			ret = -EINVAL;
5894 5895
			if (cpu >= nr_cpu_ids)
				goto err;
5896
			if (!cpu_online(cpu))
5897 5898
				goto err;

J
Jens Axboe 已提交
5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

5918 5919
	ret = io_init_wq_offload(ctx, p);
	if (ret)
J
Jens Axboe 已提交
5920 5921 5922 5923
		goto err;

	return 0;
err:
5924
	io_finish_async(ctx);
J
Jens Axboe 已提交
5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954
	mmdrop(ctx->sqo_mm);
	ctx->sqo_mm = NULL;
	return ret;
}

static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

static void io_mem_free(void *ptr)
{
5955 5956 5957 5958
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
5959

5960
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	if (sq_offset)
		*sq_offset = off;

	return off;
}

J
Jens Axboe 已提交
6002 6003
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
6004
	size_t pages;
J
Jens Axboe 已提交
6005

6006 6007 6008 6009
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
6010

6011
	return pages;
J
Jens Axboe 已提交
6012 6013
}

6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
			put_page(imu->bvec[j].bv_page);

		if (ctx->account_mem)
			io_unaccount_mem(ctx->user, imu->nr_bvecs);
6029
		kvfree(imu->bvec);
6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

6053
		dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
6091
			goto err;
6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

		if (ctx->account_mem) {
			ret = io_account_mem(ctx->user, nr_pages);
			if (ret)
				goto err;
		}

		ret = 0;
		if (!pages || nr_pages > got_pages) {
			kfree(vmas);
			kfree(pages);
6121
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
6122
						GFP_KERNEL);
6123
			vmas = kvmalloc_array(nr_pages,
6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
				if (ctx->account_mem)
					io_unaccount_mem(ctx->user, nr_pages);
				goto err;
			}
			got_pages = nr_pages;
		}

6135
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
			goto err;
		}

		ret = 0;
		down_read(&current->mm->mmap_sem);
		pret = get_user_pages_longterm(ubuf, nr_pages, FOLL_WRITE,
						pages, vmas);
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
		up_read(&current->mm->mmap_sem);
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
			if (pret > 0) {
				for (j = 0; j < pret; j++)
					put_page(pages[j]);
			}
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
6174
			kvfree(imu->bvec);
6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
6197 6198
	kvfree(pages);
	kvfree(vmas);
6199 6200
	return 0;
err:
6201 6202
	kvfree(pages);
	kvfree(vmas);
6203 6204 6205 6206
	io_sqe_buffer_unregister(ctx);
	return ret;
}

6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

J
Jens Axboe 已提交
6239 6240
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
6241
	io_finish_async(ctx);
J
Jens Axboe 已提交
6242 6243
	if (ctx->sqo_mm)
		mmdrop(ctx->sqo_mm);
J
Jens Axboe 已提交
6244 6245

	io_iopoll_reap_events(ctx);
6246
	io_sqe_buffer_unregister(ctx);
J
Jens Axboe 已提交
6247
	io_sqe_files_unregister(ctx);
6248
	io_eventfd_unregister(ctx);
J
Jens Axboe 已提交
6249

J
Jens Axboe 已提交
6250
#if defined(CONFIG_UNIX)
6251 6252
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
6253
		sock_release(ctx->ring_sock);
6254
	}
J
Jens Axboe 已提交
6255 6256
#endif

6257
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
6258 6259 6260 6261 6262 6263 6264
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	if (ctx->account_mem)
		io_unaccount_mem(ctx->user,
				ring_pages(ctx->sq_entries, ctx->cq_entries));
	free_uid(ctx->user);
6265
	put_cred(ctx->creds);
6266
	kfree(ctx->completions);
6267
	kfree(ctx->cancel_hash);
6268
	kmem_cache_free(req_cachep, ctx->fallback_req);
J
Jens Axboe 已提交
6269 6270 6271 6272 6273 6274 6275 6276 6277
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
6278 6279 6280 6281
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
6282
	smp_rmb();
6283 6284
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
6285
		mask |= EPOLLOUT | EPOLLWRNORM;
6286
	if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
J
Jens Axboe 已提交
6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309
static int io_remove_personalities(int id, void *p, void *data)
{
	struct io_ring_ctx *ctx = data;
	const struct cred *cred;

	cred = idr_remove(&ctx->personality_idr, id);
	if (cred)
		put_cred(cred);
	return 0;
}

J
Jens Axboe 已提交
6310 6311 6312 6313 6314 6315
static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
6316
	io_kill_timeouts(ctx);
6317
	io_poll_remove_all(ctx);
6318 6319 6320 6321

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

J
Jens Axboe 已提交
6322
	io_iopoll_reap_events(ctx);
6323 6324 6325
	/* if we failed setting up the ctx, we might not have any rings */
	if (ctx->rings)
		io_cqring_overflow_flush(ctx, true);
6326
	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
6327
	wait_for_completion(&ctx->completions[0]);
J
Jens Axboe 已提交
6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339
	io_ring_ctx_free(ctx);
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

6340 6341 6342 6343 6344 6345 6346
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
	struct io_kiocb *req;
	DEFINE_WAIT(wait);

	while (!list_empty_careful(&ctx->inflight_list)) {
6347
		struct io_kiocb *cancel_req = NULL;
6348 6349 6350

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
6351 6352 6353 6354 6355 6356 6357
			if (req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
6358
		}
6359
		if (cancel_req)
6360
			prepare_to_wait(&ctx->inflight_wait, &wait,
6361
						TASK_UNINTERRUPTIBLE);
6362 6363
		spin_unlock_irq(&ctx->inflight_lock);

6364 6365
		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
6366
			break;
6367 6368 6369

		io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
		io_put_req(cancel_req);
6370 6371
		schedule();
	}
6372
	finish_wait(&ctx->inflight_wait, &wait);
6373 6374 6375 6376 6377 6378 6379 6380 6381 6382
}

static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
	return 0;
}

6383 6384
static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
J
Jens Axboe 已提交
6385 6386
{
	struct io_ring_ctx *ctx = file->private_data;
6387
	loff_t offset = pgoff << PAGE_SHIFT;
J
Jens Axboe 已提交
6388 6389 6390 6391 6392
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
6393 6394
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
6395 6396 6397 6398 6399
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
6400
		return ERR_PTR(-EINVAL);
J
Jens Axboe 已提交
6401 6402 6403 6404
	}

	page = virt_to_head_page(ptr);
	if (sz > (PAGE_SIZE << compound_order(page)))
6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420
		return ERR_PTR(-EINVAL);

	return ptr;
}

#ifdef CONFIG_MMU

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);
J
Jens Axboe 已提交
6421 6422 6423 6424 6425

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452
#else /* !CONFIG_MMU */

static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
}

static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
{
	return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
}

static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len,
	unsigned long pgoff, unsigned long flags)
{
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, pgoff, len);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	return (unsigned long) ptr;
}

#endif /* !CONFIG_MMU */

J
Jens Axboe 已提交
6453 6454 6455 6456 6457 6458 6459 6460 6461
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

J
Jens Axboe 已提交
6462
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
6478 6479 6480 6481 6482
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
6483
	ret = 0;
J
Jens Axboe 已提交
6484
	if (ctx->flags & IORING_SETUP_SQPOLL) {
6485 6486
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false);
J
Jens Axboe 已提交
6487 6488 6489
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
6490
	} else if (to_submit) {
6491
		struct mm_struct *cur_mm;
J
Jens Axboe 已提交
6492 6493

		mutex_lock(&ctx->uring_lock);
6494 6495 6496 6497
		/* already have mm, so io_submit_sqes() won't try to grab it */
		cur_mm = ctx->sqo_mm;
		submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
					   &cur_mm, false);
J
Jens Axboe 已提交
6498
		mutex_unlock(&ctx->uring_lock);
6499 6500 6501

		if (submitted != to_submit)
			goto out;
J
Jens Axboe 已提交
6502 6503
	}
	if (flags & IORING_ENTER_GETEVENTS) {
J
Jens Axboe 已提交
6504 6505
		unsigned nr_events = 0;

J
Jens Axboe 已提交
6506 6507
		min_complete = min(min_complete, ctx->cq_entries);

6508 6509 6510 6511 6512 6513 6514 6515
		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
J
Jens Axboe 已提交
6516 6517 6518 6519
			ret = io_iopoll_check(ctx, &nr_events, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
6520 6521
	}

6522
out:
6523
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
6524 6525 6526 6527 6528
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602
static int io_uring_show_cred(int id, void *p, void *data)
{
	const struct cred *cred = p;
	struct seq_file *m = data;
	struct user_namespace *uns = seq_user_ns(m);
	struct group_info *gi;
	kernel_cap_t cap;
	unsigned __capi;
	int g;

	seq_printf(m, "%5d\n", id);
	seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
	seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
	seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
	seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
	seq_puts(m, "\n\tGroups:\t");
	gi = cred->group_info;
	for (g = 0; g < gi->ngroups; g++) {
		seq_put_decimal_ull(m, g ? " " : "",
					from_kgid_munged(uns, gi->gid[g]));
	}
	seq_puts(m, "\n\tCapEff:\t");
	cap = cred->cap_effective;
	CAP_FOR_EACH_U32(__capi)
		seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
	seq_putc(m, '\n');
	return 0;
}

static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	int i;

	mutex_lock(&ctx->uring_lock);
	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (!idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
	mutex_unlock(&ctx->uring_lock);
}

static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
{
	struct io_ring_ctx *ctx = f->private_data;

	if (percpu_ref_tryget(&ctx->refs)) {
		__io_uring_show_fdinfo(ctx, m);
		percpu_ref_put(&ctx->refs);
	}
}

J
Jens Axboe 已提交
6603 6604
static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
6605
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
6606
	.mmap		= io_uring_mmap,
6607 6608 6609 6610
#ifndef CONFIG_MMU
	.get_unmapped_area = io_uring_nommu_get_unmapped_area,
	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
#endif
J
Jens Axboe 已提交
6611 6612
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
6613
	.show_fdinfo	= io_uring_show_fdinfo,
J
Jens Axboe 已提交
6614 6615 6616 6617 6618
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
6619 6620
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
6621

6622 6623 6624 6625 6626 6627
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
6628 6629
		return -ENOMEM;

6630 6631 6632 6633 6634 6635 6636 6637 6638 6639
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
	ctx->sq_entries = rings->sq_ring_entries;
	ctx->cq_entries = rings->cq_ring_entries;
J
Jens Axboe 已提交
6640 6641

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
6642 6643 6644
	if (size == SIZE_MAX) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
6645
		return -EOVERFLOW;
6646
	}
J
Jens Axboe 已提交
6647 6648

	ctx->sq_sqes = io_mem_alloc(size);
6649 6650 6651
	if (!ctx->sq_sqes) {
		io_mem_free(ctx->rings);
		ctx->rings = NULL;
J
Jens Axboe 已提交
6652
		return -ENOMEM;
6653
	}
J
Jens Axboe 已提交
6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

static int io_uring_create(unsigned entries, struct io_uring_params *p)
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
	bool account_mem;
	int ret;

6708
	if (!entries)
J
Jens Axboe 已提交
6709
		return -EINVAL;
6710 6711 6712 6713 6714
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}
J
Jens Axboe 已提交
6715 6716 6717 6718 6719

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
6720 6721 6722
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
6723 6724
	 */
	p->sq_entries = roundup_pow_of_two(entries);
6725 6726 6727 6728 6729 6730
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
6731
		if (p->cq_entries < p->sq_entries)
6732
			return -EINVAL;
6733 6734 6735 6736 6737
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
6738 6739 6740 6741
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765

	user = get_uid(current_user());
	account_mem = !capable(CAP_IPC_LOCK);

	if (account_mem) {
		ret = io_account_mem(user,
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
		if (account_mem)
			io_unaccount_mem(user, ring_pages(p->sq_entries,
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->account_mem = account_mem;
	ctx->user = user;
6766
	ctx->creds = get_current_cred();
J
Jens Axboe 已提交
6767 6768 6769 6770 6771

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
6772
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
6773 6774 6775 6776
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
6777 6778 6779 6780 6781 6782 6783
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
6784 6785

	memset(&p->cq_off, 0, sizeof(p->cq_off));
6786 6787 6788 6789 6790 6791
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
6792

6793 6794 6795 6796 6797 6798 6799 6800
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

6801
	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
6802 6803
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
			IORING_FEAT_CUR_PERSONALITY;
6804
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
J
Jens Axboe 已提交
6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	long ret;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
6829
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
6830
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
6831
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
J
Jens Axboe 已提交
6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849
		return -EINVAL;

	ret = io_uring_create(entries, &p);
	if (ret < 0)
		return ret;

	if (copy_to_user(params, &p, sizeof(p)))
		return -EFAULT;

	return ret;
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888
static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}

6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927
static int io_register_personality(struct io_ring_ctx *ctx)
{
	const struct cred *creds = get_current_cred();
	int id;

	id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
				USHRT_MAX, GFP_KERNEL);
	if (id < 0)
		put_cred(creds);
	return id;
}

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}

static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}

6928 6929
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
6930 6931
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
6932 6933 6934
{
	int ret;

6935 6936 6937 6938 6939 6940 6941 6942
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

6943
	if (io_register_op_must_quiesce(opcode)) {
6944
		percpu_ref_kill(&ctx->refs);
6945

6946 6947 6948 6949 6950 6951 6952 6953 6954
		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
6955
		ret = wait_for_completion_interruptible(&ctx->completions[0]);
6956
		mutex_lock(&ctx->uring_lock);
6957 6958 6959 6960 6961
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
6962
	}
6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
6974 6975 6976 6977 6978 6979 6980 6981 6982
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
6983 6984 6985
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
6986
	case IORING_REGISTER_EVENTFD:
6987
	case IORING_REGISTER_EVENTFD_ASYNC:
6988 6989 6990 6991
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
6992 6993 6994 6995 6996 6997
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
6998 6999 7000 7001 7002 7003 7004
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
7005 7006 7007 7008 7009 7010
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
7023 7024 7025 7026 7027
	default:
		ret = -EINVAL;
		break;
	}

7028
	if (io_register_op_must_quiesce(opcode)) {
7029 7030
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
7031 7032
out:
		reinit_completion(&ctx->completions[0]);
7033
	}
7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
7057 7058
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
7059 7060 7061 7062 7063
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
7064 7065
static int __init io_uring_init(void)
{
7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
	BUILD_BUG_SQE_ELEM(28, __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);

7099
	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
J
Jens Axboe 已提交
7100 7101 7102 7103
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);