io_uring.c 105.6 KB
Newer Older
J
Jens Axboe 已提交
1 2 3 4 5 6
// SPDX-License-Identifier: GPL-2.0
/*
 * Shared application/kernel submission and completion ring pairs, for
 * supporting fast/efficient IO.
 *
 * A note on the read/write ordering memory barriers that are matched between
S
Stefan Bühler 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * the application and kernel side.
 *
 * After the application reads the CQ ring tail, it must use an
 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
 * before writing the tail (using smp_load_acquire to read the tail will
 * do). It also needs a smp_mb() before updating CQ head (ordering the
 * entry load(s) with the head store), pairing with an implicit barrier
 * through a control-dependency in io_get_cqring (smp_store_release to
 * store head will do). Failure to do so could lead to reading invalid
 * CQ entries.
 *
 * Likewise, the application must use an appropriate smp_wmb() before
 * writing the SQ tail (ordering SQ entry stores with the tail store),
 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
 * to store the tail will do). And it needs a barrier ordering the SQ
 * head load before writing new SQ entries (smp_load_acquire to read
 * head will do).
 *
 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
 * updating the SQ tail; a full memory barrier smp_mb() is needed
 * between.
J
Jens Axboe 已提交
29 30 31 32 33 34 35 36 37 38 39
 *
 * Also see the examples in the liburing library:
 *
 *	git://git.kernel.dk/liburing
 *
 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
 * from data shared between the kernel and application. This is done both
 * for ordering purposes, but also to ensure that once a value is loaded from
 * data that the application could potentially modify, it remains stable.
 *
 * Copyright (C) 2018-2019 Jens Axboe
C
Christoph Hellwig 已提交
40
 * Copyright (c) 2018-2019 Christoph Hellwig
J
Jens Axboe 已提交
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
#include <linux/refcount.h>
#include <linux/uio.h>

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/mmu_context.h>
#include <linux/percpu.h>
#include <linux/slab.h>
J
Jens Axboe 已提交
59
#include <linux/kthread.h>
J
Jens Axboe 已提交
60
#include <linux/blkdev.h>
61
#include <linux/bvec.h>
J
Jens Axboe 已提交
62 63 64
#include <linux/net.h>
#include <net/sock.h>
#include <net/af_unix.h>
J
Jens Axboe 已提交
65
#include <net/scm.h>
J
Jens Axboe 已提交
66 67 68 69
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
70 71
#include <linux/sizes.h>
#include <linux/hugetlb.h>
J
Jens Axboe 已提交
72

73 74 75
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>

J
Jens Axboe 已提交
76 77 78
#include <uapi/linux/io_uring.h>

#include "internal.h"
79
#include "io-wq.h"
J
Jens Axboe 已提交
80

81
#define IORING_MAX_ENTRIES	32768
82
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
83 84 85 86 87 88 89 90

/*
 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
 */
#define IORING_FILE_TABLE_SHIFT	9
#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)
J
Jens Axboe 已提交
91 92 93 94 95 96

struct io_uring {
	u32 head ____cacheline_aligned_in_smp;
	u32 tail ____cacheline_aligned_in_smp;
};

S
Stefan Bühler 已提交
97
/*
98 99
 * This data is shared with the application through the mmap at offsets
 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
S
Stefan Bühler 已提交
100 101 102 103
 *
 * The offsets to the member fields are published through struct
 * io_sqring_offsets when calling io_uring_setup.
 */
104
struct io_rings {
S
Stefan Bühler 已提交
105 106 107 108
	/*
	 * Head and tail offsets into the ring; the offsets need to be
	 * masked to get valid indices.
	 *
109 110 111
	 * The kernel controls head of the sq ring and the tail of the cq ring,
	 * and the application controls tail of the sq ring and the head of the
	 * cq ring.
S
Stefan Bühler 已提交
112
	 */
113
	struct io_uring		sq, cq;
S
Stefan Bühler 已提交
114
	/*
115
	 * Bitmasks to apply to head and tail offsets (constant, equals
S
Stefan Bühler 已提交
116 117
	 * ring_entries - 1)
	 */
118 119 120
	u32			sq_ring_mask, cq_ring_mask;
	/* Ring sizes (constant, power of 2) */
	u32			sq_ring_entries, cq_ring_entries;
S
Stefan Bühler 已提交
121 122 123 124 125 126 127 128 129 130 131 132
	/*
	 * Number of invalid entries dropped by the kernel due to
	 * invalid index stored in array
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * After a new SQ head value was read by the application this
	 * counter includes all submissions that were dropped reaching
	 * the new SQ head (and possibly more).
	 */
133
	u32			sq_dropped;
S
Stefan Bühler 已提交
134 135 136 137 138 139 140 141 142
	/*
	 * Runtime flags
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application.
	 *
	 * The application needs a full memory barrier before checking
	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
	 */
143
	u32			sq_flags;
S
Stefan Bühler 已提交
144 145 146 147 148 149 150 151 152 153 154 155 156
	/*
	 * Number of completion events lost because the queue was full;
	 * this should be avoided by the application by making sure
	 * there are not more requests pending thatn there is space in
	 * the completion queue.
	 *
	 * Written by the kernel, shouldn't be modified by the
	 * application (i.e. get number of "new events" by comparing to
	 * cached value).
	 *
	 * As completion events come in out of order this counter is not
	 * ordered with any other data.
	 */
157
	u32			cq_overflow;
S
Stefan Bühler 已提交
158 159 160 161 162 163 164
	/*
	 * Ring buffer of completion events.
	 *
	 * The kernel writes completion events fresh every time they are
	 * produced, so the application is allowed to modify pending
	 * entries.
	 */
165
	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
J
Jens Axboe 已提交
166 167
};

168 169 170 171 172 173 174
struct io_mapped_ubuf {
	u64		ubuf;
	size_t		len;
	struct		bio_vec *bvec;
	unsigned int	nr_bvecs;
};

175 176 177 178
struct fixed_file_table {
	struct file		**files;
};

J
Jens Axboe 已提交
179 180 181 182 183 184 185 186 187 188
struct io_ring_ctx {
	struct {
		struct percpu_ref	refs;
	} ____cacheline_aligned_in_smp;

	struct {
		unsigned int		flags;
		bool			compat;
		bool			account_mem;

189 190 191 192 193 194 195 196 197 198 199 200
		/*
		 * Ring buffer of indices into array of io_uring_sqe, which is
		 * mmapped by the application using the IORING_OFF_SQES offset.
		 *
		 * This indirection could e.g. be used to assign fixed
		 * io_uring_sqe entries to operations and only submit them to
		 * the queue when needed.
		 *
		 * The kernel modifies neither the indices array nor the entries
		 * array.
		 */
		u32			*sq_array;
J
Jens Axboe 已提交
201 202 203
		unsigned		cached_sq_head;
		unsigned		sq_entries;
		unsigned		sq_mask;
J
Jens Axboe 已提交
204
		unsigned		sq_thread_idle;
205
		unsigned		cached_sq_dropped;
J
Jens Axboe 已提交
206
		struct io_uring_sqe	*sq_sqes;
207 208

		struct list_head	defer_list;
J
Jens Axboe 已提交
209
		struct list_head	timeout_list;
210 211

		wait_queue_head_t	inflight_wait;
J
Jens Axboe 已提交
212 213 214
	} ____cacheline_aligned_in_smp;

	/* IO offload */
215
	struct io_wq		*io_wq;
J
Jens Axboe 已提交
216
	struct task_struct	*sqo_thread;	/* if using sq thread polling */
J
Jens Axboe 已提交
217
	struct mm_struct	*sqo_mm;
J
Jens Axboe 已提交
218
	wait_queue_head_t	sqo_wait;
219
	struct completion	sqo_thread_started;
J
Jens Axboe 已提交
220 221 222

	struct {
		unsigned		cached_cq_tail;
223
		atomic_t		cached_cq_overflow;
J
Jens Axboe 已提交
224 225 226 227
		unsigned		cq_entries;
		unsigned		cq_mask;
		struct wait_queue_head	cq_wait;
		struct fasync_struct	*cq_fasync;
228
		struct eventfd_ctx	*cq_ev_fd;
J
Jens Axboe 已提交
229
		atomic_t		cq_timeouts;
J
Jens Axboe 已提交
230 231
	} ____cacheline_aligned_in_smp;

232 233
	struct io_rings	*rings;

J
Jens Axboe 已提交
234 235 236 237 238
	/*
	 * If used, fixed file set. Writers must ensure that ->refs is dead,
	 * readers must ensure that ->refs is alive as long as the file* is
	 * used. Only updated through io_uring_register(2).
	 */
239
	struct fixed_file_table	*file_table;
J
Jens Axboe 已提交
240 241
	unsigned		nr_user_files;

242 243 244 245
	/* if used, fixed mapped user buffers */
	unsigned		nr_user_bufs;
	struct io_mapped_ubuf	*user_bufs;

J
Jens Axboe 已提交
246 247 248 249 250 251 252 253 254 255 256
	struct user_struct	*user;

	struct completion	ctx_done;

	struct {
		struct mutex		uring_lock;
		wait_queue_head_t	wait;
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t		completion_lock;
J
Jens Axboe 已提交
257 258 259 260 261 262 263 264
		bool			poll_multi_file;
		/*
		 * ->poll_list is protected by the ctx->uring_lock for
		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
		 * For SQPOLL, only the single threaded io_sq_thread() will
		 * manipulate the list, hence no extra locking is needed there.
		 */
		struct list_head	poll_list;
265
		struct list_head	cancel_list;
266 267 268

		spinlock_t		inflight_lock;
		struct list_head	inflight_list;
J
Jens Axboe 已提交
269 270 271 272 273 274 275 276 277
	} ____cacheline_aligned_in_smp;

#if defined(CONFIG_UNIX)
	struct socket		*ring_sock;
#endif
};

struct sqe_submit {
	const struct io_uring_sqe	*sqe;
278 279
	struct file			*ring_file;
	int				ring_fd;
280
	u32				sequence;
J
Jens Axboe 已提交
281
	bool				has_user;
282
	bool				in_async;
J
Jens Axboe 已提交
283
	bool				needs_fixed_file;
J
Jens Axboe 已提交
284 285
};

J
Jens Axboe 已提交
286 287 288 289
/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
290 291 292 293
struct io_poll_iocb {
	struct file			*file;
	struct wait_queue_head		*head;
	__poll_t			events;
J
Jens Axboe 已提交
294
	bool				done;
295 296 297 298
	bool				canceled;
	struct wait_queue_entry		wait;
};

J
Jens Axboe 已提交
299 300 301 302 303
struct io_timeout {
	struct file			*file;
	struct hrtimer			timer;
};

J
Jens Axboe 已提交
304 305 306 307 308 309
/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
J
Jens Axboe 已提交
310
struct io_kiocb {
311
	union {
J
Jens Axboe 已提交
312
		struct file		*file;
313 314
		struct kiocb		rw;
		struct io_poll_iocb	poll;
J
Jens Axboe 已提交
315
		struct io_timeout	timeout;
316
	};
J
Jens Axboe 已提交
317 318 319 320 321

	struct sqe_submit	submit;

	struct io_ring_ctx	*ctx;
	struct list_head	list;
J
Jens Axboe 已提交
322
	struct list_head	link_list;
J
Jens Axboe 已提交
323
	unsigned int		flags;
J
Jens Axboe 已提交
324
	refcount_t		refs;
325
#define REQ_F_NOWAIT		1	/* must not punt to workers */
J
Jens Axboe 已提交
326
#define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
J
Jens Axboe 已提交
327
#define REQ_F_FIXED_FILE	4	/* ctx owns file */
328
#define REQ_F_SEQ_PREV		8	/* sequential with previous */
329 330
#define REQ_F_IO_DRAIN		16	/* drain existing IO first */
#define REQ_F_IO_DRAINED	32	/* drain done */
J
Jens Axboe 已提交
331
#define REQ_F_LINK		64	/* linked sqes */
332 333
#define REQ_F_LINK_DONE		128	/* linked sqes done */
#define REQ_F_FAIL_LINK		256	/* fail rest of links */
334
#define REQ_F_SHADOW_DRAIN	512	/* link-drain shadow req */
J
Jens Axboe 已提交
335
#define REQ_F_TIMEOUT		1024	/* timeout request */
336 337
#define REQ_F_ISREG		2048	/* regular file */
#define REQ_F_MUST_PUNT		4096	/* must be punted even for NONBLOCK */
338
#define REQ_F_INFLIGHT		8192	/* on inflight list */
J
Jens Axboe 已提交
339
	u64			user_data;
J
Jens Axboe 已提交
340
	u32			result;
341
	u32			sequence;
J
Jens Axboe 已提交
342

343 344
	struct list_head	inflight_entry;

345
	struct io_wq_work	work;
J
Jens Axboe 已提交
346 347 348
};

#define IO_PLUG_THRESHOLD		2
J
Jens Axboe 已提交
349
#define IO_IOPOLL_BATCH			8
J
Jens Axboe 已提交
350

351 352 353
struct io_submit_state {
	struct blk_plug		plug;

J
Jens Axboe 已提交
354 355 356 357 358 359 360
	/*
	 * io_kiocb alloc cache
	 */
	void			*reqs[IO_IOPOLL_BATCH];
	unsigned		int free_reqs;
	unsigned		int cur_req;

361 362 363 364 365 366 367 368 369 370
	/*
	 * File reference cache
	 */
	struct file		*file;
	unsigned int		fd;
	unsigned int		has_refs;
	unsigned int		used_refs;
	unsigned int		ios_left;
};

371
static void io_wq_submit_work(struct io_wq_work **workptr);
J
Jens Axboe 已提交
372 373
static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
				 long res);
374
static void __io_free_req(struct io_kiocb *req);
375

J
Jens Axboe 已提交
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
static struct kmem_cache *req_cachep;

static const struct file_operations io_uring_fops;

struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
	if (file->f_op == &io_uring_fops) {
		struct io_ring_ctx *ctx = file->private_data;

		return ctx->ring_sock->sk;
	}
#endif
	return NULL;
}
EXPORT_SYMBOL(io_uring_get_socket);

static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

	complete(&ctx->ctx_done);
}

static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
	struct io_ring_ctx *ctx;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return NULL;

408 409
	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
			    PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
J
Jens Axboe 已提交
410 411 412 413 414 415 416
		kfree(ctx);
		return NULL;
	}

	ctx->flags = p->flags;
	init_waitqueue_head(&ctx->cq_wait);
	init_completion(&ctx->ctx_done);
417
	init_completion(&ctx->sqo_thread_started);
J
Jens Axboe 已提交
418 419 420
	mutex_init(&ctx->uring_lock);
	init_waitqueue_head(&ctx->wait);
	spin_lock_init(&ctx->completion_lock);
J
Jens Axboe 已提交
421
	INIT_LIST_HEAD(&ctx->poll_list);
422
	INIT_LIST_HEAD(&ctx->cancel_list);
423
	INIT_LIST_HEAD(&ctx->defer_list);
J
Jens Axboe 已提交
424
	INIT_LIST_HEAD(&ctx->timeout_list);
425 426 427
	init_waitqueue_head(&ctx->inflight_wait);
	spin_lock_init(&ctx->inflight_lock);
	INIT_LIST_HEAD(&ctx->inflight_list);
J
Jens Axboe 已提交
428 429 430
	return ctx;
}

431 432 433
static inline bool __io_sequence_defer(struct io_ring_ctx *ctx,
				       struct io_kiocb *req)
{
434 435
	return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
					+ atomic_read(&ctx->cached_cq_overflow);
436 437
}

438 439 440
static inline bool io_sequence_defer(struct io_ring_ctx *ctx,
				     struct io_kiocb *req)
{
441
	if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) != REQ_F_IO_DRAIN)
442 443
		return false;

444
	return __io_sequence_defer(ctx, req);
445 446
}

447
static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
448 449 450
{
	struct io_kiocb *req;

451 452
	req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
	if (req && !io_sequence_defer(ctx, req)) {
453 454 455 456 457 458 459
		list_del_init(&req->list);
		return req;
	}

	return NULL;
}

J
Jens Axboe 已提交
460 461
static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
{
462 463 464 465 466 467 468 469 470
	struct io_kiocb *req;

	req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
	if (req && !__io_sequence_defer(ctx, req)) {
		list_del_init(&req->list);
		return req;
	}

	return NULL;
J
Jens Axboe 已提交
471 472
}

473
static void __io_commit_cqring(struct io_ring_ctx *ctx)
J
Jens Axboe 已提交
474
{
475
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
476

477
	if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) {
J
Jens Axboe 已提交
478
		/* order cqe stores with ring update */
479
		smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
J
Jens Axboe 已提交
480 481 482 483 484 485 486 487

		if (wq_has_sleeper(&ctx->cq_wait)) {
			wake_up_interruptible(&ctx->cq_wait);
			kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
		}
	}
}

488
static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
489
{
490 491 492 493 494 495 496 497 498
	u8 opcode = READ_ONCE(sqe->opcode);

	return !(opcode == IORING_OP_READ_FIXED ||
		 opcode == IORING_OP_WRITE_FIXED);
}

static inline bool io_prep_async_work(struct io_kiocb *req)
{
	bool do_hashed = false;
499

500 501 502 503
	if (req->submit.sqe) {
		switch (req->submit.sqe->opcode) {
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
504
			do_hashed = true;
505 506
			break;
		}
507 508
		if (io_sqe_needs_user(req->submit.sqe))
			req->work.flags |= IO_WQ_WORK_NEEDS_USER;
509 510
	}

511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
	return do_hashed;
}

static inline void io_queue_async_work(struct io_ring_ctx *ctx,
				       struct io_kiocb *req)
{
	bool do_hashed = io_prep_async_work(req);

	trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
					req->flags);
	if (!do_hashed) {
		io_wq_enqueue(ctx->io_wq, &req->work);
	} else {
		io_wq_enqueue_hashed(ctx->io_wq, &req->work,
					file_inode(req->file));
	}
527 528
}

J
Jens Axboe 已提交
529 530 531 532 533 534 535
static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

	ret = hrtimer_try_to_cancel(&req->timeout.timer);
	if (ret != -1) {
		atomic_inc(&req->ctx->cq_timeouts);
536
		list_del_init(&req->list);
J
Jens Axboe 已提交
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
		io_cqring_fill_event(req->ctx, req->user_data, 0);
		__io_free_req(req);
	}
}

static void io_kill_timeouts(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req, *tmp;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
		io_kill_timeout(req);
	spin_unlock_irq(&ctx->completion_lock);
}

552 553 554 555
static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

J
Jens Axboe 已提交
556 557 558
	while ((req = io_get_timeout_req(ctx)) != NULL)
		io_kill_timeout(req);

559 560 561
	__io_commit_cqring(ctx);

	while ((req = io_get_deferred_req(ctx)) != NULL) {
562 563 564 565 566
		if (req->flags & REQ_F_SHADOW_DRAIN) {
			/* Just for drain, free it. */
			__io_free_req(req);
			continue;
		}
567
		req->flags |= REQ_F_IO_DRAINED;
568
		io_queue_async_work(ctx, req);
569 570 571
	}
}

J
Jens Axboe 已提交
572 573
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
574
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
575 576 577
	unsigned tail;

	tail = ctx->cached_cq_tail;
578 579 580 581 582
	/*
	 * writes to the cq entry need to come after reading head; the
	 * control dependency is enough as we're using WRITE_ONCE to
	 * fill the cq entry
	 */
583
	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
J
Jens Axboe 已提交
584 585 586
		return NULL;

	ctx->cached_cq_tail++;
587
	return &rings->cqes[tail & ctx->cq_mask];
J
Jens Axboe 已提交
588 589 590
}

static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
591
				 long res)
J
Jens Axboe 已提交
592 593 594 595 596 597 598 599 600 601 602 603
{
	struct io_uring_cqe *cqe;

	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqring(ctx);
	if (cqe) {
		WRITE_ONCE(cqe->user_data, ki_user_data);
		WRITE_ONCE(cqe->res, res);
604
		WRITE_ONCE(cqe->flags, 0);
J
Jens Axboe 已提交
605
	} else {
606 607
		WRITE_ONCE(ctx->rings->cq_overflow,
				atomic_inc_return(&ctx->cached_cq_overflow));
J
Jens Axboe 已提交
608 609 610
	}
}

J
Jens Axboe 已提交
611 612 613 614 615 616
static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
{
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);
	if (waitqueue_active(&ctx->sqo_wait))
		wake_up(&ctx->sqo_wait);
617 618
	if (ctx->cq_ev_fd)
		eventfd_signal(ctx->cq_ev_fd, 1);
J
Jens Axboe 已提交
619 620 621
}

static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
622
				long res)
J
Jens Axboe 已提交
623 624 625 626
{
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
627
	io_cqring_fill_event(ctx, user_data, res);
J
Jens Axboe 已提交
628 629 630
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

J
Jens Axboe 已提交
631
	io_cqring_ev_posted(ctx);
J
Jens Axboe 已提交
632 633
}

J
Jens Axboe 已提交
634 635
static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
				   struct io_submit_state *state)
J
Jens Axboe 已提交
636
{
637
	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
J
Jens Axboe 已提交
638 639 640 641 642
	struct io_kiocb *req;

	if (!percpu_ref_tryget(&ctx->refs))
		return NULL;

J
Jens Axboe 已提交
643
	if (!state) {
644
		req = kmem_cache_alloc(req_cachep, gfp);
J
Jens Axboe 已提交
645 646 647 648 649 650 651
		if (unlikely(!req))
			goto out;
	} else if (!state->free_reqs) {
		size_t sz;
		int ret;

		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
652 653 654 655 656 657 658 659 660 661 662 663
		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);

		/*
		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
		 * retry single alloc to be on the safe side.
		 */
		if (unlikely(ret <= 0)) {
			state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
			if (!state->reqs[0])
				goto out;
			ret = 1;
		}
J
Jens Axboe 已提交
664 665 666 667 668 669 670
		state->free_reqs = ret - 1;
		state->cur_req = 1;
		req = state->reqs[0];
	} else {
		req = state->reqs[state->cur_req];
		state->free_reqs--;
		state->cur_req++;
J
Jens Axboe 已提交
671 672
	}

673
	req->file = NULL;
J
Jens Axboe 已提交
674 675
	req->ctx = ctx;
	req->flags = 0;
676 677
	/* one is dropped after submission, the other at completion */
	refcount_set(&req->refs, 2);
J
Jens Axboe 已提交
678
	req->result = 0;
679
	INIT_IO_WORK(&req->work, io_wq_submit_work);
J
Jens Axboe 已提交
680 681
	return req;
out:
682
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
683 684 685
	return NULL;
}

J
Jens Axboe 已提交
686 687 688 689
static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
{
	if (*nr) {
		kmem_cache_free_bulk(req_cachep, *nr, reqs);
690
		percpu_ref_put_many(&ctx->refs, *nr);
J
Jens Axboe 已提交
691 692 693 694
		*nr = 0;
	}
}

J
Jens Axboe 已提交
695
static void __io_free_req(struct io_kiocb *req)
J
Jens Axboe 已提交
696
{
697 698
	struct io_ring_ctx *ctx = req->ctx;

J
Jens Axboe 已提交
699 700
	if (req->file && !(req->flags & REQ_F_FIXED_FILE))
		fput(req->file);
701 702 703 704 705 706 707 708 709 710
	if (req->flags & REQ_F_INFLIGHT) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->inflight_lock, flags);
		list_del(&req->inflight_entry);
		if (waitqueue_active(&ctx->inflight_wait))
			wake_up(&ctx->inflight_wait);
		spin_unlock_irqrestore(&ctx->inflight_lock, flags);
	}
	percpu_ref_put(&ctx->refs);
711 712 713
	kmem_cache_free(req_cachep, req);
}

714
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
J
Jens Axboe 已提交
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
{
	struct io_kiocb *nxt;

	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
	if (nxt) {
		list_del(&nxt->list);
		if (!list_empty(&req->link_list)) {
			INIT_LIST_HEAD(&nxt->link_list);
			list_splice(&req->link_list, &nxt->link_list);
			nxt->flags |= REQ_F_LINK;
		}

732
		nxt->flags |= REQ_F_LINK_DONE;
733 734 735 736
		/*
		 * If we're in async work, we can continue processing the chain
		 * in this context instead of having to queue up new async work.
		 */
737
		if (nxtptr && current_work())
738
			*nxtptr = nxt;
739
		else
740
			io_queue_async_work(req->ctx, nxt);
J
Jens Axboe 已提交
741 742 743 744 745 746 747 748 749 750 751 752 753 754
	}
}

/*
 * Called if REQ_F_LINK is set, and we fail the head request
 */
static void io_fail_links(struct io_kiocb *req)
{
	struct io_kiocb *link;

	while (!list_empty(&req->link_list)) {
		link = list_first_entry(&req->link_list, struct io_kiocb, list);
		list_del(&link->list);

755
		trace_io_uring_fail_link(req, link);
J
Jens Axboe 已提交
756 757 758 759 760
		io_cqring_add_event(req->ctx, link->user_data, -ECANCELED);
		__io_free_req(link);
	}
}

761
static void io_free_req(struct io_kiocb *req, struct io_kiocb **nxt)
J
Jens Axboe 已提交
762 763 764 765 766 767 768 769 770 771 772
{
	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
	if (req->flags & REQ_F_LINK) {
		if (req->flags & REQ_F_FAIL_LINK)
			io_fail_links(req);
		else
773
			io_req_link_next(req, nxt);
J
Jens Axboe 已提交
774 775 776 777 778
	}

	__io_free_req(req);
}

779 780 781 782 783
/*
 * Drop reference to request, return next in chain (if there is one) if this
 * was the last reference to this request.
 */
static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
784
{
785 786
	struct io_kiocb *nxt = NULL;

787
	if (refcount_dec_and_test(&req->refs))
788 789 790 791 792 793 794 795 796 797 798
		io_free_req(req, &nxt);

	return nxt;
}

static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr)
{
	struct io_kiocb *nxt;

	nxt = io_put_req_find_next(req);
	if (nxt) {
799
		if (nxtptr)
800
			*nxtptr = nxt;
801
		else
802 803
			io_queue_async_work(nxt->ctx, nxt);
	}
J
Jens Axboe 已提交
804 805
}

806
static unsigned io_cqring_events(struct io_rings *rings)
807 808 809
{
	/* See comment at the top of this file */
	smp_rmb();
810
	return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
811 812
}

813 814 815 816 817 818 819 820
static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;

	/* make sure SQ entry isn't read before tail */
	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}

J
Jens Axboe 已提交
821 822 823 824 825 826 827 828
/*
 * Find and free completed poll iocbs
 */
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
			       struct list_head *done)
{
	void *reqs[IO_IOPOLL_BATCH];
	struct io_kiocb *req;
J
Jens Axboe 已提交
829
	int to_free;
J
Jens Axboe 已提交
830

J
Jens Axboe 已提交
831
	to_free = 0;
J
Jens Axboe 已提交
832 833 834 835
	while (!list_empty(done)) {
		req = list_first_entry(done, struct io_kiocb, list);
		list_del(&req->list);

J
Jens Axboe 已提交
836
		io_cqring_fill_event(ctx, req->user_data, req->result);
J
Jens Axboe 已提交
837 838
		(*nr_events)++;

J
Jens Axboe 已提交
839 840 841 842
		if (refcount_dec_and_test(&req->refs)) {
			/* If we're not using fixed files, we have to pair the
			 * completion part with the file put. Use regular
			 * completions for those, only batch free for fixed
J
Jens Axboe 已提交
843
			 * file and non-linked commands.
J
Jens Axboe 已提交
844
			 */
J
Jens Axboe 已提交
845 846
			if ((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
			    REQ_F_FIXED_FILE) {
J
Jens Axboe 已提交
847 848 849
				reqs[to_free++] = req;
				if (to_free == ARRAY_SIZE(reqs))
					io_free_req_many(ctx, reqs, &to_free);
J
Jens Axboe 已提交
850
			} else {
851
				io_free_req(req, NULL);
J
Jens Axboe 已提交
852
			}
853
		}
J
Jens Axboe 已提交
854 855
	}

J
Jens Axboe 已提交
856
	io_commit_cqring(ctx);
J
Jens Axboe 已提交
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
	io_free_req_many(ctx, reqs, &to_free);
}

static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
			long min)
{
	struct io_kiocb *req, *tmp;
	LIST_HEAD(done);
	bool spin;
	int ret;

	/*
	 * Only spin for completions if we don't have multiple devices hanging
	 * off our complete list, and we're under the requested amount.
	 */
	spin = !ctx->poll_multi_file && *nr_events < min;

	ret = 0;
	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
		struct kiocb *kiocb = &req->rw;

		/*
		 * Move completed entries to our local list. If we find a
		 * request that requires polling, break out and complete
		 * the done list first, if we have entries there.
		 */
		if (req->flags & REQ_F_IOPOLL_COMPLETED) {
			list_move_tail(&req->list, &done);
			continue;
		}
		if (!list_empty(&done))
			break;

		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
		if (ret < 0)
			break;

		if (ret && spin)
			spin = false;
		ret = 0;
	}

	if (!list_empty(&done))
		io_iopoll_complete(ctx, nr_events, &done);

	return ret;
}

/*
 * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
 * non-spinning poll check - we'll still enter the driver poll loop, but only
 * as a non-spinning completion check.
 */
static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
				long min)
{
913
	while (!list_empty(&ctx->poll_list) && !need_resched()) {
J
Jens Axboe 已提交
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;
}

/*
 * We can't just wait for polled events to come to us, we have to actively
 * find and complete them.
 */
static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
{
	if (!(ctx->flags & IORING_SETUP_IOPOLL))
		return;

	mutex_lock(&ctx->uring_lock);
	while (!list_empty(&ctx->poll_list)) {
		unsigned int nr_events = 0;

		io_iopoll_getevents(ctx, &nr_events, 1);
940 941 942 943 944 945

		/*
		 * Ensure we allow local-to-the-cpu processing to take place,
		 * in this case we need to ensure that we reap all events.
		 */
		cond_resched();
J
Jens Axboe 已提交
946 947 948 949
	}
	mutex_unlock(&ctx->uring_lock);
}

950 951
static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			    long min)
J
Jens Axboe 已提交
952
{
953
	int iters = 0, ret = 0;
954

J
Jens Axboe 已提交
955 956 957
	do {
		int tmin = 0;

958 959 960 961 962
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
963
		if (io_cqring_events(ctx->rings))
964 965
			break;

966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
			mutex_lock(&ctx->uring_lock);
		}

J
Jens Axboe 已提交
981 982 983 984 985 986 987 988 989
		if (*nr_events < min)
			tmin = min - *nr_events;

		ret = io_iopoll_getevents(ctx, nr_events, tmin);
		if (ret <= 0)
			break;
		ret = 0;
	} while (min && !*nr_events && !need_resched());

990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
	return ret;
}

static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
			   long min)
{
	int ret;

	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
	ret = __io_iopoll_check(ctx, nr_events, min);
1005
	mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
1006 1007 1008
	return ret;
}

1009
static void kiocb_end_write(struct io_kiocb *req)
J
Jens Axboe 已提交
1010
{
1011 1012 1013 1014 1015 1016
	/*
	 * Tell lockdep we inherited freeze protection from submission
	 * thread.
	 */
	if (req->flags & REQ_F_ISREG) {
		struct inode *inode = file_inode(req->file);
J
Jens Axboe 已提交
1017

1018
		__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
J
Jens Axboe 已提交
1019
	}
1020
	file_end_write(req->file);
J
Jens Axboe 已提交
1021 1022
}

1023
static void io_complete_rw_common(struct kiocb *kiocb, long res)
J
Jens Axboe 已提交
1024 1025 1026
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);

1027 1028
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1029

J
Jens Axboe 已提交
1030 1031
	if ((req->flags & REQ_F_LINK) && res != req->result)
		req->flags |= REQ_F_FAIL_LINK;
1032
	io_cqring_add_event(req->ctx, req->user_data, res);
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
}

static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);

	io_complete_rw_common(kiocb, res);
	io_put_req(req, NULL);
}

static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);

	io_complete_rw_common(kiocb, res);
	return io_put_req_find_next(req);
J
Jens Axboe 已提交
1049 1050
}

J
Jens Axboe 已提交
1051 1052 1053 1054
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);

1055 1056
	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
J
Jens Axboe 已提交
1057

J
Jens Axboe 已提交
1058 1059 1060
	if ((req->flags & REQ_F_LINK) && res != req->result)
		req->flags |= REQ_F_FAIL_LINK;
	req->result = res;
J
Jens Axboe 已提交
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	if (res != -EAGAIN)
		req->flags |= REQ_F_IOPOLL_COMPLETED;
}

/*
 * After the iocb has been issued, it's safe to be found on the poll list.
 * Adding the kiocb to the list AFTER submission ensures that we don't
 * find it from a io_iopoll_getevents() thread before the issuer is done
 * accessing the kiocb cookie.
 */
static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->poll_list)) {
		ctx->poll_multi_file = false;
	} else if (!ctx->poll_multi_file) {
		struct io_kiocb *list_req;

		list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
						list);
		if (list_req->rw.ki_filp != req->rw.ki_filp)
			ctx->poll_multi_file = true;
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
	if (req->flags & REQ_F_IOPOLL_COMPLETED)
		list_add(&req->list, &ctx->poll_list);
	else
		list_add_tail(&req->list, &ctx->poll_list);
}

1101
static void io_file_put(struct io_submit_state *state)
1102
{
1103
	if (state->file) {
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
		int diff = state->has_refs - state->used_refs;

		if (diff)
			fput_many(state->file, diff);
		state->file = NULL;
	}
}

/*
 * Get as many references to a file as we have IOs left in this submission,
 * assuming most submissions are for one file, or at least that each file
 * has more than one submission.
 */
static struct file *io_file_get(struct io_submit_state *state, int fd)
{
	if (!state)
		return fget(fd);

	if (state->file) {
		if (state->fd == fd) {
			state->used_refs++;
			state->ios_left--;
			return state->file;
		}
1128
		io_file_put(state);
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
	}
	state->file = fget_many(fd, state->ios_left);
	if (!state->file)
		return NULL;

	state->fd = fd;
	state->has_refs = state->ios_left;
	state->used_refs = 1;
	state->ios_left--;
	return state->file;
}

J
Jens Axboe 已提交
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
/*
 * If we tracked the file through the SCM inflight mechanism, we could support
 * any file. For now, just ensure that anything potentially problematic is done
 * inline.
 */
static bool io_file_supports_async(struct file *file)
{
	umode_t mode = file_inode(file)->i_mode;

	if (S_ISBLK(mode) || S_ISCHR(mode))
		return true;
	if (S_ISREG(mode) && file->f_op != &io_uring_fops)
		return true;

	return false;
}

J
Jens Axboe 已提交
1158
static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s,
1159
		      bool force_nonblock)
J
Jens Axboe 已提交
1160
{
J
Jens Axboe 已提交
1161
	const struct io_uring_sqe *sqe = s->sqe;
J
Jens Axboe 已提交
1162
	struct io_ring_ctx *ctx = req->ctx;
J
Jens Axboe 已提交
1163
	struct kiocb *kiocb = &req->rw;
J
Jens Axboe 已提交
1164 1165
	unsigned ioprio;
	int ret;
J
Jens Axboe 已提交
1166

J
Jens Axboe 已提交
1167 1168
	if (!req->file)
		return -EBADF;
J
Jens Axboe 已提交
1169

1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	if (S_ISREG(file_inode(req->file)->i_mode))
		req->flags |= REQ_F_ISREG;

	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
	if (force_nonblock && !io_file_supports_async(req->file)) {
		req->flags |= REQ_F_MUST_PUNT;
		return -EAGAIN;
	}
J
Jens Axboe 已提交
1181

J
Jens Axboe 已提交
1182 1183 1184 1185 1186 1187 1188 1189
	kiocb->ki_pos = READ_ONCE(sqe->off);
	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
J
Jens Axboe 已提交
1190
			return ret;
J
Jens Axboe 已提交
1191 1192 1193 1194 1195 1196 1197

		kiocb->ki_ioprio = ioprio;
	} else
		kiocb->ki_ioprio = get_current_ioprio();

	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
	if (unlikely(ret))
J
Jens Axboe 已提交
1198
		return ret;
1199 1200

	/* don't allow async punt if RWF_NOWAIT was requested */
1201 1202
	if ((kiocb->ki_flags & IOCB_NOWAIT) ||
	    (req->file->f_flags & O_NONBLOCK))
1203 1204 1205
		req->flags |= REQ_F_NOWAIT;

	if (force_nonblock)
J
Jens Axboe 已提交
1206
		kiocb->ki_flags |= IOCB_NOWAIT;
1207

J
Jens Axboe 已提交
1208 1209 1210
	if (ctx->flags & IORING_SETUP_IOPOLL) {
		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
		    !kiocb->ki_filp->f_op->iopoll)
J
Jens Axboe 已提交
1211
			return -EOPNOTSUPP;
J
Jens Axboe 已提交
1212

J
Jens Axboe 已提交
1213 1214 1215
		kiocb->ki_flags |= IOCB_HIPRI;
		kiocb->ki_complete = io_complete_rw_iopoll;
	} else {
J
Jens Axboe 已提交
1216 1217
		if (kiocb->ki_flags & IOCB_HIPRI)
			return -EINVAL;
J
Jens Axboe 已提交
1218 1219
		kiocb->ki_complete = io_complete_rw;
	}
J
Jens Axboe 已提交
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
	return 0;
}

static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		/* fall through */
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}

1244 1245 1246 1247 1248 1249 1250 1251 1252
static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
		       bool in_async)
{
	if (in_async && ret >= 0 && nxt && kiocb->ki_complete == io_complete_rw)
		*nxt = __io_complete_rw(kiocb, ret);
	else
		io_rw_done(kiocb, ret);
}

1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
			   const struct io_uring_sqe *sqe,
			   struct iov_iter *iter)
{
	size_t len = READ_ONCE(sqe->len);
	struct io_mapped_ubuf *imu;
	unsigned index, buf_index;
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

	buf_index = READ_ONCE(sqe->buf_index);
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
	buf_addr = READ_ONCE(sqe->addr);

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
1319
			iter->count -= bvec->bv_len + offset;
1320 1321 1322 1323
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

1324 1325 1326
	return 0;
}

1327 1328 1329
static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
			       const struct sqe_submit *s, struct iovec **iovec,
			       struct iov_iter *iter)
J
Jens Axboe 已提交
1330 1331 1332 1333
{
	const struct io_uring_sqe *sqe = s->sqe;
	void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
	size_t sqe_len = READ_ONCE(sqe->len);
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
	u8 opcode;

	/*
	 * We're reading ->opcode for the second time, but the first read
	 * doesn't care whether it's _FIXED or not, so it doesn't matter
	 * whether ->opcode changes concurrently. The first read does care
	 * about whether it is a READ or a WRITE, so we don't trust this read
	 * for that purpose and instead let the caller pass in the read/write
	 * flag.
	 */
	opcode = READ_ONCE(sqe->opcode);
	if (opcode == IORING_OP_READ_FIXED ||
	    opcode == IORING_OP_WRITE_FIXED) {
1347
		ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
1348 1349 1350
		*iovec = NULL;
		return ret;
	}
J
Jens Axboe 已提交
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363

	if (!s->has_user)
		return -EFAULT;

#ifdef CONFIG_COMPAT
	if (ctx->compat)
		return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
						iovec, iter);
#endif

	return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}

1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
/*
 * For files that don't have ->read_iter() and ->write_iter(), handle them
 * by looping over ->read() or ->write() manually.
 */
static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
		struct iovec iovec = iov_iter_iovec(iter);
		ssize_t nr;

		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, &kiocb->ki_pos);
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, &kiocb->ki_pos);
		}

		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}

1409
static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
1410
		   struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
1411 1412 1413 1414 1415
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw;
	struct iov_iter iter;
	struct file *file;
1416
	size_t iov_count;
1417
	ssize_t read_size, ret;
J
Jens Axboe 已提交
1418

1419
	ret = io_prep_rw(req, s, force_nonblock);
J
Jens Axboe 已提交
1420 1421 1422 1423 1424
	if (ret)
		return ret;
	file = kiocb->ki_filp;

	if (unlikely(!(file->f_mode & FMODE_READ)))
J
Jens Axboe 已提交
1425
		return -EBADF;
J
Jens Axboe 已提交
1426 1427

	ret = io_import_iovec(req->ctx, READ, s, &iovec, &iter);
1428
	if (ret < 0)
J
Jens Axboe 已提交
1429
		return ret;
J
Jens Axboe 已提交
1430

1431
	read_size = ret;
J
Jens Axboe 已提交
1432 1433 1434
	if (req->flags & REQ_F_LINK)
		req->result = read_size;

1435 1436
	iov_count = iov_iter_count(&iter);
	ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
1437 1438 1439
	if (!ret) {
		ssize_t ret2;

1440 1441 1442 1443 1444
		if (file->f_op->read_iter)
			ret2 = call_read_iter(file, kiocb, &iter);
		else
			ret2 = loop_rw_iter(READ, file, kiocb, &iter);

1445 1446 1447 1448 1449 1450 1451 1452
		/*
		 * In case of a short read, punt to async. This can happen
		 * if we have data partially cached. Alternatively we can
		 * return the short read, in which case the application will
		 * need to issue another SQE and wait for it. That SQE will
		 * need async punt anyway, so it's more efficient to do it
		 * here.
		 */
1453 1454 1455
		if (force_nonblock && !(req->flags & REQ_F_NOWAIT) &&
		    (req->flags & REQ_F_ISREG) &&
		    ret2 > 0 && ret2 < read_size)
1456 1457
			ret2 = -EAGAIN;
		/* Catch -EAGAIN return for forced non-blocking submission */
1458
		if (!force_nonblock || ret2 != -EAGAIN)
1459
			kiocb_done(kiocb, ret2, nxt, s->in_async);
1460
		else
J
Jens Axboe 已提交
1461 1462 1463 1464 1465 1466
			ret = -EAGAIN;
	}
	kfree(iovec);
	return ret;
}

1467
static int io_write(struct io_kiocb *req, const struct sqe_submit *s,
1468
		    struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
1469 1470 1471 1472 1473
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw;
	struct iov_iter iter;
	struct file *file;
1474
	size_t iov_count;
1475
	ssize_t ret;
J
Jens Axboe 已提交
1476

1477
	ret = io_prep_rw(req, s, force_nonblock);
J
Jens Axboe 已提交
1478 1479 1480 1481 1482
	if (ret)
		return ret;

	file = kiocb->ki_filp;
	if (unlikely(!(file->f_mode & FMODE_WRITE)))
J
Jens Axboe 已提交
1483
		return -EBADF;
J
Jens Axboe 已提交
1484 1485

	ret = io_import_iovec(req->ctx, WRITE, s, &iovec, &iter);
1486
	if (ret < 0)
J
Jens Axboe 已提交
1487
		return ret;
J
Jens Axboe 已提交
1488

J
Jens Axboe 已提交
1489 1490 1491
	if (req->flags & REQ_F_LINK)
		req->result = ret;

1492 1493 1494
	iov_count = iov_iter_count(&iter);

	ret = -EAGAIN;
1495
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
1496 1497 1498
		goto out_free;

	ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
J
Jens Axboe 已提交
1499
	if (!ret) {
1500 1501
		ssize_t ret2;

J
Jens Axboe 已提交
1502 1503 1504 1505 1506 1507 1508
		/*
		 * Open-code file_start_write here to grab freeze protection,
		 * which will be released by another thread in
		 * io_complete_rw().  Fool lockdep by telling it the lock got
		 * released so that it doesn't complain about the held lock when
		 * we return to userspace.
		 */
1509
		if (req->flags & REQ_F_ISREG) {
J
Jens Axboe 已提交
1510 1511 1512 1513 1514 1515
			__sb_start_write(file_inode(file)->i_sb,
						SB_FREEZE_WRITE, true);
			__sb_writers_release(file_inode(file)->i_sb,
						SB_FREEZE_WRITE);
		}
		kiocb->ki_flags |= IOCB_WRITE;
1516

1517 1518 1519 1520
		if (file->f_op->write_iter)
			ret2 = call_write_iter(file, kiocb, &iter);
		else
			ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
1521
		if (!force_nonblock || ret2 != -EAGAIN)
1522
			kiocb_done(kiocb, ret2, nxt, s->in_async);
1523
		else
1524
			ret = -EAGAIN;
J
Jens Axboe 已提交
1525
	}
1526
out_free:
J
Jens Axboe 已提交
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
	kfree(iovec);
	return ret;
}

/*
 * IORING_OP_NOP just posts a completion event, nothing else.
 */
static int io_nop(struct io_kiocb *req, u64 user_data)
{
	struct io_ring_ctx *ctx = req->ctx;
	long err = 0;

J
Jens Axboe 已提交
1539 1540 1541
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

1542
	io_cqring_add_event(ctx, user_data, err);
1543
	io_put_req(req, NULL);
J
Jens Axboe 已提交
1544 1545 1546
	return 0;
}

C
Christoph Hellwig 已提交
1547 1548
static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
J
Jens Axboe 已提交
1549
	struct io_ring_ctx *ctx = req->ctx;
C
Christoph Hellwig 已提交
1550

J
Jens Axboe 已提交
1551 1552
	if (!req->file)
		return -EBADF;
C
Christoph Hellwig 已提交
1553

J
Jens Axboe 已提交
1554
	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
J
Jens Axboe 已提交
1555
		return -EINVAL;
1556
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
C
Christoph Hellwig 已提交
1557 1558 1559 1560 1561 1562
		return -EINVAL;

	return 0;
}

static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1563
		    struct io_kiocb **nxt, bool force_nonblock)
C
Christoph Hellwig 已提交
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
{
	loff_t sqe_off = READ_ONCE(sqe->off);
	loff_t sqe_len = READ_ONCE(sqe->len);
	loff_t end = sqe_off + sqe_len;
	unsigned fsync_flags;
	int ret;

	fsync_flags = READ_ONCE(sqe->fsync_flags);
	if (unlikely(fsync_flags & ~IORING_FSYNC_DATASYNC))
		return -EINVAL;

	ret = io_prep_fsync(req, sqe);
	if (ret)
		return ret;

	/* fsync always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

	ret = vfs_fsync_range(req->rw.ki_filp, sqe_off,
				end > 0 ? end : LLONG_MAX,
				fsync_flags & IORING_FSYNC_DATASYNC);

J
Jens Axboe 已提交
1587 1588
	if (ret < 0 && (req->flags & REQ_F_LINK))
		req->flags |= REQ_F_FAIL_LINK;
1589
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
1590
	io_put_req(req, nxt);
C
Christoph Hellwig 已提交
1591 1592 1593
	return 0;
}

1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_ring_ctx *ctx = req->ctx;
	int ret = 0;

	if (!req->file)
		return -EBADF;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
		return -EINVAL;

	return ret;
}

static int io_sync_file_range(struct io_kiocb *req,
			      const struct io_uring_sqe *sqe,
1612
			      struct io_kiocb **nxt,
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
			      bool force_nonblock)
{
	loff_t sqe_off;
	loff_t sqe_len;
	unsigned flags;
	int ret;

	ret = io_prep_sfr(req, sqe);
	if (ret)
		return ret;

	/* sync_file_range always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

	sqe_off = READ_ONCE(sqe->off);
	sqe_len = READ_ONCE(sqe->len);
	flags = READ_ONCE(sqe->sync_range_flags);

	ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);

J
Jens Axboe 已提交
1634 1635
	if (ret < 0 && (req->flags & REQ_F_LINK))
		req->flags |= REQ_F_FAIL_LINK;
1636
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
1637
	io_put_req(req, nxt);
1638 1639 1640
	return 0;
}

J
Jens Axboe 已提交
1641
#if defined(CONFIG_NET)
J
Jens Axboe 已提交
1642
static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1643
			   struct io_kiocb **nxt, bool force_nonblock,
J
Jens Axboe 已提交
1644 1645 1646
		   long (*fn)(struct socket *, struct user_msghdr __user *,
				unsigned int))
{
J
Jens Axboe 已提交
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct user_msghdr __user *msg;
		unsigned flags;

		flags = READ_ONCE(sqe->msg_flags);
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

		msg = (struct user_msghdr __user *) (unsigned long)
			READ_ONCE(sqe->addr);

J
Jens Axboe 已提交
1667
		ret = fn(sock, msg, flags);
J
Jens Axboe 已提交
1668 1669 1670 1671
		if (force_nonblock && ret == -EAGAIN)
			return ret;
	}

1672
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
1673
	io_put_req(req, nxt);
1674 1675
	return 0;
}
J
Jens Axboe 已提交
1676 1677 1678
#endif

static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1679
		      struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
1680 1681
{
#if defined(CONFIG_NET)
1682 1683
	return io_send_recvmsg(req, sqe, nxt, force_nonblock,
				__sys_sendmsg_sock);
J
Jens Axboe 已提交
1684 1685 1686 1687 1688 1689
#else
	return -EOPNOTSUPP;
#endif
}

static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1690
		      struct io_kiocb **nxt, bool force_nonblock)
J
Jens Axboe 已提交
1691 1692
{
#if defined(CONFIG_NET)
1693 1694
	return io_send_recvmsg(req, sqe, nxt, force_nonblock,
				__sys_recvmsg_sock);
J
Jens Axboe 已提交
1695 1696 1697 1698
#else
	return -EOPNOTSUPP;
#endif
}
1699

1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
static int io_accept(struct io_kiocb *req, const struct io_uring_sqe *sqe,
		     struct io_kiocb **nxt, bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct sockaddr __user *addr;
	int __user *addr_len;
	unsigned file_flags;
	int flags, ret;

	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;

	addr = (struct sockaddr __user *) (unsigned long) READ_ONCE(sqe->addr);
	addr_len = (int __user *) (unsigned long) READ_ONCE(sqe->addr2);
	flags = READ_ONCE(sqe->accept_flags);
	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_accept4_file(req->file, file_flags, addr, addr_len, flags);
	if (ret == -EAGAIN && force_nonblock) {
		req->work.flags |= IO_WQ_WORK_NEEDS_FILES;
		return -EAGAIN;
	}
	if (ret < 0 && (req->flags & REQ_F_LINK))
		req->flags |= REQ_F_FAIL_LINK;
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
	io_put_req(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}

1734 1735 1736 1737 1738 1739 1740 1741
static void io_poll_remove_one(struct io_kiocb *req)
{
	struct io_poll_iocb *poll = &req->poll;

	spin_lock(&poll->head->lock);
	WRITE_ONCE(poll->canceled, true);
	if (!list_empty(&poll->wait.entry)) {
		list_del_init(&poll->wait.entry);
1742
		io_queue_async_work(req->ctx, req);
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
	}
	spin_unlock(&poll->head->lock);

	list_del_init(&req->list);
}

static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	spin_lock_irq(&ctx->completion_lock);
	while (!list_empty(&ctx->cancel_list)) {
		req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
		io_poll_remove_one(req);
	}
	spin_unlock_irq(&ctx->completion_lock);
}

/*
 * Find a running poll command that matches one specified in sqe->addr,
 * and remove it if found.
 */
static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *poll_req, *next;
	int ret = -ENOENT;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->poll_events)
		return -EINVAL;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
		if (READ_ONCE(sqe->addr) == poll_req->user_data) {
			io_poll_remove_one(poll_req);
			ret = 0;
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);

1787
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
1788
	io_put_req(req, NULL);
1789 1790 1791
	return 0;
}

J
Jens Axboe 已提交
1792 1793
static void io_poll_complete(struct io_ring_ctx *ctx, struct io_kiocb *req,
			     __poll_t mask)
1794
{
J
Jens Axboe 已提交
1795
	req->poll.done = true;
1796
	io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask));
J
Jens Axboe 已提交
1797
	io_commit_cqring(ctx);
1798 1799
}

1800
static void io_poll_complete_work(struct io_wq_work **workptr)
1801
{
1802
	struct io_wq_work *work = *workptr;
1803 1804 1805 1806 1807 1808
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_poll_iocb *poll = &req->poll;
	struct poll_table_struct pt = { ._key = poll->events };
	struct io_ring_ctx *ctx = req->ctx;
	__poll_t mask = 0;

1809 1810 1811
	if (work->flags & IO_WQ_WORK_CANCEL)
		WRITE_ONCE(poll->canceled, true);

1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828
	if (!READ_ONCE(poll->canceled))
		mask = vfs_poll(poll->file, &pt) & poll->events;

	/*
	 * Note that ->ki_cancel callers also delete iocb from active_reqs after
	 * calling ->ki_cancel.  We need the ctx_lock roundtrip here to
	 * synchronize with them.  In the cancellation case the list_del_init
	 * itself is not actually needed, but harmless so we keep it in to
	 * avoid further branches in the fast path.
	 */
	spin_lock_irq(&ctx->completion_lock);
	if (!mask && !READ_ONCE(poll->canceled)) {
		add_wait_queue(poll->head, &poll->wait);
		spin_unlock_irq(&ctx->completion_lock);
		return;
	}
	list_del_init(&req->list);
J
Jens Axboe 已提交
1829
	io_poll_complete(ctx, req, mask);
1830 1831
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
1832
	io_cqring_ev_posted(ctx);
1833
	io_put_req(req, NULL);
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
}

static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
			void *key)
{
	struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
							wait);
	struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
	struct io_ring_ctx *ctx = req->ctx;
	__poll_t mask = key_to_poll(key);
J
Jens Axboe 已提交
1844
	unsigned long flags;
1845 1846

	/* for instances that support it check for an event match first: */
J
Jens Axboe 已提交
1847 1848
	if (mask && !(mask & poll->events))
		return 0;
1849

J
Jens Axboe 已提交
1850
	list_del_init(&poll->wait.entry);
1851

J
Jens Axboe 已提交
1852 1853 1854 1855
	if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
		list_del(&req->list);
		io_poll_complete(ctx, req, mask);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
1856

J
Jens Axboe 已提交
1857
		io_cqring_ev_posted(ctx);
1858
		io_put_req(req, NULL);
J
Jens Axboe 已提交
1859
	} else {
1860
		io_queue_async_work(ctx, req);
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
	}

	return 1;
}

struct io_poll_table {
	struct poll_table_struct pt;
	struct io_kiocb *req;
	int error;
};

static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
			       struct poll_table_struct *p)
{
	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);

	if (unlikely(pt->req->poll.head)) {
		pt->error = -EINVAL;
		return;
	}

	pt->error = 0;
	pt->req->poll.head = head;
	add_wait_queue(head, &pt->req->poll.wait);
}

static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
J
Jens Axboe 已提交
1892
	bool cancel = false;
1893 1894 1895 1896 1897 1898 1899
	__poll_t mask;
	u16 events;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
		return -EINVAL;
J
Jens Axboe 已提交
1900 1901
	if (!poll->file)
		return -EBADF;
1902

1903
	req->submit.sqe = NULL;
1904
	INIT_IO_WORK(&req->work, io_poll_complete_work);
1905 1906 1907 1908
	events = READ_ONCE(sqe->poll_events);
	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;

	poll->head = NULL;
J
Jens Axboe 已提交
1909
	poll->done = false;
1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	poll->canceled = false;

	ipt.pt._qproc = io_poll_queue_proc;
	ipt.pt._key = poll->events;
	ipt.req = req;
	ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */

	/* initialized the list so that we can do list_empty checks */
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, io_poll_wake);

1921 1922
	INIT_LIST_HEAD(&req->list);

1923 1924 1925
	mask = vfs_poll(poll->file, &ipt.pt) & poll->events;

	spin_lock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
	if (likely(poll->head)) {
		spin_lock(&poll->head->lock);
		if (unlikely(list_empty(&poll->wait.entry))) {
			if (ipt.error)
				cancel = true;
			ipt.error = 0;
			mask = 0;
		}
		if (mask || ipt.error)
			list_del_init(&poll->wait.entry);
		else if (cancel)
			WRITE_ONCE(poll->canceled, true);
		else if (!poll->done) /* actually waiting for an event */
			list_add_tail(&req->list, &ctx->cancel_list);
		spin_unlock(&poll->head->lock);
	}
	if (mask) { /* no async, we'd stolen it */
1943
		ipt.error = 0;
J
Jens Axboe 已提交
1944
		io_poll_complete(ctx, req, mask);
1945 1946 1947
	}
	spin_unlock_irq(&ctx->completion_lock);

J
Jens Axboe 已提交
1948 1949
	if (mask) {
		io_cqring_ev_posted(ctx);
1950
		io_put_req(req, NULL);
1951
	}
J
Jens Axboe 已提交
1952
	return ipt.error;
1953 1954
}

J
Jens Axboe 已提交
1955 1956 1957
static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
	struct io_ring_ctx *ctx;
1958
	struct io_kiocb *req;
J
Jens Axboe 已提交
1959 1960 1961 1962 1963 1964 1965
	unsigned long flags;

	req = container_of(timer, struct io_kiocb, timeout.timer);
	ctx = req->ctx;
	atomic_inc(&ctx->cq_timeouts);

	spin_lock_irqsave(&ctx->completion_lock, flags);
1966
	/*
1967 1968
	 * We could be racing with timeout deletion. If the list is empty,
	 * then timeout lookup already found it and will be handling it.
1969
	 */
1970
	if (!list_empty(&req->list)) {
1971
		struct io_kiocb *prev;
J
Jens Axboe 已提交
1972

1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
		/*
		 * Adjust the reqs sequence before the current one because it
		 * will consume a slot in the cq_ring and the the cq_tail
		 * pointer will be increased, otherwise other timeout reqs may
		 * return in advance without waiting for enough wait_nr.
		 */
		prev = req;
		list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
			prev->sequence++;
		list_del_init(&req->list);
	}
1984 1985 1986

	io_cqring_fill_event(ctx, req->user_data, -ETIME);
	io_commit_cqring(ctx);
J
Jens Axboe 已提交
1987 1988
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

1989 1990
	io_cqring_ev_posted(ctx);
	io_put_req(req, NULL);
1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044
	return HRTIMER_NORESTART;
}

/*
 * Remove or update an existing timeout command
 */
static int io_timeout_remove(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *treq;
	int ret = -ENOENT;
	__u64 user_data;
	unsigned flags;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
		return -EINVAL;
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags)
		return -EINVAL;

	user_data = READ_ONCE(sqe->addr);
	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry(treq, &ctx->timeout_list, list) {
		if (user_data == treq->user_data) {
			list_del_init(&treq->list);
			ret = 0;
			break;
		}
	}

	/* didn't find timeout */
	if (ret) {
fill_ev:
		io_cqring_fill_event(ctx, req->user_data, ret);
		io_commit_cqring(ctx);
		spin_unlock_irq(&ctx->completion_lock);
		io_cqring_ev_posted(ctx);
		io_put_req(req, NULL);
		return 0;
	}

	ret = hrtimer_try_to_cancel(&treq->timeout.timer);
	if (ret == -1) {
		ret = -EBUSY;
		goto fill_ev;
	}

	io_cqring_fill_event(ctx, req->user_data, 0);
	io_cqring_fill_event(ctx, treq->user_data, -ECANCELED);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
2045 2046
	io_cqring_ev_posted(ctx);

2047
	io_put_req(treq, NULL);
2048
	io_put_req(req, NULL);
2049
	return 0;
J
Jens Axboe 已提交
2050 2051 2052 2053
}

static int io_timeout(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
2054
	unsigned count;
J
Jens Axboe 已提交
2055 2056
	struct io_ring_ctx *ctx = req->ctx;
	struct list_head *entry;
2057
	enum hrtimer_mode mode;
2058
	struct timespec64 ts;
2059
	unsigned span = 0;
2060
	unsigned flags;
J
Jens Axboe 已提交
2061 2062 2063

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
2064 2065 2066 2067
	if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len != 1)
		return -EINVAL;
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
J
Jens Axboe 已提交
2068
		return -EINVAL;
2069 2070

	if (get_timespec64(&ts, u64_to_user_ptr(sqe->addr)))
J
Jens Axboe 已提交
2071 2072
		return -EFAULT;

2073 2074 2075 2076 2077 2078 2079
	if (flags & IORING_TIMEOUT_ABS)
		mode = HRTIMER_MODE_ABS;
	else
		mode = HRTIMER_MODE_REL;

	hrtimer_init(&req->timeout.timer, CLOCK_MONOTONIC, mode);

J
Jens Axboe 已提交
2080 2081 2082 2083 2084 2085 2086 2087 2088
	/*
	 * sqe->off holds how many events that need to occur for this
	 * timeout event to be satisfied.
	 */
	count = READ_ONCE(sqe->off);
	if (!count)
		count = 1;

	req->sequence = ctx->cached_sq_head + count - 1;
2089 2090
	/* reuse it to store the count */
	req->submit.sequence = count;
J
Jens Axboe 已提交
2091 2092 2093 2094 2095 2096 2097 2098 2099
	req->flags |= REQ_F_TIMEOUT;

	/*
	 * Insertion sort, ensuring the first entry in the list is always
	 * the one we need first.
	 */
	spin_lock_irq(&ctx->completion_lock);
	list_for_each_prev(entry, &ctx->timeout_list) {
		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
2100 2101
		unsigned nxt_sq_head;
		long long tmp, tmp_nxt;
J
Jens Axboe 已提交
2102

2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
		/*
		 * Since cached_sq_head + count - 1 can overflow, use type long
		 * long to store it.
		 */
		tmp = (long long)ctx->cached_sq_head + count - 1;
		nxt_sq_head = nxt->sequence - nxt->submit.sequence + 1;
		tmp_nxt = (long long)nxt_sq_head + nxt->submit.sequence - 1;

		/*
		 * cached_sq_head may overflow, and it will never overflow twice
		 * once there is some timeout req still be valid.
		 */
		if (ctx->cached_sq_head < nxt_sq_head)
2116
			tmp += UINT_MAX;
2117

2118
		if (tmp > tmp_nxt)
J
Jens Axboe 已提交
2119
			break;
2120 2121 2122 2123 2124 2125 2126

		/*
		 * Sequence of reqs after the insert one and itself should
		 * be adjusted because each timeout req consumes a slot.
		 */
		span++;
		nxt->sequence++;
J
Jens Axboe 已提交
2127
	}
2128
	req->sequence -= span;
J
Jens Axboe 已提交
2129 2130
	list_add(&req->list, entry);
	req->timeout.timer.function = io_timeout_fn;
2131
	hrtimer_start(&req->timeout.timer, timespec64_to_ktime(ts), mode);
2132
	spin_unlock_irq(&ctx->completion_lock);
J
Jens Axboe 已提交
2133 2134 2135
	return 0;
}

2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	return req->user_data == (unsigned long) data;
}

static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			   struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;
	enum io_wq_cancel cancel_ret;
	void *sqe_addr;
	int ret = 0;

	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
	    sqe->cancel_flags)
		return -EINVAL;

	sqe_addr = (void *) (unsigned long) READ_ONCE(sqe->addr);
	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
	switch (cancel_ret) {
	case IO_WQ_CANCEL_OK:
		ret = 0;
		break;
	case IO_WQ_CANCEL_RUNNING:
		ret = -EALREADY;
		break;
	case IO_WQ_CANCEL_NOTFOUND:
		ret = -ENOENT;
		break;
	}

	if (ret < 0 && (req->flags & REQ_F_LINK))
		req->flags |= REQ_F_FAIL_LINK;
	io_cqring_add_event(req->ctx, sqe->user_data, ret);
	io_put_req(req, nxt);
	return 0;
}

2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199
static int io_req_defer(struct io_ring_ctx *ctx, struct io_kiocb *req,
			const struct io_uring_sqe *sqe)
{
	struct io_uring_sqe *sqe_copy;

	if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list))
		return 0;

	sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
	if (!sqe_copy)
		return -EAGAIN;

	spin_lock_irq(&ctx->completion_lock);
	if (!io_sequence_defer(ctx, req) && list_empty(&ctx->defer_list)) {
		spin_unlock_irq(&ctx->completion_lock);
		kfree(sqe_copy);
		return 0;
	}

	memcpy(sqe_copy, sqe, sizeof(*sqe_copy));
	req->submit.sqe = sqe_copy;

2200
	trace_io_uring_defer(ctx, req, false);
2201 2202 2203 2204 2205
	list_add_tail(&req->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}

J
Jens Axboe 已提交
2206
static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
2207 2208
			   const struct sqe_submit *s, struct io_kiocb **nxt,
			   bool force_nonblock)
J
Jens Axboe 已提交
2209
{
2210
	int ret, opcode;
J
Jens Axboe 已提交
2211

J
Jens Axboe 已提交
2212 2213
	req->user_data = READ_ONCE(s->sqe->user_data);

J
Jens Axboe 已提交
2214 2215 2216 2217 2218 2219
	opcode = READ_ONCE(s->sqe->opcode);
	switch (opcode) {
	case IORING_OP_NOP:
		ret = io_nop(req, req->user_data);
		break;
	case IORING_OP_READV:
2220 2221
		if (unlikely(s->sqe->buf_index))
			return -EINVAL;
2222
		ret = io_read(req, s, nxt, force_nonblock);
J
Jens Axboe 已提交
2223 2224
		break;
	case IORING_OP_WRITEV:
2225 2226
		if (unlikely(s->sqe->buf_index))
			return -EINVAL;
2227
		ret = io_write(req, s, nxt, force_nonblock);
2228 2229
		break;
	case IORING_OP_READ_FIXED:
2230
		ret = io_read(req, s, nxt, force_nonblock);
2231 2232
		break;
	case IORING_OP_WRITE_FIXED:
2233
		ret = io_write(req, s, nxt, force_nonblock);
J
Jens Axboe 已提交
2234
		break;
C
Christoph Hellwig 已提交
2235
	case IORING_OP_FSYNC:
2236
		ret = io_fsync(req, s->sqe, nxt, force_nonblock);
C
Christoph Hellwig 已提交
2237
		break;
2238 2239 2240 2241 2242 2243
	case IORING_OP_POLL_ADD:
		ret = io_poll_add(req, s->sqe);
		break;
	case IORING_OP_POLL_REMOVE:
		ret = io_poll_remove(req, s->sqe);
		break;
2244
	case IORING_OP_SYNC_FILE_RANGE:
2245
		ret = io_sync_file_range(req, s->sqe, nxt, force_nonblock);
2246
		break;
J
Jens Axboe 已提交
2247
	case IORING_OP_SENDMSG:
2248
		ret = io_sendmsg(req, s->sqe, nxt, force_nonblock);
J
Jens Axboe 已提交
2249
		break;
J
Jens Axboe 已提交
2250
	case IORING_OP_RECVMSG:
2251
		ret = io_recvmsg(req, s->sqe, nxt, force_nonblock);
J
Jens Axboe 已提交
2252
		break;
J
Jens Axboe 已提交
2253 2254 2255
	case IORING_OP_TIMEOUT:
		ret = io_timeout(req, s->sqe);
		break;
2256 2257 2258
	case IORING_OP_TIMEOUT_REMOVE:
		ret = io_timeout_remove(req, s->sqe);
		break;
2259 2260 2261
	case IORING_OP_ACCEPT:
		ret = io_accept(req, s->sqe, nxt, force_nonblock);
		break;
2262 2263 2264
	case IORING_OP_ASYNC_CANCEL:
		ret = io_async_cancel(req, s->sqe, nxt);
		break;
J
Jens Axboe 已提交
2265 2266 2267 2268 2269
	default:
		ret = -EINVAL;
		break;
	}

J
Jens Axboe 已提交
2270 2271 2272 2273
	if (ret)
		return ret;

	if (ctx->flags & IORING_SETUP_IOPOLL) {
J
Jens Axboe 已提交
2274
		if (req->result == -EAGAIN)
J
Jens Axboe 已提交
2275 2276 2277
			return -EAGAIN;

		/* workqueue context doesn't hold uring_lock, grab it now */
2278
		if (s->in_async)
J
Jens Axboe 已提交
2279 2280
			mutex_lock(&ctx->uring_lock);
		io_iopoll_req_issued(req);
2281
		if (s->in_async)
J
Jens Axboe 已提交
2282 2283 2284 2285
			mutex_unlock(&ctx->uring_lock);
	}

	return 0;
J
Jens Axboe 已提交
2286 2287
}

2288
static void io_wq_submit_work(struct io_wq_work **workptr)
J
Jens Axboe 已提交
2289
{
2290
	struct io_wq_work *work = *workptr;
J
Jens Axboe 已提交
2291 2292
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_ring_ctx *ctx = req->ctx;
2293 2294 2295 2296
	struct sqe_submit *s = &req->submit;
	const struct io_uring_sqe *sqe = s->sqe;
	struct io_kiocb *nxt = NULL;
	int ret = 0;
J
Jens Axboe 已提交
2297

2298 2299
	/* Ensure we clear previously set non-block flag */
	req->rw.ki_flags &= ~IOCB_NOWAIT;
J
Jens Axboe 已提交
2300

2301 2302
	if (work->flags & IO_WQ_WORK_CANCEL)
		ret = -ECANCELED;
2303

2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
	if (!ret) {
		s->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
		s->in_async = true;
		do {
			ret = __io_submit_sqe(ctx, req, s, &nxt, false);
			/*
			 * We can get EAGAIN for polled IO even though we're
			 * forcing a sync submission from here, since we can't
			 * wait for request slots on the block side.
			 */
			if (ret != -EAGAIN)
				break;
			cond_resched();
		} while (1);
	}
2319

2320 2321
	/* drop submission reference */
	io_put_req(req, NULL);
2322

2323 2324
	if (ret) {
		io_cqring_add_event(ctx, sqe->user_data, ret);
2325
		io_put_req(req, NULL);
J
Jens Axboe 已提交
2326
	}
2327

2328 2329
	/* async context always use a copy of the sqe */
	kfree(sqe);
2330

2331 2332 2333 2334
	/* if a dependent link is ready, pass it back */
	if (!ret && nxt) {
		io_prep_async_work(nxt);
		*workptr = &nxt->work;
2335
	}
J
Jens Axboe 已提交
2336 2337
}

J
Jens Axboe 已提交
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
static bool io_op_needs_file(const struct io_uring_sqe *sqe)
{
	int op = READ_ONCE(sqe->opcode);

	switch (op) {
	case IORING_OP_NOP:
	case IORING_OP_POLL_REMOVE:
		return false;
	default:
		return true;
	}
}

2351 2352 2353 2354 2355 2356 2357 2358 2359
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

	table = &ctx->file_table[index >> IORING_FILE_TABLE_SHIFT];
	return table->files[index & IORING_FILE_TABLE_MASK];
}

J
Jens Axboe 已提交
2360 2361 2362 2363 2364 2365 2366 2367 2368
static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s,
			   struct io_submit_state *state, struct io_kiocb *req)
{
	unsigned flags;
	int fd;

	flags = READ_ONCE(s->sqe->flags);
	fd = READ_ONCE(s->sqe->fd);

2369
	if (flags & IOSQE_IO_DRAIN)
2370
		req->flags |= REQ_F_IO_DRAIN;
2371 2372 2373 2374 2375 2376
	/*
	 * All io need record the previous position, if LINK vs DARIN,
	 * it can be used to mark the position of the first IO in the
	 * link list.
	 */
	req->sequence = s->sequence;
2377

2378
	if (!io_op_needs_file(s->sqe))
J
Jens Axboe 已提交
2379 2380 2381
		return 0;

	if (flags & IOSQE_FIXED_FILE) {
2382
		if (unlikely(!ctx->file_table ||
J
Jens Axboe 已提交
2383 2384
		    (unsigned) fd >= ctx->nr_user_files))
			return -EBADF;
2385
		fd = array_index_nospec(fd, ctx->nr_user_files);
2386 2387
		req->file = io_file_from_index(ctx, fd);
		if (!req->file)
2388
			return -EBADF;
J
Jens Axboe 已提交
2389 2390 2391 2392
		req->flags |= REQ_F_FIXED_FILE;
	} else {
		if (s->needs_fixed_file)
			return -EBADF;
2393
		trace_io_uring_file_get(ctx, fd);
J
Jens Axboe 已提交
2394 2395 2396 2397 2398 2399 2400 2401
		req->file = io_file_get(state, fd);
		if (unlikely(!req->file))
			return -EBADF;
	}

	return 0;
}

2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
static int io_grab_files(struct io_ring_ctx *ctx, struct io_kiocb *req)
{
	int ret = -EBADF;

	rcu_read_lock();
	spin_lock_irq(&ctx->inflight_lock);
	/*
	 * We use the f_ops->flush() handler to ensure that we can flush
	 * out work accessing these files if the fd is closed. Check if
	 * the fd has changed since we started down this path, and disallow
	 * this operation if it has.
	 */
	if (fcheck(req->submit.ring_fd) == req->submit.ring_file) {
		list_add(&req->inflight_entry, &ctx->inflight_list);
		req->flags |= REQ_F_INFLIGHT;
		req->work.files = current->files;
		ret = 0;
	}
	spin_unlock_irq(&ctx->inflight_lock);
	rcu_read_unlock();

	return ret;
}

2426
static int __io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
2427
			struct sqe_submit *s)
J
Jens Axboe 已提交
2428
{
2429
	int ret;
J
Jens Axboe 已提交
2430

2431
	ret = __io_submit_sqe(ctx, req, s, NULL, true);
2432 2433 2434 2435 2436 2437 2438

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
	    (req->flags & REQ_F_MUST_PUNT))) {
J
Jens Axboe 已提交
2439 2440
		struct io_uring_sqe *sqe_copy;

2441
		sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
J
Jens Axboe 已提交
2442 2443 2444
		if (sqe_copy) {
			s->sqe = sqe_copy;
			memcpy(&req->submit, s, sizeof(*s));
2445 2446 2447 2448 2449 2450 2451
			if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
				ret = io_grab_files(ctx, req);
				if (ret) {
					kfree(sqe_copy);
					goto err;
				}
			}
2452 2453 2454

			/*
			 * Queued up for async execution, worker will release
J
Jens Axboe 已提交
2455
			 * submit reference when the iocb is actually submitted.
2456
			 */
2457
			io_queue_async_work(ctx, req);
2458
			return 0;
J
Jens Axboe 已提交
2459 2460
		}
	}
2461 2462

	/* drop submission reference */
2463
err:
2464
	io_put_req(req, NULL);
2465 2466

	/* and drop final reference, if we failed */
J
Jens Axboe 已提交
2467 2468 2469 2470
	if (ret) {
		io_cqring_add_event(ctx, req->user_data, ret);
		if (req->flags & REQ_F_LINK)
			req->flags |= REQ_F_FAIL_LINK;
2471
		io_put_req(req, NULL);
J
Jens Axboe 已提交
2472
	}
J
Jens Axboe 已提交
2473 2474 2475 2476

	return ret;
}

2477
static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
2478
			struct sqe_submit *s)
2479 2480 2481 2482 2483 2484
{
	int ret;

	ret = io_req_defer(ctx, req, s->sqe);
	if (ret) {
		if (ret != -EIOCBQUEUED) {
2485
			io_free_req(req, NULL);
2486 2487 2488 2489 2490
			io_cqring_add_event(ctx, s->sqe->user_data, ret);
		}
		return 0;
	}

2491
	return __io_queue_sqe(ctx, req, s);
2492 2493 2494
}

static int io_queue_link_head(struct io_ring_ctx *ctx, struct io_kiocb *req,
2495
			      struct sqe_submit *s, struct io_kiocb *shadow)
2496 2497 2498 2499 2500
{
	int ret;
	int need_submit = false;

	if (!shadow)
2501
		return io_queue_sqe(ctx, req, s);
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511

	/*
	 * Mark the first IO in link list as DRAIN, let all the following
	 * IOs enter the defer list. all IO needs to be completed before link
	 * list.
	 */
	req->flags |= REQ_F_IO_DRAIN;
	ret = io_req_defer(ctx, req, s->sqe);
	if (ret) {
		if (ret != -EIOCBQUEUED) {
2512
			io_free_req(req, NULL);
P
Pavel Begunkov 已提交
2513
			__io_free_req(shadow);
2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
			io_cqring_add_event(ctx, s->sqe->user_data, ret);
			return 0;
		}
	} else {
		/*
		 * If ret == 0 means that all IOs in front of link io are
		 * running done. let's queue link head.
		 */
		need_submit = true;
	}

	/* Insert shadow req to defer_list, blocking next IOs */
	spin_lock_irq(&ctx->completion_lock);
2527
	trace_io_uring_defer(ctx, shadow, true);
2528 2529 2530 2531
	list_add_tail(&shadow->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);

	if (need_submit)
2532
		return __io_queue_sqe(ctx, req, s);
2533 2534 2535 2536

	return 0;
}

J
Jens Axboe 已提交
2537 2538 2539
#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)

static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
2540
			  struct io_submit_state *state, struct io_kiocb **link)
J
Jens Axboe 已提交
2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
{
	struct io_uring_sqe *sqe_copy;
	struct io_kiocb *req;
	int ret;

	/* enforce forwards compatibility on users */
	if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
		ret = -EINVAL;
		goto err;
	}

	req = io_get_req(ctx, state);
	if (unlikely(!req)) {
		ret = -EAGAIN;
		goto err;
	}

	ret = io_req_set_file(ctx, s, state, req);
	if (unlikely(ret)) {
err_req:
2561
		io_free_req(req, NULL);
J
Jens Axboe 已提交
2562 2563 2564 2565 2566
err:
		io_cqring_add_event(ctx, s->sqe->user_data, ret);
		return;
	}

P
Pavel Begunkov 已提交
2567 2568
	req->user_data = s->sqe->user_data;

J
Jens Axboe 已提交
2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (*link) {
		struct io_kiocb *prev = *link;

		sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
		if (!sqe_copy) {
			ret = -EAGAIN;
			goto err_req;
		}

		s->sqe = sqe_copy;
		memcpy(&req->submit, s, sizeof(*s));
2587
		trace_io_uring_link(ctx, req, prev);
J
Jens Axboe 已提交
2588 2589 2590 2591 2592 2593 2594 2595
		list_add_tail(&req->list, &prev->link_list);
	} else if (s->sqe->flags & IOSQE_IO_LINK) {
		req->flags |= REQ_F_LINK;

		memcpy(&req->submit, s, sizeof(*s));
		INIT_LIST_HEAD(&req->link_list);
		*link = req;
	} else {
2596
		io_queue_sqe(ctx, req, s);
J
Jens Axboe 已提交
2597 2598 2599
	}
}

2600 2601 2602 2603 2604 2605
/*
 * Batched submission is done, ensure local IO is flushed out.
 */
static void io_submit_state_end(struct io_submit_state *state)
{
	blk_finish_plug(&state->plug);
2606
	io_file_put(state);
J
Jens Axboe 已提交
2607 2608 2609
	if (state->free_reqs)
		kmem_cache_free_bulk(req_cachep, state->free_reqs,
					&state->reqs[state->cur_req]);
2610 2611 2612 2613 2614 2615 2616 2617 2618
}

/*
 * Start submission side cache.
 */
static void io_submit_state_start(struct io_submit_state *state,
				  struct io_ring_ctx *ctx, unsigned max_ios)
{
	blk_start_plug(&state->plug);
J
Jens Axboe 已提交
2619
	state->free_reqs = 0;
2620 2621 2622 2623
	state->file = NULL;
	state->ios_left = max_ios;
}

J
Jens Axboe 已提交
2624 2625
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
2626
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
2627

2628
	if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) {
J
Jens Axboe 已提交
2629 2630 2631 2632 2633
		/*
		 * Ensure any loads from the SQEs are done at this point,
		 * since once we write the new head, the application could
		 * write new data to them.
		 */
2634
		smp_store_release(&rings->sq.head, ctx->cached_sq_head);
J
Jens Axboe 已提交
2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647
	}
}

/*
 * Fetch an sqe, if one is available. Note that s->sqe will point to memory
 * that is mapped by userspace. This means that care needs to be taken to
 * ensure that reads are stable, as we cannot rely on userspace always
 * being a good citizen. If members of the sqe are validated and then later
 * used, it's important that those reads are done through READ_ONCE() to
 * prevent a re-load down the line.
 */
static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
{
2648 2649
	struct io_rings *rings = ctx->rings;
	u32 *sq_array = ctx->sq_array;
J
Jens Axboe 已提交
2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660
	unsigned head;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
	head = ctx->cached_sq_head;
2661
	/* make sure SQ entry isn't read before tail */
2662
	if (head == smp_load_acquire(&rings->sq.tail))
J
Jens Axboe 已提交
2663 2664
		return false;

2665
	head = READ_ONCE(sq_array[head & ctx->sq_mask]);
J
Jens Axboe 已提交
2666
	if (head < ctx->sq_entries) {
2667
		s->ring_file = NULL;
J
Jens Axboe 已提交
2668
		s->sqe = &ctx->sq_sqes[head];
2669
		s->sequence = ctx->cached_sq_head;
J
Jens Axboe 已提交
2670 2671 2672 2673 2674 2675
		ctx->cached_sq_head++;
		return true;
	}

	/* drop invalid entries */
	ctx->cached_sq_head++;
2676 2677
	ctx->cached_sq_dropped++;
	WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped);
J
Jens Axboe 已提交
2678 2679 2680
	return false;
}

2681
static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
2682
			  struct mm_struct **mm)
J
Jens Axboe 已提交
2683 2684
{
	struct io_submit_state state, *statep = NULL;
J
Jens Axboe 已提交
2685
	struct io_kiocb *link = NULL;
2686
	struct io_kiocb *shadow_req = NULL;
J
Jens Axboe 已提交
2687 2688
	bool prev_was_link = false;
	int i, submitted = 0;
2689
	bool mm_fault = false;
J
Jens Axboe 已提交
2690 2691 2692 2693 2694 2695 2696

	if (nr > IO_PLUG_THRESHOLD) {
		io_submit_state_start(&state, ctx, nr);
		statep = &state;
	}

	for (i = 0; i < nr; i++) {
2697 2698 2699 2700 2701
		struct sqe_submit s;

		if (!io_get_sqring(ctx, &s))
			break;

2702 2703 2704 2705 2706 2707 2708 2709
		if (io_sqe_needs_user(s.sqe) && !*mm) {
			mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
			if (!mm_fault) {
				use_mm(ctx->sqo_mm);
				*mm = ctx->sqo_mm;
			}
		}

J
Jens Axboe 已提交
2710 2711 2712 2713 2714
		/*
		 * If previous wasn't linked and we have a linked command,
		 * that's the end of the chain. Submit the previous link.
		 */
		if (!prev_was_link && link) {
2715
			io_queue_link_head(ctx, link, &link->submit, shadow_req);
J
Jens Axboe 已提交
2716
			link = NULL;
2717
			shadow_req = NULL;
J
Jens Axboe 已提交
2718
		}
2719
		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
J
Jens Axboe 已提交
2720

2721
		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
2722 2723
			if (!shadow_req) {
				shadow_req = io_get_req(ctx, NULL);
2724 2725
				if (unlikely(!shadow_req))
					goto out;
2726 2727 2728
				shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
				refcount_dec(&shadow_req->refs);
			}
2729
			shadow_req->sequence = s.sequence;
2730 2731
		}

2732
out:
2733 2734 2735 2736 2737 2738
		s.has_user = *mm != NULL;
		s.in_async = true;
		s.needs_fixed_file = true;
		trace_io_uring_submit_sqe(ctx, true, true);
		io_submit_sqe(ctx, &s, statep, &link);
		submitted++;
J
Jens Axboe 已提交
2739 2740
	}

J
Jens Axboe 已提交
2741
	if (link)
2742
		io_queue_link_head(ctx, link, &link->submit, shadow_req);
J
Jens Axboe 已提交
2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
	if (statep)
		io_submit_state_end(&state);

	return submitted;
}

static int io_sq_thread(void *data)
{
	struct io_ring_ctx *ctx = data;
	struct mm_struct *cur_mm = NULL;
	mm_segment_t old_fs;
	DEFINE_WAIT(wait);
	unsigned inflight;
	unsigned long timeout;

2758 2759
	complete(&ctx->sqo_thread_started);

J
Jens Axboe 已提交
2760 2761 2762 2763
	old_fs = get_fs();
	set_fs(USER_DS);

	timeout = inflight = 0;
2764
	while (!kthread_should_park()) {
2765
		unsigned int to_submit;
J
Jens Axboe 已提交
2766 2767 2768 2769 2770

		if (inflight) {
			unsigned nr_events = 0;

			if (ctx->flags & IORING_SETUP_IOPOLL) {
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
				/*
				 * inflight is the count of the maximum possible
				 * entries we submitted, but it can be smaller
				 * if we dropped some of them. If we don't have
				 * poll entries available, then we know that we
				 * have nothing left to poll for. Reset the
				 * inflight count to zero in that case.
				 */
				mutex_lock(&ctx->uring_lock);
				if (!list_empty(&ctx->poll_list))
					__io_iopoll_check(ctx, &nr_events, 0);
				else
					inflight = 0;
				mutex_unlock(&ctx->uring_lock);
J
Jens Axboe 已提交
2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
			} else {
				/*
				 * Normal IO, just pretend everything completed.
				 * We don't have to poll completions for that.
				 */
				nr_events = inflight;
			}

			inflight -= nr_events;
			if (!inflight)
				timeout = jiffies + ctx->sq_thread_idle;
		}

2798 2799
		to_submit = io_sqring_entries(ctx);
		if (!to_submit) {
J
Jens Axboe 已提交
2800 2801 2802 2803 2804 2805
			/*
			 * We're polling. If we're within the defined idle
			 * period, then let us spin without work before going
			 * to sleep.
			 */
			if (inflight || !time_after(jiffies, timeout)) {
2806
				cond_resched();
J
Jens Axboe 已提交
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
				continue;
			}

			/*
			 * Drop cur_mm before scheduling, we can't hold it for
			 * long periods (or over schedule()). Do this before
			 * adding ourselves to the waitqueue, as the unuse/drop
			 * may sleep.
			 */
			if (cur_mm) {
				unuse_mm(cur_mm);
				mmput(cur_mm);
				cur_mm = NULL;
			}

			prepare_to_wait(&ctx->sqo_wait, &wait,
						TASK_INTERRUPTIBLE);

			/* Tell userspace we may need a wakeup call */
2826
			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
2827 2828
			/* make sure to read SQ tail after writing flags */
			smp_mb();
J
Jens Axboe 已提交
2829

2830 2831
			to_submit = io_sqring_entries(ctx);
			if (!to_submit) {
2832
				if (kthread_should_park()) {
J
Jens Axboe 已提交
2833 2834 2835 2836 2837 2838 2839 2840
					finish_wait(&ctx->sqo_wait, &wait);
					break;
				}
				if (signal_pending(current))
					flush_signals(current);
				schedule();
				finish_wait(&ctx->sqo_wait, &wait);

2841
				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
2842 2843 2844 2845
				continue;
			}
			finish_wait(&ctx->sqo_wait, &wait);

2846
			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
J
Jens Axboe 已提交
2847 2848
		}

2849
		to_submit = min(to_submit, ctx->sq_entries);
2850
		inflight += io_submit_sqes(ctx, to_submit, &cur_mm);
J
Jens Axboe 已提交
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860

		/* Commit SQ ring head once we've consumed all SQEs */
		io_commit_sqring(ctx);
	}

	set_fs(old_fs);
	if (cur_mm) {
		unuse_mm(cur_mm);
		mmput(cur_mm);
	}
2861

2862
	kthread_parkme();
2863

J
Jens Axboe 已提交
2864 2865 2866
	return 0;
}

2867 2868
static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit,
			  struct file *ring_file, int ring_fd)
J
Jens Axboe 已提交
2869
{
2870
	struct io_submit_state state, *statep = NULL;
J
Jens Axboe 已提交
2871
	struct io_kiocb *link = NULL;
2872
	struct io_kiocb *shadow_req = NULL;
J
Jens Axboe 已提交
2873
	bool prev_was_link = false;
2874
	int i, submit = 0;
J
Jens Axboe 已提交
2875

2876 2877 2878 2879
	if (to_submit > IO_PLUG_THRESHOLD) {
		io_submit_state_start(&state, ctx, to_submit);
		statep = &state;
	}
J
Jens Axboe 已提交
2880 2881 2882 2883 2884 2885 2886

	for (i = 0; i < to_submit; i++) {
		struct sqe_submit s;

		if (!io_get_sqring(ctx, &s))
			break;

J
Jens Axboe 已提交
2887 2888 2889 2890 2891
		/*
		 * If previous wasn't linked and we have a linked command,
		 * that's the end of the chain. Submit the previous link.
		 */
		if (!prev_was_link && link) {
2892
			io_queue_link_head(ctx, link, &link->submit, shadow_req);
J
Jens Axboe 已提交
2893
			link = NULL;
2894
			shadow_req = NULL;
J
Jens Axboe 已提交
2895 2896 2897
		}
		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;

2898 2899 2900
		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
			if (!shadow_req) {
				shadow_req = io_get_req(ctx, NULL);
2901 2902
				if (unlikely(!shadow_req))
					goto out;
2903 2904 2905 2906 2907 2908
				shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
				refcount_dec(&shadow_req->refs);
			}
			shadow_req->sequence = s.sequence;
		}

2909
out:
2910
		s.ring_file = ring_file;
J
Jens Axboe 已提交
2911
		s.has_user = true;
2912
		s.in_async = false;
J
Jens Axboe 已提交
2913
		s.needs_fixed_file = false;
2914
		s.ring_fd = ring_fd;
2915
		submit++;
2916
		trace_io_uring_submit_sqe(ctx, true, false);
2917
		io_submit_sqe(ctx, &s, statep, &link);
J
Jens Axboe 已提交
2918 2919
	}

J
Jens Axboe 已提交
2920
	if (link)
2921
		io_queue_link_head(ctx, link, &link->submit, shadow_req);
2922 2923
	if (statep)
		io_submit_state_end(statep);
J
Jens Axboe 已提交
2924

2925 2926
	io_commit_sqring(ctx);

2927
	return submit;
J
Jens Axboe 已提交
2928 2929
}

2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961
struct io_wait_queue {
	struct wait_queue_entry wq;
	struct io_ring_ctx *ctx;
	unsigned to_wait;
	unsigned nr_timeouts;
};

static inline bool io_should_wake(struct io_wait_queue *iowq)
{
	struct io_ring_ctx *ctx = iowq->ctx;

	/*
	 * Wake up if we have enough events, or if a timeout occured since we
	 * started waiting. For timeouts, we always want to return to userspace,
	 * regardless of event count.
	 */
	return io_cqring_events(ctx->rings) >= iowq->to_wait ||
			atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
}

static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
			    int wake_flags, void *key)
{
	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
							wq);

	if (!io_should_wake(iowq))
		return -1;

	return autoremove_wake_function(curr, mode, wake_flags, key);
}

J
Jens Axboe 已提交
2962 2963 2964 2965 2966 2967 2968
/*
 * Wait until events become available, if we don't already have some. The
 * application must reap them itself, as they reside on the shared cq ring.
 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz)
{
2969 2970 2971 2972 2973 2974 2975 2976 2977
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
2978
	struct io_rings *rings = ctx->rings;
J
Jens Axboe 已提交
2979 2980
	int ret;

2981
	if (io_cqring_events(rings) >= min_events)
J
Jens Axboe 已提交
2982 2983 2984
		return 0;

	if (sig) {
2985 2986 2987
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
2988
						      sigsz);
2989 2990
		else
#endif
2991
			ret = set_user_sigmask(sig, sigsz);
2992

J
Jens Axboe 已提交
2993 2994 2995 2996
		if (ret)
			return ret;
	}

2997 2998
	ret = 0;
	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
2999
	trace_io_uring_cqring_wait(ctx, min_events);
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012
	do {
		prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
		if (io_should_wake(&iowq))
			break;
		schedule();
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
	} while (1);
	finish_wait(&ctx->wait, &iowq.wq);

3013
	restore_saved_sigmask_unless(ret == -ERESTARTSYS);
3014 3015
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
J
Jens Axboe 已提交
3016

3017
	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
J
Jens Axboe 已提交
3018 3019
}

J
Jens Axboe 已提交
3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		struct sock *sock = ctx->ring_sock->sk;
		struct sk_buff *skb;

		while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
			kfree_skb(skb);
	}
#else
	int i;

3033 3034 3035 3036 3037 3038 3039
	for (i = 0; i < ctx->nr_user_files; i++) {
		struct file *file;

		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
J
Jens Axboe 已提交
3040 3041 3042 3043 3044
#endif
}

static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{
3045 3046 3047
	unsigned nr_tables, i;

	if (!ctx->file_table)
J
Jens Axboe 已提交
3048 3049 3050
		return -ENXIO;

	__io_sqe_files_unregister(ctx);
3051 3052 3053 3054 3055
	nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
	for (i = 0; i < nr_tables; i++)
		kfree(ctx->file_table[i].files);
	kfree(ctx->file_table);
	ctx->file_table = NULL;
J
Jens Axboe 已提交
3056 3057 3058 3059
	ctx->nr_user_files = 0;
	return 0;
}

J
Jens Axboe 已提交
3060 3061 3062
static void io_sq_thread_stop(struct io_ring_ctx *ctx)
{
	if (ctx->sqo_thread) {
3063
		wait_for_completion(&ctx->sqo_thread_started);
3064 3065 3066 3067 3068
		/*
		 * The park is a bit of a work-around, without it we get
		 * warning spews on shutdown with SQPOLL set and affinity
		 * set to a single CPU.
		 */
3069
		kthread_park(ctx->sqo_thread);
J
Jens Axboe 已提交
3070 3071 3072 3073 3074
		kthread_stop(ctx->sqo_thread);
		ctx->sqo_thread = NULL;
	}
}

J
Jens Axboe 已提交
3075 3076
static void io_finish_async(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
3077 3078
	io_sq_thread_stop(ctx);

3079 3080 3081
	if (ctx->io_wq) {
		io_wq_destroy(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
3082 3083 3084 3085 3086 3087 3088
	}
}

#if defined(CONFIG_UNIX)
static void io_destruct_skb(struct sk_buff *skb)
{
	struct io_ring_ctx *ctx = skb->sk->sk_user_data;
3089

3090 3091
	if (ctx->io_wq)
		io_wq_flush(ctx->io_wq);
J
Jens Axboe 已提交
3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105

	unix_destruct_scm(skb);
}

/*
 * Ensure the UNIX gc is aware of our file set, so we are certain that
 * the io_uring can be safely unregistered on process exit, even if we have
 * loops in the file referencing.
 */
static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
{
	struct sock *sk = ctx->ring_sock->sk;
	struct scm_fp_list *fpl;
	struct sk_buff *skb;
3106
	int i, nr_files;
J
Jens Axboe 已提交
3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126

	if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
		unsigned long inflight = ctx->user->unix_inflight + nr;

		if (inflight > task_rlimit(current, RLIMIT_NOFILE))
			return -EMFILE;
	}

	fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	if (!fpl)
		return -ENOMEM;

	skb = alloc_skb(0, GFP_KERNEL);
	if (!skb) {
		kfree(fpl);
		return -ENOMEM;
	}

	skb->sk = sk;

3127
	nr_files = 0;
J
Jens Axboe 已提交
3128 3129
	fpl->user = get_uid(ctx->user);
	for (i = 0; i < nr; i++) {
3130 3131 3132
		struct file *file = io_file_from_index(ctx, i + offset);

		if (!file)
3133
			continue;
3134
		fpl->fp[nr_files] = get_file(file);
3135 3136
		unix_inflight(fpl->user, fpl->fp[nr_files]);
		nr_files++;
J
Jens Axboe 已提交
3137 3138
	}

3139 3140 3141 3142 3143 3144 3145
	if (nr_files) {
		fpl->max = SCM_MAX_FD;
		fpl->count = nr_files;
		UNIXCB(skb).fp = fpl;
		skb->destructor = io_destruct_skb;
		refcount_add(skb->truesize, &sk->sk_wmem_alloc);
		skb_queue_head(&sk->sk_receive_queue, skb);
J
Jens Axboe 已提交
3146

3147 3148 3149 3150 3151 3152
		for (i = 0; i < nr_files; i++)
			fput(fpl->fp[i]);
	} else {
		kfree_skb(skb);
		kfree(fpl);
	}
J
Jens Axboe 已提交
3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182

	return 0;
}

/*
 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
 * causes regular reference counting to break down. We rely on the UNIX
 * garbage collection to take care of this problem for us.
 */
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	unsigned left, total;
	int ret = 0;

	total = 0;
	left = ctx->nr_user_files;
	while (left) {
		unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);

		ret = __io_sqe_files_scm(ctx, this_files, total);
		if (ret)
			break;
		left -= this_files;
		total += this_files;
	}

	if (!ret)
		return 0;

	while (total < ctx->nr_user_files) {
3183 3184 3185 3186
		struct file *file = io_file_from_index(ctx, total);

		if (file)
			fput(file);
J
Jens Axboe 已提交
3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198
		total++;
	}

	return ret;
}
#else
static int io_sqe_files_scm(struct io_ring_ctx *ctx)
{
	return 0;
}
#endif

3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225
static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
				    unsigned nr_files)
{
	int i;

	for (i = 0; i < nr_tables; i++) {
		struct fixed_file_table *table = &ctx->file_table[i];
		unsigned this_files;

		this_files = min(nr_files, IORING_MAX_FILES_TABLE);
		table->files = kcalloc(this_files, sizeof(struct file *),
					GFP_KERNEL);
		if (!table->files)
			break;
		nr_files -= this_files;
	}

	if (i == nr_tables)
		return 0;

	for (i = 0; i < nr_tables; i++) {
		struct fixed_file_table *table = &ctx->file_table[i];
		kfree(table->files);
	}
	return 1;
}

J
Jens Axboe 已提交
3226 3227 3228 3229
static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args)
{
	__s32 __user *fds = (__s32 __user *) arg;
3230
	unsigned nr_tables;
J
Jens Axboe 已提交
3231 3232 3233
	int fd, ret = 0;
	unsigned i;

3234
	if (ctx->file_table)
J
Jens Axboe 已提交
3235 3236 3237 3238 3239 3240
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;

3241 3242 3243 3244
	nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
	ctx->file_table = kcalloc(nr_tables, sizeof(struct fixed_file_table),
					GFP_KERNEL);
	if (!ctx->file_table)
J
Jens Axboe 已提交
3245 3246
		return -ENOMEM;

3247 3248 3249 3250 3251
	if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
		kfree(ctx->file_table);
		return -ENOMEM;
	}

3252
	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
3253 3254 3255
		struct fixed_file_table *table;
		unsigned index;

J
Jens Axboe 已提交
3256 3257 3258
		ret = -EFAULT;
		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
			break;
3259 3260 3261 3262 3263
		/* allow sparse sets */
		if (fd == -1) {
			ret = 0;
			continue;
		}
J
Jens Axboe 已提交
3264

3265 3266 3267
		table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
		index = i & IORING_FILE_TABLE_MASK;
		table->files[index] = fget(fd);
J
Jens Axboe 已提交
3268 3269

		ret = -EBADF;
3270
		if (!table->files[index])
J
Jens Axboe 已提交
3271 3272 3273 3274 3275 3276 3277 3278
			break;
		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
3279 3280
		if (table->files[index]->f_op == &io_uring_fops) {
			fput(table->files[index]);
J
Jens Axboe 已提交
3281 3282 3283 3284 3285 3286
			break;
		}
		ret = 0;
	}

	if (ret) {
3287 3288
		for (i = 0; i < ctx->nr_user_files; i++) {
			struct file *file;
J
Jens Axboe 已提交
3289

3290 3291 3292 3293 3294 3295 3296 3297 3298
			file = io_file_from_index(ctx, i);
			if (file)
				fput(file);
		}
		for (i = 0; i < nr_tables; i++)
			kfree(ctx->file_table[i].files);

		kfree(ctx->file_table);
		ctx->file_table = NULL;
J
Jens Axboe 已提交
3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
		ctx->nr_user_files = 0;
		return ret;
	}

	ret = io_sqe_files_scm(ctx);
	if (ret)
		io_sqe_files_unregister(ctx);

	return ret;
}

3310 3311 3312
static void io_sqe_file_unregister(struct io_ring_ctx *ctx, int index)
{
#if defined(CONFIG_UNIX)
3313
	struct file *file = io_file_from_index(ctx, index);
3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head list, *head = &sock->sk_receive_queue;
	struct sk_buff *skb;
	int i;

	__skb_queue_head_init(&list);

	/*
	 * Find the skb that holds this file in its SCM_RIGHTS. When found,
	 * remove this entry and rearrange the file array.
	 */
	skb = skb_dequeue(head);
	while (skb) {
		struct scm_fp_list *fp;

		fp = UNIXCB(skb).fp;
		for (i = 0; i < fp->count; i++) {
			int left;

			if (fp->fp[i] != file)
				continue;

			unix_notinflight(fp->user, fp->fp[i]);
			left = fp->count - 1 - i;
			if (left) {
				memmove(&fp->fp[i], &fp->fp[i + 1],
						left * sizeof(struct file *));
			}
			fp->count--;
			if (!fp->count) {
				kfree_skb(skb);
				skb = NULL;
			} else {
				__skb_queue_tail(&list, skb);
			}
			fput(file);
			file = NULL;
			break;
		}

		if (!file)
			break;

		__skb_queue_tail(&list, skb);

		skb = skb_dequeue(head);
	}

	if (skb_peek(&list)) {
		spin_lock_irq(&head->lock);
		while ((skb = __skb_dequeue(&list)) != NULL)
			__skb_queue_tail(head, skb);
		spin_unlock_irq(&head->lock);
	}
#else
3369
	fput(io_file_from_index(ctx, index));
3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
#endif
}

static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
				int index)
{
#if defined(CONFIG_UNIX)
	struct sock *sock = ctx->ring_sock->sk;
	struct sk_buff_head *head = &sock->sk_receive_queue;
	struct sk_buff *skb;

	/*
	 * See if we can merge this file into an existing skb SCM_RIGHTS
	 * file set. If there's no room, fall back to allocating a new skb
	 * and filling it in.
	 */
	spin_lock_irq(&head->lock);
	skb = skb_peek(head);
	if (skb) {
		struct scm_fp_list *fpl = UNIXCB(skb).fp;

		if (fpl->count < SCM_MAX_FD) {
			__skb_unlink(skb, head);
			spin_unlock_irq(&head->lock);
			fpl->fp[fpl->count] = get_file(file);
			unix_inflight(fpl->user, fpl->fp[fpl->count]);
			fpl->count++;
			spin_lock_irq(&head->lock);
			__skb_queue_head(head, skb);
		} else {
			skb = NULL;
		}
	}
	spin_unlock_irq(&head->lock);

	if (skb) {
		fput(file);
		return 0;
	}

	return __io_sqe_files_scm(ctx, 1, index);
#else
	return 0;
#endif
}

static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;
	__s32 __user *fds;
	int fd, i, err;
	__u32 done;

3424
	if (!ctx->file_table)
3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (check_add_overflow(up.offset, nr_args, &done))
		return -EOVERFLOW;
	if (done > ctx->nr_user_files)
		return -EINVAL;

	done = 0;
	fds = (__s32 __user *) up.fds;
	while (nr_args) {
3438 3439 3440
		struct fixed_file_table *table;
		unsigned index;

3441 3442 3443 3444 3445 3446
		err = 0;
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			err = -EFAULT;
			break;
		}
		i = array_index_nospec(up.offset, ctx->nr_user_files);
3447 3448 3449
		table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
		index = i & IORING_FILE_TABLE_MASK;
		if (table->files[index]) {
3450
			io_sqe_file_unregister(ctx, i);
3451
			table->files[index] = NULL;
3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473
		}
		if (fd != -1) {
			struct file *file;

			file = fget(fd);
			if (!file) {
				err = -EBADF;
				break;
			}
			/*
			 * Don't allow io_uring instances to be registered. If
			 * UNIX isn't enabled, then this causes a reference
			 * cycle and this instance can never get freed. If UNIX
			 * is enabled we'll handle it just fine, but there's
			 * still no point in allowing a ring fd as it doesn't
			 * support regular read/write anyway.
			 */
			if (file->f_op == &io_uring_fops) {
				fput(file);
				err = -EBADF;
				break;
			}
3474
			table->files[index] = file;
3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486
			err = io_sqe_file_register(ctx, file, i);
			if (err)
				break;
		}
		nr_args--;
		done++;
		up.offset++;
	}

	return done ? done : err;
}

J
Jens Axboe 已提交
3487 3488
static int io_sq_offload_start(struct io_ring_ctx *ctx,
			       struct io_uring_params *p)
J
Jens Axboe 已提交
3489
{
3490
	unsigned concurrency;
J
Jens Axboe 已提交
3491 3492
	int ret;

J
Jens Axboe 已提交
3493
	init_waitqueue_head(&ctx->sqo_wait);
J
Jens Axboe 已提交
3494 3495 3496
	mmgrab(current->mm);
	ctx->sqo_mm = current->mm;

J
Jens Axboe 已提交
3497
	if (ctx->flags & IORING_SETUP_SQPOLL) {
3498 3499 3500 3501
		ret = -EPERM;
		if (!capable(CAP_SYS_ADMIN))
			goto err;

3502 3503 3504 3505
		ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
		if (!ctx->sq_thread_idle)
			ctx->sq_thread_idle = HZ;

J
Jens Axboe 已提交
3506
		if (p->flags & IORING_SETUP_SQ_AFF) {
3507
			int cpu = p->sq_thread_cpu;
J
Jens Axboe 已提交
3508

3509
			ret = -EINVAL;
3510 3511
			if (cpu >= nr_cpu_ids)
				goto err;
3512
			if (!cpu_online(cpu))
3513 3514
				goto err;

J
Jens Axboe 已提交
3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
							ctx, cpu,
							"io_uring-sq");
		} else {
			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
							"io_uring-sq");
		}
		if (IS_ERR(ctx->sqo_thread)) {
			ret = PTR_ERR(ctx->sqo_thread);
			ctx->sqo_thread = NULL;
			goto err;
		}
		wake_up_process(ctx->sqo_thread);
	} else if (p->flags & IORING_SETUP_SQ_AFF) {
		/* Can't have SQ_AFF without SQPOLL */
		ret = -EINVAL;
		goto err;
	}

3534 3535 3536
	/* Do QD, or 4 * CPUS, whatever is smallest */
	concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
	ctx->io_wq = io_wq_create(concurrency, ctx->sqo_mm);
3537 3538 3539
	if (IS_ERR(ctx->io_wq)) {
		ret = PTR_ERR(ctx->io_wq);
		ctx->io_wq = NULL;
J
Jens Axboe 已提交
3540 3541 3542 3543 3544
		goto err;
	}

	return 0;
err:
3545
	io_finish_async(ctx);
J
Jens Axboe 已提交
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
	mmdrop(ctx->sqo_mm);
	ctx->sqo_mm = NULL;
	return ret;
}

static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
{
	atomic_long_sub(nr_pages, &user->locked_vm);
}

static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
	unsigned long page_limit, cur_pages, new_pages;

	/* Don't allow more pages than we can safely lock */
	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	do {
		cur_pages = atomic_long_read(&user->locked_vm);
		new_pages = cur_pages + nr_pages;
		if (new_pages > page_limit)
			return -ENOMEM;
	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
					new_pages) != cur_pages);

	return 0;
}

static void io_mem_free(void *ptr)
{
3576 3577 3578 3579
	struct page *page;

	if (!ptr)
		return;
J
Jens Axboe 已提交
3580

3581
	page = virt_to_head_page(ptr);
J
Jens Axboe 已提交
3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593
	if (put_page_testzero(page))
		free_compound_page(page);
}

static void *io_mem_alloc(size_t size)
{
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
				__GFP_NORETRY;

	return (void *) __get_free_pages(gfp_flags, get_order(size));
}

3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	if (sq_offset)
		*sq_offset = off;

	return off;
}

J
Jens Axboe 已提交
3623 3624
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
3625
	size_t pages;
J
Jens Axboe 已提交
3626

3627 3628 3629 3630
	pages = (size_t)1 << get_order(
		rings_size(sq_entries, cq_entries, NULL));
	pages += (size_t)1 << get_order(
		array_size(sizeof(struct io_uring_sqe), sq_entries));
J
Jens Axboe 已提交
3631

3632
	return pages;
J
Jens Axboe 已提交
3633 3634
}

3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645
static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
3646
			put_user_page(imu->bvec[j].bv_page);
3647 3648 3649

		if (ctx->account_mem)
			io_unaccount_mem(ctx->user, imu->nr_bvecs);
3650
		kvfree(imu->bvec);
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}

static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
		       void __user *arg, unsigned index)
{
	struct iovec __user *src;

#ifdef CONFIG_COMPAT
	if (ctx->compat) {
		struct compat_iovec __user *ciovs;
		struct compat_iovec ciov;

		ciovs = (struct compat_iovec __user *) arg;
		if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
			return -EFAULT;

		dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
		dst->iov_len = ciov.iov_len;
		return 0;
	}
#endif
	src = (struct iovec __user *) arg;
	if (copy_from_user(dst, &src[index], sizeof(*dst)))
		return -EFAULT;
	return 0;
}

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
				  unsigned nr_args)
{
	struct vm_area_struct **vmas = NULL;
	struct page **pages = NULL;
	int i, j, got_pages = 0;
	int ret = -EINVAL;

	if (ctx->user_bufs)
		return -EBUSY;
	if (!nr_args || nr_args > UIO_MAXIOV)
		return -EINVAL;

	ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
					GFP_KERNEL);
	if (!ctx->user_bufs)
		return -ENOMEM;

	for (i = 0; i < nr_args; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
		unsigned long off, start, end, ubuf;
		int pret, nr_pages;
		struct iovec iov;
		size_t size;

		ret = io_copy_iov(ctx, &iov, arg, i);
		if (ret)
3712
			goto err;
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741

		/*
		 * Don't impose further limits on the size and buffer
		 * constraints here, we'll -EINVAL later when IO is
		 * submitted if they are wrong.
		 */
		ret = -EFAULT;
		if (!iov.iov_base || !iov.iov_len)
			goto err;

		/* arbitrary limit, but we need something */
		if (iov.iov_len > SZ_1G)
			goto err;

		ubuf = (unsigned long) iov.iov_base;
		end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		start = ubuf >> PAGE_SHIFT;
		nr_pages = end - start;

		if (ctx->account_mem) {
			ret = io_account_mem(ctx->user, nr_pages);
			if (ret)
				goto err;
		}

		ret = 0;
		if (!pages || nr_pages > got_pages) {
			kfree(vmas);
			kfree(pages);
3742
			pages = kvmalloc_array(nr_pages, sizeof(struct page *),
3743
						GFP_KERNEL);
3744
			vmas = kvmalloc_array(nr_pages,
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755
					sizeof(struct vm_area_struct *),
					GFP_KERNEL);
			if (!pages || !vmas) {
				ret = -ENOMEM;
				if (ctx->account_mem)
					io_unaccount_mem(ctx->user, nr_pages);
				goto err;
			}
			got_pages = nr_pages;
		}

3756
		imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
3757 3758 3759 3760 3761 3762 3763 3764 3765 3766
						GFP_KERNEL);
		ret = -ENOMEM;
		if (!imu->bvec) {
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
			goto err;
		}

		ret = 0;
		down_read(&current->mm->mmap_sem);
3767 3768 3769
		pret = get_user_pages(ubuf, nr_pages,
				      FOLL_WRITE | FOLL_LONGTERM,
				      pages, vmas);
3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789
		if (pret == nr_pages) {
			/* don't support file backed memory */
			for (j = 0; j < nr_pages; j++) {
				struct vm_area_struct *vma = vmas[j];

				if (vma->vm_file &&
				    !is_file_hugepages(vma->vm_file)) {
					ret = -EOPNOTSUPP;
					break;
				}
			}
		} else {
			ret = pret < 0 ? pret : -EFAULT;
		}
		up_read(&current->mm->mmap_sem);
		if (ret) {
			/*
			 * if we did partial map, or found file backed vmas,
			 * release any pages we did get
			 */
3790 3791
			if (pret > 0)
				put_user_pages(pages, pret);
3792 3793
			if (ctx->account_mem)
				io_unaccount_mem(ctx->user, nr_pages);
3794
			kvfree(imu->bvec);
3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
			goto err;
		}

		off = ubuf & ~PAGE_MASK;
		size = iov.iov_len;
		for (j = 0; j < nr_pages; j++) {
			size_t vec_len;

			vec_len = min_t(size_t, size, PAGE_SIZE - off);
			imu->bvec[j].bv_page = pages[j];
			imu->bvec[j].bv_len = vec_len;
			imu->bvec[j].bv_offset = off;
			off = 0;
			size -= vec_len;
		}
		/* store original address for later verification */
		imu->ubuf = ubuf;
		imu->len = iov.iov_len;
		imu->nr_bvecs = nr_pages;

		ctx->nr_user_bufs++;
	}
3817 3818
	kvfree(pages);
	kvfree(vmas);
3819 3820
	return 0;
err:
3821 3822
	kvfree(pages);
	kvfree(vmas);
3823 3824 3825 3826
	io_sqe_buffer_unregister(ctx);
	return ret;
}

3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858
static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;
}

static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;
}

J
Jens Axboe 已提交
3859 3860
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
J
Jens Axboe 已提交
3861
	io_finish_async(ctx);
J
Jens Axboe 已提交
3862 3863
	if (ctx->sqo_mm)
		mmdrop(ctx->sqo_mm);
J
Jens Axboe 已提交
3864 3865

	io_iopoll_reap_events(ctx);
3866
	io_sqe_buffer_unregister(ctx);
J
Jens Axboe 已提交
3867
	io_sqe_files_unregister(ctx);
3868
	io_eventfd_unregister(ctx);
J
Jens Axboe 已提交
3869

J
Jens Axboe 已提交
3870
#if defined(CONFIG_UNIX)
3871 3872
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
J
Jens Axboe 已提交
3873
		sock_release(ctx->ring_sock);
3874
	}
J
Jens Axboe 已提交
3875 3876
#endif

3877
	io_mem_free(ctx->rings);
J
Jens Axboe 已提交
3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	if (ctx->account_mem)
		io_unaccount_mem(ctx->user,
				ring_pages(ctx->sq_entries, ctx->cq_entries));
	free_uid(ctx->user);
	kfree(ctx);
}

static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
3894 3895 3896 3897
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
J
Jens Axboe 已提交
3898
	smp_rmb();
3899 3900
	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
	    ctx->rings->sq_ring_entries)
J
Jens Axboe 已提交
3901
		mask |= EPOLLOUT | EPOLLWRNORM;
3902
	if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
J
Jens Axboe 已提交
3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;
}

static int io_uring_fasync(int fd, struct file *file, int on)
{
	struct io_ring_ctx *ctx = file->private_data;

	return fasync_helper(fd, file, on, &ctx->cq_fasync);
}

static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
{
	mutex_lock(&ctx->uring_lock);
	percpu_ref_kill(&ctx->refs);
	mutex_unlock(&ctx->uring_lock);

J
Jens Axboe 已提交
3921
	io_kill_timeouts(ctx);
3922
	io_poll_remove_all(ctx);
3923 3924 3925 3926

	if (ctx->io_wq)
		io_wq_cancel_all(ctx->io_wq);

J
Jens Axboe 已提交
3927
	io_iopoll_reap_events(ctx);
J
Jens Axboe 已提交
3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940
	wait_for_completion(&ctx->ctx_done);
	io_ring_ctx_free(ctx);
}

static int io_uring_release(struct inode *inode, struct file *file)
{
	struct io_ring_ctx *ctx = file->private_data;

	file->private_data = NULL;
	io_ring_ctx_wait_and_kill(ctx);
	return 0;
}

3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987
static void io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
	struct io_kiocb *req;
	DEFINE_WAIT(wait);

	while (!list_empty_careful(&ctx->inflight_list)) {
		enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
			if (req->work.files == files) {
				ret = io_wq_cancel_work(ctx->io_wq, &req->work);
				break;
			}
		}
		if (ret == IO_WQ_CANCEL_RUNNING)
			prepare_to_wait(&ctx->inflight_wait, &wait,
					TASK_UNINTERRUPTIBLE);

		spin_unlock_irq(&ctx->inflight_lock);

		/*
		 * We need to keep going until we get NOTFOUND. We only cancel
		 * one work at the time.
		 *
		 * If we get CANCEL_RUNNING, then wait for a work to complete
		 * before continuing.
		 */
		if (ret == IO_WQ_CANCEL_OK)
			continue;
		else if (ret != IO_WQ_CANCEL_RUNNING)
			break;
		schedule();
	}
}

static int io_uring_flush(struct file *file, void *data)
{
	struct io_ring_ctx *ctx = file->private_data;

	io_uring_cancel_files(ctx, data);
	if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
		io_wq_cancel_all(ctx->io_wq);
	return 0;
}

J
Jens Axboe 已提交
3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998
static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
	unsigned long sz = vma->vm_end - vma->vm_start;
	struct io_ring_ctx *ctx = file->private_data;
	unsigned long pfn;
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
3999 4000
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
J
Jens Axboe 已提交
4001 4002 4003 4004 4005 4006 4007 4008 4009
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
		return -EINVAL;
	}

	page = virt_to_head_page(ptr);
4010
	if (sz > page_size(page))
J
Jens Axboe 已提交
4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025
		return -EINVAL;

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}

SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

J
Jens Axboe 已提交
4026
	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
J
Jens Axboe 已提交
4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

J
Jens Axboe 已提交
4042 4043 4044 4045 4046
	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
4047
	ret = 0;
J
Jens Axboe 已提交
4048 4049 4050 4051
	if (ctx->flags & IORING_SETUP_SQPOLL) {
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
4052
	} else if (to_submit) {
J
Jens Axboe 已提交
4053 4054 4055
		to_submit = min(to_submit, ctx->sq_entries);

		mutex_lock(&ctx->uring_lock);
4056
		submitted = io_ring_submit(ctx, to_submit, f.file, fd);
J
Jens Axboe 已提交
4057 4058 4059
		mutex_unlock(&ctx->uring_lock);
	}
	if (flags & IORING_ENTER_GETEVENTS) {
J
Jens Axboe 已提交
4060 4061
		unsigned nr_events = 0;

J
Jens Axboe 已提交
4062 4063
		min_complete = min(min_complete, ctx->cq_entries);

J
Jens Axboe 已提交
4064 4065 4066 4067 4068
		if (ctx->flags & IORING_SETUP_IOPOLL) {
			ret = io_iopoll_check(ctx, &nr_events, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
J
Jens Axboe 已提交
4069 4070
	}

4071
	percpu_ref_put(&ctx->refs);
J
Jens Axboe 已提交
4072 4073 4074 4075 4076 4077 4078
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}

static const struct file_operations io_uring_fops = {
	.release	= io_uring_release,
4079
	.flush		= io_uring_flush,
J
Jens Axboe 已提交
4080 4081 4082 4083 4084 4085 4086 4087
	.mmap		= io_uring_mmap,
	.poll		= io_uring_poll,
	.fasync		= io_uring_fasync,
};

static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
				  struct io_uring_params *p)
{
4088 4089
	struct io_rings *rings;
	size_t size, sq_array_offset;
J
Jens Axboe 已提交
4090

4091 4092 4093 4094 4095 4096
	size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	rings = io_mem_alloc(size);
	if (!rings)
J
Jens Axboe 已提交
4097 4098
		return -ENOMEM;

4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
	ctx->rings = rings;
	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
	rings->sq_ring_mask = p->sq_entries - 1;
	rings->cq_ring_mask = p->cq_entries - 1;
	rings->sq_ring_entries = p->sq_entries;
	rings->cq_ring_entries = p->cq_entries;
	ctx->sq_mask = rings->sq_ring_mask;
	ctx->cq_mask = rings->cq_ring_mask;
	ctx->sq_entries = rings->sq_ring_entries;
	ctx->cq_entries = rings->cq_ring_entries;
J
Jens Axboe 已提交
4109 4110 4111 4112 4113 4114

	size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	ctx->sq_sqes = io_mem_alloc(size);
4115
	if (!ctx->sq_sqes)
J
Jens Axboe 已提交
4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152
		return -ENOMEM;

	return 0;
}

/*
 * Allocate an anonymous fd, this is what constitutes the application
 * visible backing of an io_uring instance. The application mmaps this
 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
 * we have to tie this fd to a socket for file garbage collection purposes.
 */
static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
J
Jens Axboe 已提交
4153
	ctx->ring_sock->sk->sk_user_data = ctx;
J
Jens Axboe 已提交
4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178
#endif
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}

static int io_uring_create(unsigned entries, struct io_uring_params *p)
{
	struct user_struct *user = NULL;
	struct io_ring_ctx *ctx;
	bool account_mem;
	int ret;

	if (!entries || entries > IORING_MAX_ENTRIES)
		return -EINVAL;

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
4179 4180 4181
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
J
Jens Axboe 已提交
4182 4183
	 */
	p->sq_entries = roundup_pow_of_two(entries);
4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
		if (p->cq_entries < p->sq_entries || p->cq_entries > IORING_MAX_CQ_ENTRIES)
			return -EINVAL;
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}
J
Jens Axboe 已提交
4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224

	user = get_uid(current_user());
	account_mem = !capable(CAP_IPC_LOCK);

	if (account_mem) {
		ret = io_account_mem(user,
				ring_pages(p->sq_entries, p->cq_entries));
		if (ret) {
			free_uid(user);
			return ret;
		}
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx) {
		if (account_mem)
			io_unaccount_mem(user, ring_pages(p->sq_entries,
								p->cq_entries));
		free_uid(user);
		return -ENOMEM;
	}
	ctx->compat = in_compat_syscall();
	ctx->account_mem = account_mem;
	ctx->user = user;

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

J
Jens Axboe 已提交
4225
	ret = io_sq_offload_start(ctx, p);
J
Jens Axboe 已提交
4226 4227 4228 4229
	if (ret)
		goto err;

	memset(&p->sq_off, 0, sizeof(p->sq_off));
4230 4231 4232 4233 4234 4235 4236
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
J
Jens Axboe 已提交
4237 4238

	memset(&p->cq_off, 0, sizeof(p->cq_off));
4239 4240 4241 4242 4243 4244
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
4245

4246 4247 4248 4249 4250 4251 4252 4253
	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_get_fd(ctx);
	if (ret < 0)
		goto err;

4254
	p->features = IORING_FEAT_SINGLE_MMAP;
4255
	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
J
Jens Axboe 已提交
4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;
}

/*
 * Sets up an aio uring context, and returns the fd. Applications asks for a
 * ring size, we return the actual sq/cq ring sizes (among other things) in the
 * params structure passed in.
 */
static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	long ret;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

J
Jens Axboe 已提交
4280
	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
4281
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE))
J
Jens Axboe 已提交
4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
		return -EINVAL;

	ret = io_uring_create(entries, &p);
	if (ret < 0)
		return ret;

	if (copy_to_user(params, &p, sizeof(p)))
		return -EFAULT;

	return ret;
}

SYSCALL_DEFINE2(io_uring_setup, u32, entries,
		struct io_uring_params __user *, params)
{
	return io_uring_setup(entries, params);
}

4300 4301
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
			       void __user *arg, unsigned nr_args)
4302 4303
	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
4304 4305 4306
{
	int ret;

4307 4308 4309 4310 4311 4312 4313 4314
	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

4315
	percpu_ref_kill(&ctx->refs);
4316 4317 4318 4319 4320 4321 4322 4323 4324

	/*
	 * Drop uring mutex before waiting for references to exit. If another
	 * thread is currently inside io_uring_enter() it might need to grab
	 * the uring_lock to make progress. If we hold it here across the drain
	 * wait, then we can deadlock. It's safe to drop the mutex here, since
	 * no new references will come in after we've killed the percpu ref.
	 */
	mutex_unlock(&ctx->uring_lock);
4325
	wait_for_completion(&ctx->ctx_done);
4326
	mutex_lock(&ctx->uring_lock);
4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
J
Jens Axboe 已提交
4338 4339 4340 4341 4342 4343 4344 4345 4346
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
4347 4348 4349
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361
	case IORING_REGISTER_EVENTFD:
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392
	default:
		ret = -EINVAL;
		break;
	}

	/* bring the ctx back to life */
	reinit_completion(&ctx->ctx_done);
	percpu_ref_reinit(&ctx->refs);
	return ret;
}

SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
		void __user *, arg, unsigned int, nr_args)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	struct fd f;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ctx = f.file->private_data;

	mutex_lock(&ctx->uring_lock);
	ret = __io_uring_register(ctx, opcode, arg, nr_args);
	mutex_unlock(&ctx->uring_lock);
4393 4394
	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
							ctx->cq_ev_fd != NULL, ret);
4395 4396 4397 4398 4399
out_fput:
	fdput(f);
	return ret;
}

J
Jens Axboe 已提交
4400 4401 4402 4403 4404 4405
static int __init io_uring_init(void)
{
	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
	return 0;
};
__initcall(io_uring_init);