aio.c 46.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 *	An async IO implementation for Linux
 *	Written by Benjamin LaHaise <bcrl@kvack.org>
 *
 *	Implements an efficient asynchronous io interface.
 *
 *	Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved.
 *
 *	See ../COPYING for licensing terms.
 */
K
Kent Overstreet 已提交
11 12
#define pr_fmt(fmt) "%s: " fmt, __func__

L
Linus Torvalds 已提交
13 14 15 16 17
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/aio_abi.h>
18
#include <linux/export.h>
L
Linus Torvalds 已提交
19
#include <linux/syscalls.h>
20
#include <linux/backing-dev.h>
21
#include <linux/uio.h>
L
Linus Torvalds 已提交
22

23
#include <linux/sched/signal.h>
L
Linus Torvalds 已提交
24 25 26 27
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
28
#include <linux/mmu_context.h>
K
Kent Overstreet 已提交
29
#include <linux/percpu.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/aio.h>
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
36
#include <linux/eventfd.h>
J
Jeff Moyer 已提交
37
#include <linux/blkdev.h>
38
#include <linux/compat.h>
39 40
#include <linux/migrate.h>
#include <linux/ramfs.h>
K
Kent Overstreet 已提交
41
#include <linux/percpu-refcount.h>
42
#include <linux/mount.h>
L
Linus Torvalds 已提交
43 44

#include <asm/kmap_types.h>
45
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
46

A
Al Viro 已提交
47 48
#include "internal.h"

49 50 51 52 53 54
#define AIO_RING_MAGIC			0xa10a10a1
#define AIO_RING_COMPAT_FEATURES	1
#define AIO_RING_INCOMPAT_FEATURES	0
struct aio_ring {
	unsigned	id;	/* kernel internal index number */
	unsigned	nr;	/* number of io_events */
55 56
	unsigned	head;	/* Written to by userland or under ring_lock
				 * mutex by aio_read_events_ring(). */
57 58 59 60 61 62 63 64 65 66 67 68 69
	unsigned	tail;

	unsigned	magic;
	unsigned	compat_features;
	unsigned	incompat_features;
	unsigned	header_length;	/* size of aio_ring */


	struct io_event		io_events[0];
}; /* 128 bytes + ring size */

#define AIO_RING_PAGES	8

70 71 72 73 74 75
struct kioctx_table {
	struct rcu_head	rcu;
	unsigned	nr;
	struct kioctx	*table[];
};

K
Kent Overstreet 已提交
76 77 78 79
struct kioctx_cpu {
	unsigned		reqs_available;
};

80 81 82 83 84
struct ctx_rq_wait {
	struct completion comp;
	atomic_t count;
};

85
struct kioctx {
K
Kent Overstreet 已提交
86
	struct percpu_ref	users;
K
Kent Overstreet 已提交
87
	atomic_t		dead;
88

K
Kent Overstreet 已提交
89 90
	struct percpu_ref	reqs;

91 92
	unsigned long		user_id;

K
Kent Overstreet 已提交
93 94 95 96 97 98 99
	struct __percpu kioctx_cpu *cpu;

	/*
	 * For percpu reqs_available, number of slots we move to/from global
	 * counter at a time:
	 */
	unsigned		req_batch;
100 101 102 103
	/*
	 * This is what userspace passed to io_setup(), it's not used for
	 * anything but counting against the global max_reqs quota.
	 *
K
Kent Overstreet 已提交
104
	 * The real limit is nr_events - 1, which will be larger (see
105 106
	 * aio_setup_ring())
	 */
107 108
	unsigned		max_reqs;

K
Kent Overstreet 已提交
109 110
	/* Size of ringbuffer, in units of struct io_event */
	unsigned		nr_events;
111

K
Kent Overstreet 已提交
112 113 114 115 116 117
	unsigned long		mmap_base;
	unsigned long		mmap_size;

	struct page		**ring_pages;
	long			nr_pages;

K
Kent Overstreet 已提交
118
	struct work_struct	free_work;
119

120 121 122
	/*
	 * signals when all in-flight requests are done
	 */
123
	struct ctx_rq_wait	*rq_wait;
124

125
	struct {
126 127 128 129 130
		/*
		 * This counts the number of available slots in the ringbuffer,
		 * so we avoid overflowing it: it's decremented (if positive)
		 * when allocating a kiocb and incremented when the resulting
		 * io_event is pulled off the ringbuffer.
K
Kent Overstreet 已提交
131 132
		 *
		 * We batch accesses to it with a percpu version.
133 134
		 */
		atomic_t	reqs_available;
135 136 137 138 139 140 141
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t	ctx_lock;
		struct list_head active_reqs;	/* used for cancellation */
	} ____cacheline_aligned_in_smp;

K
Kent Overstreet 已提交
142 143
	struct {
		struct mutex	ring_lock;
144 145
		wait_queue_head_t wait;
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
146 147 148

	struct {
		unsigned	tail;
149
		unsigned	completed_events;
K
Kent Overstreet 已提交
150
		spinlock_t	completion_lock;
151
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
152 153

	struct page		*internal_pages[AIO_RING_PAGES];
154
	struct file		*aio_ring_file;
155 156

	unsigned		id;
157 158
};

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
/*
 * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
 * cancelled or completed (this makes a certain amount of sense because
 * successful cancellation - io_cancel() - does deliver the completion to
 * userspace).
 *
 * And since most things don't implement kiocb cancellation and we'd really like
 * kiocb completion to be lockless when possible, we use ki_cancel to
 * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
 * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
 */
#define KIOCB_CANCELLED		((void *) (~0ULL))

struct aio_kiocb {
	struct kiocb		common;

	struct kioctx		*ki_ctx;
	kiocb_cancel_fn		*ki_cancel;

	struct iocb __user	*ki_user_iocb;	/* user's aiocb */
	__u64			ki_user_data;	/* user's data for completion */

	struct list_head	ki_list;	/* the aio core uses this
						 * for cancellation */

	/*
	 * If the aio_resfd field of the userspace iocb is not zero,
	 * this is the underlying eventfd context to deliver events to.
	 */
	struct eventfd_ctx	*ki_eventfd;
};

L
Linus Torvalds 已提交
191
/*------ sysctl variables----*/
192 193 194
static DEFINE_SPINLOCK(aio_nr_lock);
unsigned long aio_nr;		/* current system wide number of aio requests */
unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
L
Linus Torvalds 已提交
195 196
/*----end sysctl variables---*/

197 198
static struct kmem_cache	*kiocb_cachep;
static struct kmem_cache	*kioctx_cachep;
L
Linus Torvalds 已提交
199

200 201 202 203 204 205 206 207 208 209 210
static struct vfsmount *aio_mnt;

static const struct file_operations aio_ring_fops;
static const struct address_space_operations aio_ctx_aops;

static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
{
	struct qstr this = QSTR_INIT("[aio]", 5);
	struct file *file;
	struct path path;
	struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
211 212
	if (IS_ERR(inode))
		return ERR_CAST(inode);
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241

	inode->i_mapping->a_ops = &aio_ctx_aops;
	inode->i_mapping->private_data = ctx;
	inode->i_size = PAGE_SIZE * nr_pages;

	path.dentry = d_alloc_pseudo(aio_mnt->mnt_sb, &this);
	if (!path.dentry) {
		iput(inode);
		return ERR_PTR(-ENOMEM);
	}
	path.mnt = mntget(aio_mnt);

	d_instantiate(path.dentry, inode);
	file = alloc_file(&path, FMODE_READ | FMODE_WRITE, &aio_ring_fops);
	if (IS_ERR(file)) {
		path_put(&path);
		return file;
	}

	file->f_flags = O_RDWR;
	return file;
}

static struct dentry *aio_mount(struct file_system_type *fs_type,
				int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname	= simple_dname,
	};
J
Jann Horn 已提交
242 243 244 245 246 247
	struct dentry *root = mount_pseudo(fs_type, "aio:", NULL, &ops,
					   AIO_RING_MAGIC);

	if (!IS_ERR(root))
		root->d_sb->s_iflags |= SB_I_NOEXEC;
	return root;
248 249
}

L
Linus Torvalds 已提交
250 251 252 253 254 255
/* aio_setup
 *	Creates the slab caches used by the aio routines, panic on
 *	failure as this is done early during the boot sequence.
 */
static int __init aio_setup(void)
{
256 257 258 259 260 261 262 263 264
	static struct file_system_type aio_fs = {
		.name		= "aio",
		.mount		= aio_mount,
		.kill_sb	= kill_anon_super,
	};
	aio_mnt = kern_mount(&aio_fs);
	if (IS_ERR(aio_mnt))
		panic("Failed to create aio fs mount.");

265
	kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
266
	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
L
Linus Torvalds 已提交
267

K
Kent Overstreet 已提交
268
	pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
L
Linus Torvalds 已提交
269 270 271

	return 0;
}
272
__initcall(aio_setup);
L
Linus Torvalds 已提交
273

274 275 276
static void put_aio_ring_file(struct kioctx *ctx)
{
	struct file *aio_ring_file = ctx->aio_ring_file;
277 278
	struct address_space *i_mapping;

279
	if (aio_ring_file) {
A
Al Viro 已提交
280
		truncate_setsize(file_inode(aio_ring_file), 0);
281 282

		/* Prevent further access to the kioctx from migratepages */
A
Al Viro 已提交
283
		i_mapping = aio_ring_file->f_mapping;
284 285
		spin_lock(&i_mapping->private_lock);
		i_mapping->private_data = NULL;
286
		ctx->aio_ring_file = NULL;
287
		spin_unlock(&i_mapping->private_lock);
288 289 290 291 292

		fput(aio_ring_file);
	}
}

L
Linus Torvalds 已提交
293 294
static void aio_free_ring(struct kioctx *ctx)
{
295
	int i;
L
Linus Torvalds 已提交
296

297 298 299 300 301
	/* Disconnect the kiotx from the ring file.  This prevents future
	 * accesses to the kioctx from page migration.
	 */
	put_aio_ring_file(ctx);

302
	for (i = 0; i < ctx->nr_pages; i++) {
303
		struct page *page;
304 305
		pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
				page_count(ctx->ring_pages[i]));
306 307 308 309 310
		page = ctx->ring_pages[i];
		if (!page)
			continue;
		ctx->ring_pages[i] = NULL;
		put_page(page);
311
	}
L
Linus Torvalds 已提交
312

313
	if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
K
Kent Overstreet 已提交
314
		kfree(ctx->ring_pages);
315 316
		ctx->ring_pages = NULL;
	}
317 318
}

319
static int aio_ring_mremap(struct vm_area_struct *vma)
320
{
321
	struct file *file = vma->vm_file;
322 323
	struct mm_struct *mm = vma->vm_mm;
	struct kioctx_table *table;
A
Al Viro 已提交
324
	int i, res = -EINVAL;
325 326 327 328 329 330 331 332 333

	spin_lock(&mm->ioctx_lock);
	rcu_read_lock();
	table = rcu_dereference(mm->ioctx_table);
	for (i = 0; i < table->nr; i++) {
		struct kioctx *ctx;

		ctx = table->table[i];
		if (ctx && ctx->aio_ring_file == file) {
A
Al Viro 已提交
334 335 336 337
			if (!atomic_read(&ctx->dead)) {
				ctx->user_id = ctx->mmap_base = vma->vm_start;
				res = 0;
			}
338 339 340 341 342 343
			break;
		}
	}

	rcu_read_unlock();
	spin_unlock(&mm->ioctx_lock);
A
Al Viro 已提交
344
	return res;
345 346
}

347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
static const struct vm_operations_struct aio_ring_vm_ops = {
	.mremap		= aio_ring_mremap,
#if IS_ENABLED(CONFIG_MMU)
	.fault		= filemap_fault,
	.map_pages	= filemap_map_pages,
	.page_mkwrite	= filemap_page_mkwrite,
#endif
};

static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_flags |= VM_DONTEXPAND;
	vma->vm_ops = &aio_ring_vm_ops;
	return 0;
}

363 364 365 366
static const struct file_operations aio_ring_fops = {
	.mmap = aio_ring_mmap,
};

367
#if IS_ENABLED(CONFIG_MIGRATION)
368 369 370
static int aio_migratepage(struct address_space *mapping, struct page *new,
			struct page *old, enum migrate_mode mode)
{
371
	struct kioctx *ctx;
372
	unsigned long flags;
373
	pgoff_t idx;
374 375
	int rc;

376 377 378 379 380 381 382 383
	/*
	 * We cannot support the _NO_COPY case here, because copy needs to
	 * happen under the ctx->completion_lock. That does not work with the
	 * migration workflow of MIGRATE_SYNC_NO_COPY.
	 */
	if (mode == MIGRATE_SYNC_NO_COPY)
		return -EINVAL;

384 385
	rc = 0;

386
	/* mapping->private_lock here protects against the kioctx teardown.  */
387 388
	spin_lock(&mapping->private_lock);
	ctx = mapping->private_data;
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
	if (!ctx) {
		rc = -EINVAL;
		goto out;
	}

	/* The ring_lock mutex.  The prevents aio_read_events() from writing
	 * to the ring's head, and prevents page migration from mucking in
	 * a partially initialized kiotx.
	 */
	if (!mutex_trylock(&ctx->ring_lock)) {
		rc = -EAGAIN;
		goto out;
	}

	idx = old->index;
	if (idx < (pgoff_t)ctx->nr_pages) {
		/* Make sure the old page hasn't already been changed */
		if (ctx->ring_pages[idx] != old)
			rc = -EAGAIN;
408 409 410 411
	} else
		rc = -EINVAL;

	if (rc != 0)
412
		goto out_unlock;
413

414 415
	/* Writeback must be complete */
	BUG_ON(PageWriteback(old));
416
	get_page(new);
417

418
	rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1);
419
	if (rc != MIGRATEPAGE_SUCCESS) {
420
		put_page(new);
421
		goto out_unlock;
422 423
	}

424 425 426
	/* Take completion_lock to prevent other writes to the ring buffer
	 * while the old page is copied to the new.  This prevents new
	 * events from being lost.
427
	 */
428 429 430 431 432
	spin_lock_irqsave(&ctx->completion_lock, flags);
	migrate_page_copy(new, old);
	BUG_ON(ctx->ring_pages[idx] != old);
	ctx->ring_pages[idx] = new;
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
433

434 435
	/* The old page is no longer accessible. */
	put_page(old);
436

437 438 439 440
out_unlock:
	mutex_unlock(&ctx->ring_lock);
out:
	spin_unlock(&mapping->private_lock);
441
	return rc;
L
Linus Torvalds 已提交
442
}
443
#endif
L
Linus Torvalds 已提交
444

445
static const struct address_space_operations aio_ctx_aops = {
446
	.set_page_dirty = __set_page_dirty_no_writeback,
447
#if IS_ENABLED(CONFIG_MIGRATION)
448
	.migratepage	= aio_migratepage,
449
#endif
450 451
};

452
static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
L
Linus Torvalds 已提交
453 454
{
	struct aio_ring *ring;
Z
Zach Brown 已提交
455
	struct mm_struct *mm = current->mm;
456
	unsigned long size, unused;
L
Linus Torvalds 已提交
457
	int nr_pages;
458 459
	int i;
	struct file *file;
L
Linus Torvalds 已提交
460 461 462 463 464 465 466

	/* Compensate for the ring buffer's head/tail overlap entry */
	nr_events += 2;	/* 1 is required, 2 for good luck */

	size = sizeof(struct aio_ring);
	size += sizeof(struct io_event) * nr_events;

467
	nr_pages = PFN_UP(size);
L
Linus Torvalds 已提交
468 469 470
	if (nr_pages < 0)
		return -EINVAL;

471
	file = aio_private_file(ctx, nr_pages);
472 473
	if (IS_ERR(file)) {
		ctx->aio_ring_file = NULL;
474
		return -ENOMEM;
475 476
	}

477 478 479 480 481 482 483 484 485 486 487 488 489 490
	ctx->aio_ring_file = file;
	nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
			/ sizeof(struct io_event);

	ctx->ring_pages = ctx->internal_pages;
	if (nr_pages > AIO_RING_PAGES) {
		ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
					  GFP_KERNEL);
		if (!ctx->ring_pages) {
			put_aio_ring_file(ctx);
			return -ENOMEM;
		}
	}

491 492
	for (i = 0; i < nr_pages; i++) {
		struct page *page;
A
Al Viro 已提交
493
		page = find_or_create_page(file->f_mapping,
494 495 496 497 498 499 500
					   i, GFP_HIGHUSER | __GFP_ZERO);
		if (!page)
			break;
		pr_debug("pid(%d) page[%d]->count=%d\n",
			 current->pid, i, page_count(page));
		SetPageUptodate(page);
		unlock_page(page);
501 502

		ctx->ring_pages[i] = page;
503
	}
504
	ctx->nr_pages = i;
L
Linus Torvalds 已提交
505

506 507
	if (unlikely(i != nr_pages)) {
		aio_free_ring(ctx);
508
		return -ENOMEM;
L
Linus Torvalds 已提交
509 510
	}

K
Kent Overstreet 已提交
511 512
	ctx->mmap_size = nr_pages * PAGE_SIZE;
	pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
513

M
Michal Hocko 已提交
514 515 516 517 518 519
	if (down_write_killable(&mm->mmap_sem)) {
		ctx->mmap_size = 0;
		aio_free_ring(ctx);
		return -EINTR;
	}

520 521
	ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
				       PROT_READ | PROT_WRITE,
522
				       MAP_SHARED, 0, &unused, NULL);
523
	up_write(&mm->mmap_sem);
K
Kent Overstreet 已提交
524 525
	if (IS_ERR((void *)ctx->mmap_base)) {
		ctx->mmap_size = 0;
L
Linus Torvalds 已提交
526
		aio_free_ring(ctx);
527
		return -ENOMEM;
L
Linus Torvalds 已提交
528 529
	}

K
Kent Overstreet 已提交
530
	pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
531

K
Kent Overstreet 已提交
532 533
	ctx->user_id = ctx->mmap_base;
	ctx->nr_events = nr_events; /* trusted copy */
L
Linus Torvalds 已提交
534

K
Kent Overstreet 已提交
535
	ring = kmap_atomic(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
536
	ring->nr = nr_events;	/* user copy */
537
	ring->id = ~0U;
L
Linus Torvalds 已提交
538 539 540 541 542
	ring->head = ring->tail = 0;
	ring->magic = AIO_RING_MAGIC;
	ring->compat_features = AIO_RING_COMPAT_FEATURES;
	ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
	ring->header_length = sizeof(struct aio_ring);
543
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
544
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
545 546 547 548 549 550 551 552

	return 0;
}

#define AIO_EVENTS_PER_PAGE	(PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE	((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET	(AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)

553
void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
554
{
555
	struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common);
556 557 558 559 560 561 562 563 564 565 566 567 568 569
	struct kioctx *ctx = req->ki_ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->ctx_lock, flags);

	if (!req->ki_list.next)
		list_add(&req->ki_list, &ctx->active_reqs);

	req->ki_cancel = cancel;

	spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
EXPORT_SYMBOL(kiocb_set_cancel_fn);

570
static int kiocb_cancel(struct aio_kiocb *kiocb)
K
Kent Overstreet 已提交
571
{
572
	kiocb_cancel_fn *old, *cancel;
K
Kent Overstreet 已提交
573

574 575 576 577 578
	/*
	 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
	 * actually has a cancel function, hence the cmpxchg()
	 */

579
	cancel = READ_ONCE(kiocb->ki_cancel);
580 581
	do {
		if (!cancel || cancel == KIOCB_CANCELLED)
K
Kent Overstreet 已提交
582
			return -EINVAL;
K
Kent Overstreet 已提交
583

584 585 586
		old = cancel;
		cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
	} while (cancel != old);
K
Kent Overstreet 已提交
587

588
	return cancel(&kiocb->common);
K
Kent Overstreet 已提交
589 590
}

K
Kent Overstreet 已提交
591
static void free_ioctx(struct work_struct *work)
K
Kent Overstreet 已提交
592
{
K
Kent Overstreet 已提交
593
	struct kioctx *ctx = container_of(work, struct kioctx, free_work);
K
Kent Overstreet 已提交
594

K
Kent Overstreet 已提交
595
	pr_debug("freeing %p\n", ctx);
K
Kent Overstreet 已提交
596

K
Kent Overstreet 已提交
597
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
598
	free_percpu(ctx->cpu);
599 600
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
K
Kent Overstreet 已提交
601 602 603
	kmem_cache_free(kioctx_cachep, ctx);
}

K
Kent Overstreet 已提交
604 605 606 607
static void free_ioctx_reqs(struct percpu_ref *ref)
{
	struct kioctx *ctx = container_of(ref, struct kioctx, reqs);

608
	/* At this point we know that there are no any in-flight requests */
609 610
	if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
		complete(&ctx->rq_wait->comp);
611

K
Kent Overstreet 已提交
612 613 614 615
	INIT_WORK(&ctx->free_work, free_ioctx);
	schedule_work(&ctx->free_work);
}

K
Kent Overstreet 已提交
616 617 618 619 620
/*
 * When this function runs, the kioctx has been removed from the "hash table"
 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
 * now it's safe to cancel any that need to be.
 */
K
Kent Overstreet 已提交
621
static void free_ioctx_users(struct percpu_ref *ref)
K
Kent Overstreet 已提交
622
{
K
Kent Overstreet 已提交
623
	struct kioctx *ctx = container_of(ref, struct kioctx, users);
624
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
625 626 627 628 629

	spin_lock_irq(&ctx->ctx_lock);

	while (!list_empty(&ctx->active_reqs)) {
		req = list_first_entry(&ctx->active_reqs,
630
				       struct aio_kiocb, ki_list);
K
Kent Overstreet 已提交
631 632

		list_del_init(&req->ki_list);
633
		kiocb_cancel(req);
K
Kent Overstreet 已提交
634 635 636 637
	}

	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
638 639
	percpu_ref_kill(&ctx->reqs);
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
640 641
}

642 643 644 645 646 647 648
static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
{
	unsigned i, new_nr;
	struct kioctx_table *table, *old;
	struct aio_ring *ring;

	spin_lock(&mm->ioctx_lock);
649
	table = rcu_dereference_raw(mm->ioctx_table);
650 651 652 653 654 655 656 657 658

	while (1) {
		if (table)
			for (i = 0; i < table->nr; i++)
				if (!table->table[i]) {
					ctx->id = i;
					table->table[i] = ctx;
					spin_unlock(&mm->ioctx_lock);

659 660 661 662
					/* While kioctx setup is in progress,
					 * we are protected from page migration
					 * changes ring_pages by ->ring_lock.
					 */
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
					ring = kmap_atomic(ctx->ring_pages[0]);
					ring->id = ctx->id;
					kunmap_atomic(ring);
					return 0;
				}

		new_nr = (table ? table->nr : 1) * 4;
		spin_unlock(&mm->ioctx_lock);

		table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
				new_nr, GFP_KERNEL);
		if (!table)
			return -ENOMEM;

		table->nr = new_nr;

		spin_lock(&mm->ioctx_lock);
680
		old = rcu_dereference_raw(mm->ioctx_table);
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696

		if (!old) {
			rcu_assign_pointer(mm->ioctx_table, table);
		} else if (table->nr > old->nr) {
			memcpy(table->table, old->table,
			       old->nr * sizeof(struct kioctx *));

			rcu_assign_pointer(mm->ioctx_table, table);
			kfree_rcu(old, rcu);
		} else {
			kfree(table);
			table = old;
		}
	}
}

K
Kent Overstreet 已提交
697 698 699 700 701 702 703 704 705 706
static void aio_nr_sub(unsigned nr)
{
	spin_lock(&aio_nr_lock);
	if (WARN_ON(aio_nr - nr > aio_nr))
		aio_nr = 0;
	else
		aio_nr -= nr;
	spin_unlock(&aio_nr_lock);
}

L
Linus Torvalds 已提交
707 708 709 710 711
/* ioctx_alloc
 *	Allocates and initializes an ioctx.  Returns an ERR_PTR if it failed.
 */
static struct kioctx *ioctx_alloc(unsigned nr_events)
{
Z
Zach Brown 已提交
712
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
713
	struct kioctx *ctx;
714
	int err = -ENOMEM;
L
Linus Torvalds 已提交
715

716 717 718 719 720 721
	/*
	 * Store the original nr_events -- what userspace passed to io_setup(),
	 * for counting against the global limit -- before it changes.
	 */
	unsigned int max_reqs = nr_events;

K
Kent Overstreet 已提交
722 723 724 725 726 727 728 729 730 731 732 733
	/*
	 * We keep track of the number of available ringbuffer slots, to prevent
	 * overflow (reqs_available), and we also use percpu counters for this.
	 *
	 * So since up to half the slots might be on other cpu's percpu counters
	 * and unavailable, double nr_events so userspace sees what they
	 * expected: additionally, we move req_batch slots to/from percpu
	 * counters at a time, so make sure that isn't 0:
	 */
	nr_events = max(nr_events, num_possible_cpus() * 4);
	nr_events *= 2;

L
Linus Torvalds 已提交
734
	/* Prevent overflows */
A
Al Viro 已提交
735
	if (nr_events > (0x10000000U / sizeof(struct io_event))) {
L
Linus Torvalds 已提交
736 737 738 739
		pr_debug("ENOMEM: nr_events too high\n");
		return ERR_PTR(-EINVAL);
	}

740
	if (!nr_events || (unsigned long)max_reqs > aio_max_nr)
L
Linus Torvalds 已提交
741 742
		return ERR_PTR(-EAGAIN);

743
	ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
744 745 746
	if (!ctx)
		return ERR_PTR(-ENOMEM);

747
	ctx->max_reqs = max_reqs;
L
Linus Torvalds 已提交
748 749

	spin_lock_init(&ctx->ctx_lock);
750
	spin_lock_init(&ctx->completion_lock);
K
Kent Overstreet 已提交
751
	mutex_init(&ctx->ring_lock);
752 753 754
	/* Protect against page migration throughout kiotx setup by keeping
	 * the ring_lock mutex held until setup is complete. */
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
755 756 757 758
	init_waitqueue_head(&ctx->wait);

	INIT_LIST_HEAD(&ctx->active_reqs);

759
	if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
760 761
		goto err;

762
	if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
763 764
		goto err;

K
Kent Overstreet 已提交
765 766
	ctx->cpu = alloc_percpu(struct kioctx_cpu);
	if (!ctx->cpu)
K
Kent Overstreet 已提交
767
		goto err;
L
Linus Torvalds 已提交
768

769
	err = aio_setup_ring(ctx, nr_events);
770
	if (err < 0)
K
Kent Overstreet 已提交
771
		goto err;
K
Kent Overstreet 已提交
772

773
	atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
K
Kent Overstreet 已提交
774
	ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
775 776
	if (ctx->req_batch < 1)
		ctx->req_batch = 1;
777

L
Linus Torvalds 已提交
778
	/* limit the number of system wide aios */
779
	spin_lock(&aio_nr_lock);
780 781
	if (aio_nr + ctx->max_reqs > aio_max_nr ||
	    aio_nr + ctx->max_reqs < aio_nr) {
782
		spin_unlock(&aio_nr_lock);
K
Kent Overstreet 已提交
783
		err = -EAGAIN;
G
Gu Zheng 已提交
784
		goto err_ctx;
785 786
	}
	aio_nr += ctx->max_reqs;
787
	spin_unlock(&aio_nr_lock);
L
Linus Torvalds 已提交
788

789 790
	percpu_ref_get(&ctx->users);	/* io_setup() will drop this ref */
	percpu_ref_get(&ctx->reqs);	/* free_ioctx_users() will drop this */
K
Kent Overstreet 已提交
791

792 793
	err = ioctx_add_table(ctx, mm);
	if (err)
K
Kent Overstreet 已提交
794
		goto err_cleanup;
795

796 797 798
	/* Release the ring_lock mutex now that all setup is complete. */
	mutex_unlock(&ctx->ring_lock);

K
Kent Overstreet 已提交
799
	pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
K
Kent Overstreet 已提交
800
		 ctx, ctx->user_id, mm, ctx->nr_events);
L
Linus Torvalds 已提交
801 802
	return ctx;

K
Kent Overstreet 已提交
803 804
err_cleanup:
	aio_nr_sub(ctx->max_reqs);
G
Gu Zheng 已提交
805
err_ctx:
806 807 808
	atomic_set(&ctx->dead, 1);
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
G
Gu Zheng 已提交
809
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
810
err:
811
	mutex_unlock(&ctx->ring_lock);
K
Kent Overstreet 已提交
812
	free_percpu(ctx->cpu);
813 814
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
L
Linus Torvalds 已提交
815
	kmem_cache_free(kioctx_cachep, ctx);
K
Kent Overstreet 已提交
816
	pr_debug("error allocating ioctx %d\n", err);
817
	return ERR_PTR(err);
L
Linus Torvalds 已提交
818 819
}

K
Kent Overstreet 已提交
820 821 822 823 824
/* kill_ioctx
 *	Cancels all outstanding aio requests on an aio context.  Used
 *	when the processes owning a context have all exited to encourage
 *	the rapid destruction of the kioctx.
 */
825
static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
826
		      struct ctx_rq_wait *wait)
K
Kent Overstreet 已提交
827
{
828
	struct kioctx_table *table;
829

A
Al Viro 已提交
830 831 832
	spin_lock(&mm->ioctx_lock);
	if (atomic_xchg(&ctx->dead, 1)) {
		spin_unlock(&mm->ioctx_lock);
833
		return -EINVAL;
A
Al Viro 已提交
834
	}
835

836
	table = rcu_dereference_raw(mm->ioctx_table);
837 838 839
	WARN_ON(ctx != table->table[ctx->id]);
	table->table[ctx->id] = NULL;
	spin_unlock(&mm->ioctx_lock);
840

841 842
	/* percpu_ref_kill() will do the necessary call_rcu() */
	wake_up_all(&ctx->wait);
843

844 845 846 847 848 849 850 851
	/*
	 * It'd be more correct to do this in free_ioctx(), after all
	 * the outstanding kiocbs have finished - but by then io_destroy
	 * has already returned, so io_setup() could potentially return
	 * -EAGAIN with no ioctxs actually in use (as far as userspace
	 *  could tell).
	 */
	aio_nr_sub(ctx->max_reqs);
852

853 854
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
855

856
	ctx->rq_wait = wait;
857 858
	percpu_ref_kill(&ctx->users);
	return 0;
L
Linus Torvalds 已提交
859 860
}

K
Kent Overstreet 已提交
861 862 863 864 865 866 867
/*
 * exit_aio: called when the last user of mm goes away.  At this point, there is
 * no way for any new requests to be submited or any of the io_* syscalls to be
 * called on the context.
 *
 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
 * them.
L
Linus Torvalds 已提交
868
 */
869
void exit_aio(struct mm_struct *mm)
L
Linus Torvalds 已提交
870
{
871
	struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
872 873
	struct ctx_rq_wait wait;
	int i, skipped;
874

875 876
	if (!table)
		return;
877

878 879 880 881
	atomic_set(&wait.count, table->nr);
	init_completion(&wait.comp);

	skipped = 0;
882 883
	for (i = 0; i < table->nr; ++i) {
		struct kioctx *ctx = table->table[i];
J
Jens Axboe 已提交
884

885 886
		if (!ctx) {
			skipped++;
887
			continue;
888 889
		}

890
		/*
891 892 893 894 895
		 * We don't need to bother with munmap() here - exit_mmap(mm)
		 * is coming and it'll unmap everything. And we simply can't,
		 * this is not necessarily our ->mm.
		 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
		 * that it needs to unmap the area, just set it to 0.
896
		 */
K
Kent Overstreet 已提交
897
		ctx->mmap_size = 0;
898 899
		kill_ioctx(mm, ctx, &wait);
	}
K
Kent Overstreet 已提交
900

901
	if (!atomic_sub_and_test(skipped, &wait.count)) {
902
		/* Wait until all IO for the context are done. */
903
		wait_for_completion(&wait.comp);
L
Linus Torvalds 已提交
904
	}
905 906 907

	RCU_INIT_POINTER(mm->ioctx_table, NULL);
	kfree(table);
L
Linus Torvalds 已提交
908 909
}

K
Kent Overstreet 已提交
910 911 912
static void put_reqs_available(struct kioctx *ctx, unsigned nr)
{
	struct kioctx_cpu *kcpu;
913
	unsigned long flags;
K
Kent Overstreet 已提交
914

915
	local_irq_save(flags);
916
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
917
	kcpu->reqs_available += nr;
918

K
Kent Overstreet 已提交
919 920 921 922 923
	while (kcpu->reqs_available >= ctx->req_batch * 2) {
		kcpu->reqs_available -= ctx->req_batch;
		atomic_add(ctx->req_batch, &ctx->reqs_available);
	}

924
	local_irq_restore(flags);
K
Kent Overstreet 已提交
925 926 927 928 929 930
}

static bool get_reqs_available(struct kioctx *ctx)
{
	struct kioctx_cpu *kcpu;
	bool ret = false;
931
	unsigned long flags;
K
Kent Overstreet 已提交
932

933
	local_irq_save(flags);
934
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
	if (!kcpu->reqs_available) {
		int old, avail = atomic_read(&ctx->reqs_available);

		do {
			if (avail < ctx->req_batch)
				goto out;

			old = avail;
			avail = atomic_cmpxchg(&ctx->reqs_available,
					       avail, avail - ctx->req_batch);
		} while (avail != old);

		kcpu->reqs_available += ctx->req_batch;
	}

	ret = true;
	kcpu->reqs_available--;
out:
953
	local_irq_restore(flags);
K
Kent Overstreet 已提交
954 955 956
	return ret;
}

957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
/* refill_reqs_available
 *	Updates the reqs_available reference counts used for tracking the
 *	number of free slots in the completion ring.  This can be called
 *	from aio_complete() (to optimistically update reqs_available) or
 *	from aio_get_req() (the we're out of events case).  It must be
 *	called holding ctx->completion_lock.
 */
static void refill_reqs_available(struct kioctx *ctx, unsigned head,
                                  unsigned tail)
{
	unsigned events_in_ring, completed;

	/* Clamp head since userland can write to it. */
	head %= ctx->nr_events;
	if (head <= tail)
		events_in_ring = tail - head;
	else
		events_in_ring = ctx->nr_events - (head - tail);

	completed = ctx->completed_events;
	if (events_in_ring < completed)
		completed -= events_in_ring;
	else
		completed = 0;

	if (!completed)
		return;

	ctx->completed_events -= completed;
	put_reqs_available(ctx, completed);
}

/* user_refill_reqs_available
 *	Called to refill reqs_available when aio_get_req() encounters an
 *	out of space in the completion ring.
 */
static void user_refill_reqs_available(struct kioctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	if (ctx->completed_events) {
		struct aio_ring *ring;
		unsigned head;

		/* Access of ring->head may race with aio_read_events_ring()
		 * here, but that's okay since whether we read the old version
		 * or the new version, and either will be valid.  The important
		 * part is that head cannot pass tail since we prevent
		 * aio_complete() from updating tail by holding
		 * ctx->completion_lock.  Even if head is invalid, the check
		 * against ctx->completed_events below will make sure we do the
		 * safe/right thing.
		 */
		ring = kmap_atomic(ctx->ring_pages[0]);
		head = ring->head;
		kunmap_atomic(ring);

		refill_reqs_available(ctx, head, ctx->tail);
	}

	spin_unlock_irq(&ctx->completion_lock);
}

L
Linus Torvalds 已提交
1019
/* aio_get_req
K
Kent Overstreet 已提交
1020 1021
 *	Allocate a slot for an aio request.
 * Returns NULL if no requests are free.
L
Linus Torvalds 已提交
1022
 */
1023
static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
L
Linus Torvalds 已提交
1024
{
1025
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
1026

1027 1028 1029 1030 1031
	if (!get_reqs_available(ctx)) {
		user_refill_reqs_available(ctx);
		if (!get_reqs_available(ctx))
			return NULL;
	}
K
Kent Overstreet 已提交
1032

1033
	req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
L
Linus Torvalds 已提交
1034
	if (unlikely(!req))
K
Kent Overstreet 已提交
1035
		goto out_put;
L
Linus Torvalds 已提交
1036

K
Kent Overstreet 已提交
1037 1038
	percpu_ref_get(&ctx->reqs);

L
Linus Torvalds 已提交
1039
	req->ki_ctx = ctx;
J
Jeff Moyer 已提交
1040
	return req;
K
Kent Overstreet 已提交
1041
out_put:
K
Kent Overstreet 已提交
1042
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1043
	return NULL;
L
Linus Torvalds 已提交
1044 1045
}

1046
static void kiocb_free(struct aio_kiocb *req)
L
Linus Torvalds 已提交
1047
{
1048 1049
	if (req->common.ki_filp)
		fput(req->common.ki_filp);
1050 1051
	if (req->ki_eventfd != NULL)
		eventfd_ctx_put(req->ki_eventfd);
L
Linus Torvalds 已提交
1052 1053 1054
	kmem_cache_free(kiocb_cachep, req);
}

A
Adrian Bunk 已提交
1055
static struct kioctx *lookup_ioctx(unsigned long ctx_id)
L
Linus Torvalds 已提交
1056
{
1057
	struct aio_ring __user *ring  = (void __user *)ctx_id;
J
Jens Axboe 已提交
1058
	struct mm_struct *mm = current->mm;
1059
	struct kioctx *ctx, *ret = NULL;
1060 1061 1062 1063 1064
	struct kioctx_table *table;
	unsigned id;

	if (get_user(id, &ring->id))
		return NULL;
L
Linus Torvalds 已提交
1065

J
Jens Axboe 已提交
1066
	rcu_read_lock();
1067
	table = rcu_dereference(mm->ioctx_table);
J
Jens Axboe 已提交
1068

1069 1070
	if (!table || id >= table->nr)
		goto out;
L
Linus Torvalds 已提交
1071

1072
	ctx = table->table[id];
1073
	if (ctx && ctx->user_id == ctx_id) {
1074 1075 1076 1077
		percpu_ref_get(&ctx->users);
		ret = ctx;
	}
out:
J
Jens Axboe 已提交
1078
	rcu_read_unlock();
1079
	return ret;
L
Linus Torvalds 已提交
1080 1081 1082 1083 1084
}

/* aio_complete
 *	Called when the io request on the given iocb is complete.
 */
1085
static void aio_complete(struct kiocb *kiocb, long res, long res2)
L
Linus Torvalds 已提交
1086
{
1087
	struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, common);
L
Linus Torvalds 已提交
1088 1089
	struct kioctx	*ctx = iocb->ki_ctx;
	struct aio_ring	*ring;
K
Kent Overstreet 已提交
1090
	struct io_event	*ev_page, *event;
1091
	unsigned tail, pos, head;
L
Linus Torvalds 已提交
1092 1093
	unsigned long	flags;

1094 1095 1096 1097 1098 1099 1100
	if (kiocb->ki_flags & IOCB_WRITE) {
		struct file *file = kiocb->ki_filp;

		/*
		 * Tell lockdep we inherited freeze protection from submission
		 * thread.
		 */
S
Shaohua Li 已提交
1101 1102
		if (S_ISREG(file_inode(file)->i_mode))
			__sb_writers_acquired(file_inode(file)->i_sb, SB_FREEZE_WRITE);
1103 1104 1105
		file_end_write(file);
	}

1106 1107 1108 1109 1110 1111
	/*
	 * Special case handling for sync iocbs:
	 *  - events go directly into the iocb for fast handling
	 *  - the sync task with the iocb in its stack holds the single iocb
	 *    ref, no other paths have a way to get another ref
	 *  - the sync task helpfully left a reference to itself in the iocb
L
Linus Torvalds 已提交
1112
	 */
1113
	BUG_ON(is_sync_kiocb(kiocb));
L
Linus Torvalds 已提交
1114

1115 1116 1117 1118 1119 1120 1121
	if (iocb->ki_list.next) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->ctx_lock, flags);
		list_del(&iocb->ki_list);
		spin_unlock_irqrestore(&ctx->ctx_lock, flags);
	}
1122

1123 1124
	/*
	 * Add a completion event to the ring buffer. Must be done holding
1125
	 * ctx->completion_lock to prevent other code from messing with the tail
1126 1127 1128 1129
	 * pointer since we might be called from irq context.
	 */
	spin_lock_irqsave(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1130
	tail = ctx->tail;
K
Kent Overstreet 已提交
1131 1132
	pos = tail + AIO_EVENTS_OFFSET;

K
Kent Overstreet 已提交
1133
	if (++tail >= ctx->nr_events)
1134
		tail = 0;
L
Linus Torvalds 已提交
1135

K
Kent Overstreet 已提交
1136
	ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1137 1138
	event = ev_page + pos % AIO_EVENTS_PER_PAGE;

1139
	event->obj = (u64)(unsigned long)iocb->ki_user_iocb;
L
Linus Torvalds 已提交
1140 1141 1142 1143
	event->data = iocb->ki_user_data;
	event->res = res;
	event->res2 = res2;

K
Kent Overstreet 已提交
1144
	kunmap_atomic(ev_page);
K
Kent Overstreet 已提交
1145
	flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1146 1147

	pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
1148
		 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data,
K
Kent Overstreet 已提交
1149
		 res, res2);
L
Linus Torvalds 已提交
1150 1151 1152 1153 1154 1155

	/* after flagging the request as done, we
	 * must never even look at it again
	 */
	smp_wmb();	/* make event visible before updating tail */

K
Kent Overstreet 已提交
1156
	ctx->tail = tail;
L
Linus Torvalds 已提交
1157

K
Kent Overstreet 已提交
1158
	ring = kmap_atomic(ctx->ring_pages[0]);
1159
	head = ring->head;
K
Kent Overstreet 已提交
1160
	ring->tail = tail;
1161
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1162
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
1163

1164 1165 1166
	ctx->completed_events++;
	if (ctx->completed_events > 1)
		refill_reqs_available(ctx, head, tail);
1167 1168
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1169
	pr_debug("added to ring %p at [%u]\n", iocb, tail);
D
Davide Libenzi 已提交
1170 1171 1172 1173 1174 1175

	/*
	 * Check if the user asked us to deliver the result through an
	 * eventfd. The eventfd_signal() function is safe to be called
	 * from IRQ context.
	 */
1176
	if (iocb->ki_eventfd != NULL)
D
Davide Libenzi 已提交
1177 1178
		eventfd_signal(iocb->ki_eventfd, 1);

L
Linus Torvalds 已提交
1179
	/* everything turned out well, dispose of the aiocb. */
K
Kent Overstreet 已提交
1180
	kiocb_free(iocb);
L
Linus Torvalds 已提交
1181

1182 1183 1184 1185 1186 1187 1188 1189
	/*
	 * We have to order our ring_info tail store above and test
	 * of the wait list below outside the wait lock.  This is
	 * like in wake_up_bit() where clearing a bit has to be
	 * ordered with the unlocked test.
	 */
	smp_mb();

L
Linus Torvalds 已提交
1190 1191 1192
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);

K
Kent Overstreet 已提交
1193
	percpu_ref_put(&ctx->reqs);
L
Linus Torvalds 已提交
1194 1195
}

G
Gu Zheng 已提交
1196
/* aio_read_events_ring
1197 1198
 *	Pull an event off of the ioctx's event ring.  Returns the number of
 *	events fetched
L
Linus Torvalds 已提交
1199
 */
1200 1201
static long aio_read_events_ring(struct kioctx *ctx,
				 struct io_event __user *event, long nr)
L
Linus Torvalds 已提交
1202 1203
{
	struct aio_ring *ring;
1204
	unsigned head, tail, pos;
1205 1206 1207
	long ret = 0;
	int copy_ret;

1208 1209 1210 1211 1212 1213 1214
	/*
	 * The mutex can block and wake us up and that will cause
	 * wait_event_interruptible_hrtimeout() to schedule without sleeping
	 * and repeat. This should be rare enough that it doesn't cause
	 * peformance issues. See the comment in read_events() for more detail.
	 */
	sched_annotate_sleep();
K
Kent Overstreet 已提交
1215
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
1216

1217
	/* Access to ->ring_pages here is protected by ctx->ring_lock. */
K
Kent Overstreet 已提交
1218
	ring = kmap_atomic(ctx->ring_pages[0]);
1219
	head = ring->head;
1220
	tail = ring->tail;
1221 1222
	kunmap_atomic(ring);

1223 1224 1225 1226 1227 1228
	/*
	 * Ensure that once we've read the current tail pointer, that
	 * we also see the events that were stored up to the tail.
	 */
	smp_rmb();

1229
	pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
L
Linus Torvalds 已提交
1230

1231
	if (head == tail)
L
Linus Torvalds 已提交
1232 1233
		goto out;

1234 1235 1236
	head %= ctx->nr_events;
	tail %= ctx->nr_events;

1237 1238 1239 1240 1241
	while (ret < nr) {
		long avail;
		struct io_event *ev;
		struct page *page;

1242 1243
		avail = (head <= tail ?  tail : ctx->nr_events) - head;
		if (head == tail)
1244 1245 1246 1247 1248 1249 1250
			break;

		avail = min(avail, nr - ret);
		avail = min_t(long, avail, AIO_EVENTS_PER_PAGE -
			    ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE));

		pos = head + AIO_EVENTS_OFFSET;
K
Kent Overstreet 已提交
1251
		page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
		pos %= AIO_EVENTS_PER_PAGE;

		ev = kmap(page);
		copy_ret = copy_to_user(event + ret, ev + pos,
					sizeof(*ev) * avail);
		kunmap(page);

		if (unlikely(copy_ret)) {
			ret = -EFAULT;
			goto out;
		}

		ret += avail;
		head += avail;
K
Kent Overstreet 已提交
1266
		head %= ctx->nr_events;
L
Linus Torvalds 已提交
1267 1268
	}

K
Kent Overstreet 已提交
1269
	ring = kmap_atomic(ctx->ring_pages[0]);
1270
	ring->head = head;
1271
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1272
	flush_dcache_page(ctx->ring_pages[0]);
1273

1274
	pr_debug("%li  h%u t%u\n", ret, head, tail);
1275
out:
K
Kent Overstreet 已提交
1276
	mutex_unlock(&ctx->ring_lock);
1277

L
Linus Torvalds 已提交
1278 1279 1280
	return ret;
}

1281 1282
static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
			    struct io_event __user *event, long *i)
L
Linus Torvalds 已提交
1283
{
1284
	long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
L
Linus Torvalds 已提交
1285

1286 1287
	if (ret > 0)
		*i += ret;
L
Linus Torvalds 已提交
1288

1289 1290
	if (unlikely(atomic_read(&ctx->dead)))
		ret = -EINVAL;
L
Linus Torvalds 已提交
1291

1292 1293
	if (!*i)
		*i = ret;
L
Linus Torvalds 已提交
1294

1295
	return ret < 0 || *i >= min_nr;
L
Linus Torvalds 已提交
1296 1297
}

1298
static long read_events(struct kioctx *ctx, long min_nr, long nr,
L
Linus Torvalds 已提交
1299
			struct io_event __user *event,
1300
			ktime_t until)
L
Linus Torvalds 已提交
1301
{
1302
	long ret = 0;
L
Linus Torvalds 已提交
1303

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
	/*
	 * Note that aio_read_events() is being called as the conditional - i.e.
	 * we're calling it after prepare_to_wait() has set task state to
	 * TASK_INTERRUPTIBLE.
	 *
	 * But aio_read_events() can block, and if it blocks it's going to flip
	 * the task state back to TASK_RUNNING.
	 *
	 * This should be ok, provided it doesn't flip the state back to
	 * TASK_RUNNING and return 0 too much - that causes us to spin. That
	 * will only happen if the mutex_lock() call blocks, and we then find
	 * the ringbuffer empty. So in practice we should be ok, but it's
	 * something to be aware of when touching this code.
	 */
T
Thomas Gleixner 已提交
1318
	if (until == 0)
1319 1320 1321 1322 1323
		aio_read_events(ctx, min_nr, nr, event, &ret);
	else
		wait_event_interruptible_hrtimeout(ctx->wait,
				aio_read_events(ctx, min_nr, nr, event, &ret),
				until);
L
Linus Torvalds 已提交
1324

1325 1326
	if (!ret && signal_pending(current))
		ret = -EINTR;
L
Linus Torvalds 已提交
1327

1328
	return ret;
L
Linus Torvalds 已提交
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343
}

/* sys_io_setup:
 *	Create an aio_context capable of receiving at least nr_events.
 *	ctxp must not point to an aio_context that already exists, and
 *	must be initialized to 0 prior to the call.  On successful
 *	creation of the aio_context, *ctxp is filled in with the resulting 
 *	handle.  May fail with -EINVAL if *ctxp is not initialized,
 *	if the specified nr_events exceeds internal limits.  May fail 
 *	with -EAGAIN if the specified nr_events exceeds the user's limit 
 *	of available events.  May fail with -ENOMEM if insufficient kernel
 *	resources are available.  May fail with -EFAULT if an invalid
 *	pointer is passed for ctxp.  Will fail with -ENOSYS if not
 *	implemented.
 */
1344
SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
L
Linus Torvalds 已提交
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
{
	struct kioctx *ioctx = NULL;
	unsigned long ctx;
	long ret;

	ret = get_user(ctx, ctxp);
	if (unlikely(ret))
		goto out;

	ret = -EINVAL;
1355
	if (unlikely(ctx || nr_events == 0)) {
1356
		pr_debug("EINVAL: ctx %lu nr_events %u\n",
1357
		         ctx, nr_events);
L
Linus Torvalds 已提交
1358 1359 1360 1361 1362 1363 1364
		goto out;
	}

	ioctx = ioctx_alloc(nr_events);
	ret = PTR_ERR(ioctx);
	if (!IS_ERR(ioctx)) {
		ret = put_user(ioctx->user_id, ctxp);
1365
		if (ret)
1366
			kill_ioctx(current->mm, ioctx, NULL);
K
Kent Overstreet 已提交
1367
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1368 1369 1370 1371 1372 1373
	}

out:
	return ret;
}

A
Al Viro 已提交
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p)
{
	struct kioctx *ioctx = NULL;
	unsigned long ctx;
	long ret;

	ret = get_user(ctx, ctx32p);
	if (unlikely(ret))
		goto out;

	ret = -EINVAL;
	if (unlikely(ctx || nr_events == 0)) {
		pr_debug("EINVAL: ctx %lu nr_events %u\n",
		         ctx, nr_events);
		goto out;
	}

	ioctx = ioctx_alloc(nr_events);
	ret = PTR_ERR(ioctx);
	if (!IS_ERR(ioctx)) {
		/* truncating is ok because it's a user address */
		ret = put_user((u32)ioctx->user_id, ctx32p);
		if (ret)
			kill_ioctx(current->mm, ioctx, NULL);
		percpu_ref_put(&ioctx->users);
	}

out:
	return ret;
}
#endif

L
Linus Torvalds 已提交
1407 1408 1409
/* sys_io_destroy:
 *	Destroy the aio_context specified.  May cancel any outstanding 
 *	AIOs and block on completion.  Will fail with -ENOSYS if not
1410
 *	implemented.  May fail with -EINVAL if the context pointed to
L
Linus Torvalds 已提交
1411 1412
 *	is invalid.
 */
1413
SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
L
Linus Torvalds 已提交
1414 1415 1416
{
	struct kioctx *ioctx = lookup_ioctx(ctx);
	if (likely(NULL != ioctx)) {
1417
		struct ctx_rq_wait wait;
1418
		int ret;
1419

1420 1421 1422
		init_completion(&wait.comp);
		atomic_set(&wait.count, 1);

1423 1424 1425 1426
		/* Pass requests_done to kill_ioctx() where it can be set
		 * in a thread-safe way. If we try to set it here then we have
		 * a race condition if two io_destroy() called simultaneously.
		 */
1427
		ret = kill_ioctx(current->mm, ioctx, &wait);
K
Kent Overstreet 已提交
1428
		percpu_ref_put(&ioctx->users);
1429 1430 1431 1432 1433

		/* Wait until all IO for the context are done. Otherwise kernel
		 * keep using user-space buffers even if user thinks the context
		 * is destroyed.
		 */
1434
		if (!ret)
1435
			wait_for_completion(&wait.comp);
1436

1437
		return ret;
L
Linus Torvalds 已提交
1438
	}
1439
	pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1440 1441 1442
	return -EINVAL;
}

C
Christoph Hellwig 已提交
1443 1444
static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec,
		bool vectored, bool compat, struct iov_iter *iter)
B
Badari Pulavarty 已提交
1445
{
C
Christoph Hellwig 已提交
1446 1447 1448 1449 1450 1451 1452 1453
	void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;
	size_t len = iocb->aio_nbytes;

	if (!vectored) {
		ssize_t ret = import_single_range(rw, buf, len, *iovec, iter);
		*iovec = NULL;
		return ret;
	}
1454 1455
#ifdef CONFIG_COMPAT
	if (compat)
C
Christoph Hellwig 已提交
1456 1457
		return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec,
				iter);
1458
#endif
C
Christoph Hellwig 已提交
1459
	return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter);
B
Badari Pulavarty 已提交
1460 1461
}

C
Christoph Hellwig 已提交
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484
static inline ssize_t aio_ret(struct kiocb *req, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		return ret;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		ret = -EINTR;
		/*FALLTHRU*/
	default:
		aio_complete(req, ret, 0);
		return 0;
	}
}

static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored,
		bool compat)
L
Linus Torvalds 已提交
1485
{
K
Kent Overstreet 已提交
1486
	struct file *file = req->ki_filp;
1487
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1488
	struct iov_iter iter;
C
Christoph Hellwig 已提交
1489
	ssize_t ret;
L
Linus Torvalds 已提交
1490

C
Christoph Hellwig 已提交
1491 1492 1493 1494
	if (unlikely(!(file->f_mode & FMODE_READ)))
		return -EBADF;
	if (unlikely(!file->f_op->read_iter))
		return -EINVAL;
K
Kent Overstreet 已提交
1495

C
Christoph Hellwig 已提交
1496 1497 1498 1499 1500
	ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
	if (ret)
		return ret;
	ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
	if (!ret)
1501
		ret = aio_ret(req, call_read_iter(file, req, &iter));
C
Christoph Hellwig 已提交
1502 1503 1504
	kfree(iovec);
	return ret;
}
K
Kent Overstreet 已提交
1505

C
Christoph Hellwig 已提交
1506 1507 1508 1509 1510 1511 1512
static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored,
		bool compat)
{
	struct file *file = req->ki_filp;
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct iov_iter iter;
	ssize_t ret;
K
Kent Overstreet 已提交
1513

C
Christoph Hellwig 已提交
1514 1515 1516
	if (unlikely(!(file->f_mode & FMODE_WRITE)))
		return -EBADF;
	if (unlikely(!file->f_op->write_iter))
K
Kent Overstreet 已提交
1517
		return -EINVAL;
L
Linus Torvalds 已提交
1518

C
Christoph Hellwig 已提交
1519 1520 1521 1522 1523
	ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
	if (ret)
		return ret;
	ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
	if (!ret) {
1524
		req->ki_flags |= IOCB_WRITE;
C
Christoph Hellwig 已提交
1525
		file_start_write(file);
1526
		ret = aio_ret(req, call_write_iter(file, req, &iter));
1527 1528 1529 1530 1531
		/*
		 * We release freeze protection in aio_complete().  Fool lockdep
		 * by telling it the lock got released so that it doesn't
		 * complain about held lock when we return to userspace.
		 */
S
Shaohua Li 已提交
1532 1533
		if (S_ISREG(file_inode(file)->i_mode))
			__sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE);
K
Kent Overstreet 已提交
1534
	}
C
Christoph Hellwig 已提交
1535 1536
	kfree(iovec);
	return ret;
L
Linus Torvalds 已提交
1537 1538
}

A
Adrian Bunk 已提交
1539
static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
K
Kent Overstreet 已提交
1540
			 struct iocb *iocb, bool compat)
L
Linus Torvalds 已提交
1541
{
1542
	struct aio_kiocb *req;
C
Christoph Hellwig 已提交
1543
	struct file *file;
L
Linus Torvalds 已提交
1544 1545 1546
	ssize_t ret;

	/* enforce forwards compatibility on users */
1547
	if (unlikely(iocb->aio_reserved2)) {
K
Kent Overstreet 已提交
1548
		pr_debug("EINVAL: reserve field set\n");
L
Linus Torvalds 已提交
1549 1550 1551 1552 1553 1554 1555 1556 1557
		return -EINVAL;
	}

	/* prevent overflows */
	if (unlikely(
	    (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
	    (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
	    ((ssize_t)iocb->aio_nbytes < 0)
	   )) {
1558
		pr_debug("EINVAL: overflow check\n");
L
Linus Torvalds 已提交
1559 1560 1561
		return -EINVAL;
	}

K
Kent Overstreet 已提交
1562
	req = aio_get_req(ctx);
1563
	if (unlikely(!req))
L
Linus Torvalds 已提交
1564
		return -EAGAIN;
1565

C
Christoph Hellwig 已提交
1566
	req->common.ki_filp = file = fget(iocb->aio_fildes);
1567
	if (unlikely(!req->common.ki_filp)) {
1568 1569
		ret = -EBADF;
		goto out_put_req;
L
Linus Torvalds 已提交
1570
	}
1571 1572
	req->common.ki_pos = iocb->aio_offset;
	req->common.ki_complete = aio_complete;
1573
	req->common.ki_flags = iocb_flags(req->common.ki_filp);
1574
	req->common.ki_hint = file_write_hint(file);
1575

1576 1577 1578 1579 1580 1581 1582
	if (iocb->aio_flags & IOCB_FLAG_RESFD) {
		/*
		 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
		 * instance of the file* now. The file descriptor must be
		 * an eventfd() fd, and will be signaled for each completed
		 * event using the eventfd_signal() function.
		 */
1583
		req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd);
1584
		if (IS_ERR(req->ki_eventfd)) {
1585
			ret = PTR_ERR(req->ki_eventfd);
1586
			req->ki_eventfd = NULL;
1587 1588
			goto out_put_req;
		}
1589 1590

		req->common.ki_flags |= IOCB_EVENTFD;
1591
	}
L
Linus Torvalds 已提交
1592

1593 1594 1595 1596 1597 1598
	ret = kiocb_set_rw_flags(&req->common, iocb->aio_rw_flags);
	if (unlikely(ret)) {
		pr_debug("EINVAL: aio_rw_flags\n");
		goto out_put_req;
	}

K
Kent Overstreet 已提交
1599
	ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
L
Linus Torvalds 已提交
1600
	if (unlikely(ret)) {
K
Kent Overstreet 已提交
1601
		pr_debug("EFAULT: aio_key\n");
L
Linus Torvalds 已提交
1602 1603 1604
		goto out_put_req;
	}

1605
	req->ki_user_iocb = user_iocb;
L
Linus Torvalds 已提交
1606 1607
	req->ki_user_data = iocb->aio_data;

C
Christoph Hellwig 已提交
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627
	get_file(file);
	switch (iocb->aio_lio_opcode) {
	case IOCB_CMD_PREAD:
		ret = aio_read(&req->common, iocb, false, compat);
		break;
	case IOCB_CMD_PWRITE:
		ret = aio_write(&req->common, iocb, false, compat);
		break;
	case IOCB_CMD_PREADV:
		ret = aio_read(&req->common, iocb, true, compat);
		break;
	case IOCB_CMD_PWRITEV:
		ret = aio_write(&req->common, iocb, true, compat);
		break;
	default:
		pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode);
		ret = -EINVAL;
		break;
	}
	fput(file);
Z
Zach Brown 已提交
1628

C
Christoph Hellwig 已提交
1629 1630
	if (ret && ret != -EIOCBQUEUED)
		goto out_put_req;
L
Linus Torvalds 已提交
1631 1632
	return 0;
out_put_req:
K
Kent Overstreet 已提交
1633
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1634
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
1635
	kiocb_free(req);
L
Linus Torvalds 已提交
1636 1637 1638
	return ret;
}

A
Al Viro 已提交
1639 1640
static long do_io_submit(aio_context_t ctx_id, long nr,
			  struct iocb __user *__user *iocbpp, bool compat)
L
Linus Torvalds 已提交
1641 1642 1643
{
	struct kioctx *ctx;
	long ret = 0;
J
Jeff Moyer 已提交
1644
	int i = 0;
S
Shaohua Li 已提交
1645
	struct blk_plug plug;
L
Linus Torvalds 已提交
1646 1647 1648 1649

	if (unlikely(nr < 0))
		return -EINVAL;

1650 1651 1652
	if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
		nr = LONG_MAX/sizeof(*iocbpp);

L
Linus Torvalds 已提交
1653 1654 1655 1656 1657
	if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx)) {
K
Kent Overstreet 已提交
1658
		pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1659 1660 1661
		return -EINVAL;
	}

S
Shaohua Li 已提交
1662 1663
	blk_start_plug(&plug);

L
Linus Torvalds 已提交
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681
	/*
	 * AKPM: should this return a partial result if some of the IOs were
	 * successfully submitted?
	 */
	for (i=0; i<nr; i++) {
		struct iocb __user *user_iocb;
		struct iocb tmp;

		if (unlikely(__get_user(user_iocb, iocbpp + i))) {
			ret = -EFAULT;
			break;
		}

		if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
			ret = -EFAULT;
			break;
		}

K
Kent Overstreet 已提交
1682
		ret = io_submit_one(ctx, user_iocb, &tmp, compat);
L
Linus Torvalds 已提交
1683 1684 1685
		if (ret)
			break;
	}
S
Shaohua Li 已提交
1686
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
1687

K
Kent Overstreet 已提交
1688
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1689 1690 1691
	return i ? i : ret;
}

1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
/* sys_io_submit:
 *	Queue the nr iocbs pointed to by iocbpp for processing.  Returns
 *	the number of iocbs queued.  May return -EINVAL if the aio_context
 *	specified by ctx_id is invalid, if nr is < 0, if the iocb at
 *	*iocbpp[0] is not properly initialized, if the operation specified
 *	is invalid for the file descriptor in the iocb.  May fail with
 *	-EFAULT if any of the data structures point to invalid data.  May
 *	fail with -EBADF if the file descriptor specified in the first
 *	iocb is invalid.  May fail with -EAGAIN if insufficient resources
 *	are available to queue any iocbs.  Will return 0 if nr is 0.  Will
 *	fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
		struct iocb __user * __user *, iocbpp)
{
	return do_io_submit(ctx_id, nr, iocbpp, 0);
}

A
Al Viro 已提交
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
#ifdef CONFIG_COMPAT
static inline long
copy_iocb(long nr, u32 __user *ptr32, struct iocb __user * __user *ptr64)
{
	compat_uptr_t uptr;
	int i;

	for (i = 0; i < nr; ++i) {
		if (get_user(uptr, ptr32 + i))
			return -EFAULT;
		if (put_user(compat_ptr(uptr), ptr64 + i))
			return -EFAULT;
	}
	return 0;
}

#define MAX_AIO_SUBMITS 	(PAGE_SIZE/sizeof(struct iocb *))

COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,
		       int, nr, u32 __user *, iocb)
{
	struct iocb __user * __user *iocb64;
	long ret;

	if (unlikely(nr < 0))
		return -EINVAL;

	if (nr > MAX_AIO_SUBMITS)
		nr = MAX_AIO_SUBMITS;

	iocb64 = compat_alloc_user_space(nr * sizeof(*iocb64));
	ret = copy_iocb(nr, iocb, iocb64);
	if (!ret)
		ret = do_io_submit(ctx_id, nr, iocb64, 1);
	return ret;
}
#endif

L
Linus Torvalds 已提交
1748 1749 1750
/* lookup_kiocb
 *	Finds a given iocb for cancellation.
 */
1751 1752
static struct aio_kiocb *
lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb, u32 key)
L
Linus Torvalds 已提交
1753
{
1754
	struct aio_kiocb *kiocb;
1755 1756 1757

	assert_spin_locked(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1758 1759 1760
	if (key != KIOCB_KEY)
		return NULL;

L
Linus Torvalds 已提交
1761
	/* TODO: use a hash or array, this sucks. */
1762 1763
	list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
		if (kiocb->ki_user_iocb == iocb)
L
Linus Torvalds 已提交
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
			return kiocb;
	}
	return NULL;
}

/* sys_io_cancel:
 *	Attempts to cancel an iocb previously passed to io_submit.  If
 *	the operation is successfully cancelled, the resulting event is
 *	copied into the memory pointed to by result without being placed
 *	into the completion queue and 0 is returned.  May fail with
 *	-EFAULT if any of the data structures pointed to are invalid.
 *	May fail with -EINVAL if aio_context specified by ctx_id is
 *	invalid.  May fail with -EAGAIN if the iocb specified was not
 *	cancelled.  Will fail with -ENOSYS if not implemented.
 */
1779 1780
SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
		struct io_event __user *, result)
L
Linus Torvalds 已提交
1781 1782
{
	struct kioctx *ctx;
1783
	struct aio_kiocb *kiocb;
L
Linus Torvalds 已提交
1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
	u32 key;
	int ret;

	ret = get_user(key, &iocb->aio_key);
	if (unlikely(ret))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx))
		return -EINVAL;

	spin_lock_irq(&ctx->ctx_lock);
K
Kent Overstreet 已提交
1796

L
Linus Torvalds 已提交
1797
	kiocb = lookup_kiocb(ctx, iocb, key);
K
Kent Overstreet 已提交
1798
	if (kiocb)
1799
		ret = kiocb_cancel(kiocb);
K
Kent Overstreet 已提交
1800 1801 1802
	else
		ret = -EINVAL;

L
Linus Torvalds 已提交
1803 1804
	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1805
	if (!ret) {
1806 1807 1808 1809
		/*
		 * The result argument is no longer used - the io_event is
		 * always delivered via the ring buffer. -EINPROGRESS indicates
		 * cancellation is progress:
K
Kent Overstreet 已提交
1810
		 */
1811
		ret = -EINPROGRESS;
K
Kent Overstreet 已提交
1812
	}
L
Linus Torvalds 已提交
1813

K
Kent Overstreet 已提交
1814
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1815 1816 1817 1818

	return ret;
}

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
static long do_io_getevents(aio_context_t ctx_id,
		long min_nr,
		long nr,
		struct io_event __user *events,
		struct timespec64 *ts)
{
	ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX;
	struct kioctx *ioctx = lookup_ioctx(ctx_id);
	long ret = -EINVAL;

	if (likely(ioctx)) {
		if (likely(min_nr <= nr && min_nr >= 0))
			ret = read_events(ioctx, min_nr, nr, events, until);
		percpu_ref_put(&ioctx->users);
	}

	return ret;
}

L
Linus Torvalds 已提交
1838 1839
/* io_getevents:
 *	Attempts to read at least min_nr events and up to nr events from
1840 1841 1842 1843 1844 1845 1846 1847
 *	the completion queue for the aio_context specified by ctx_id. If
 *	it succeeds, the number of read events is returned. May fail with
 *	-EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
 *	out of range, if timeout is out of range.  May fail with -EFAULT
 *	if any of the memory specified is invalid.  May return 0 or
 *	< min_nr if the timeout specified by timeout has elapsed
 *	before sufficient events are available, where timeout == NULL
 *	specifies an infinite timeout. Note that the timeout pointed to by
J
Jeff Moyer 已提交
1848
 *	timeout is relative.  Will fail with -ENOSYS if not implemented.
L
Linus Torvalds 已提交
1849
 */
1850 1851 1852 1853 1854
SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
		long, min_nr,
		long, nr,
		struct io_event __user *, events,
		struct timespec __user *, timeout)
L
Linus Torvalds 已提交
1855
{
1856
	struct timespec64	ts;
L
Linus Torvalds 已提交
1857

1858 1859 1860
	if (timeout) {
		if (unlikely(get_timespec64(&ts, timeout)))
			return -EFAULT;
L
Linus Torvalds 已提交
1861
	}
1862 1863

	return do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
L
Linus Torvalds 已提交
1864
}
A
Al Viro 已提交
1865 1866 1867 1868 1869 1870 1871 1872

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE5(io_getevents, compat_aio_context_t, ctx_id,
		       compat_long_t, min_nr,
		       compat_long_t, nr,
		       struct io_event __user *, events,
		       struct compat_timespec __user *, timeout)
{
1873
	struct timespec64 t;
A
Al Viro 已提交
1874 1875

	if (timeout) {
1876
		if (compat_get_timespec64(&t, timeout))
A
Al Viro 已提交
1877 1878 1879
			return -EFAULT;

	}
1880 1881

	return do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
A
Al Viro 已提交
1882 1883
}
#endif