aio.c 42.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 *	An async IO implementation for Linux
 *	Written by Benjamin LaHaise <bcrl@kvack.org>
 *
 *	Implements an efficient asynchronous io interface.
 *
 *	Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved.
 *
 *	See ../COPYING for licensing terms.
 */
K
Kent Overstreet 已提交
11 12
#define pr_fmt(fmt) "%s: " fmt, __func__

L
Linus Torvalds 已提交
13 14 15 16 17
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/aio_abi.h>
18
#include <linux/export.h>
L
Linus Torvalds 已提交
19
#include <linux/syscalls.h>
20
#include <linux/backing-dev.h>
21
#include <linux/uio.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
28
#include <linux/mmu_context.h>
K
Kent Overstreet 已提交
29
#include <linux/percpu.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/aio.h>
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
36
#include <linux/eventfd.h>
J
Jeff Moyer 已提交
37
#include <linux/blkdev.h>
38
#include <linux/compat.h>
39 40
#include <linux/migrate.h>
#include <linux/ramfs.h>
K
Kent Overstreet 已提交
41
#include <linux/percpu-refcount.h>
42
#include <linux/mount.h>
L
Linus Torvalds 已提交
43 44 45 46

#include <asm/kmap_types.h>
#include <asm/uaccess.h>

A
Al Viro 已提交
47 48
#include "internal.h"

49 50 51 52 53 54
#define AIO_RING_MAGIC			0xa10a10a1
#define AIO_RING_COMPAT_FEATURES	1
#define AIO_RING_INCOMPAT_FEATURES	0
struct aio_ring {
	unsigned	id;	/* kernel internal index number */
	unsigned	nr;	/* number of io_events */
55 56
	unsigned	head;	/* Written to by userland or under ring_lock
				 * mutex by aio_read_events_ring(). */
57 58 59 60 61 62 63 64 65 66 67 68 69
	unsigned	tail;

	unsigned	magic;
	unsigned	compat_features;
	unsigned	incompat_features;
	unsigned	header_length;	/* size of aio_ring */


	struct io_event		io_events[0];
}; /* 128 bytes + ring size */

#define AIO_RING_PAGES	8

70 71 72 73 74 75
struct kioctx_table {
	struct rcu_head	rcu;
	unsigned	nr;
	struct kioctx	*table[];
};

K
Kent Overstreet 已提交
76 77 78 79
struct kioctx_cpu {
	unsigned		reqs_available;
};

80
struct kioctx {
K
Kent Overstreet 已提交
81
	struct percpu_ref	users;
K
Kent Overstreet 已提交
82
	atomic_t		dead;
83

K
Kent Overstreet 已提交
84 85
	struct percpu_ref	reqs;

86 87
	unsigned long		user_id;

K
Kent Overstreet 已提交
88 89 90 91 92 93 94
	struct __percpu kioctx_cpu *cpu;

	/*
	 * For percpu reqs_available, number of slots we move to/from global
	 * counter at a time:
	 */
	unsigned		req_batch;
95 96 97 98
	/*
	 * This is what userspace passed to io_setup(), it's not used for
	 * anything but counting against the global max_reqs quota.
	 *
K
Kent Overstreet 已提交
99
	 * The real limit is nr_events - 1, which will be larger (see
100 101
	 * aio_setup_ring())
	 */
102 103
	unsigned		max_reqs;

K
Kent Overstreet 已提交
104 105
	/* Size of ringbuffer, in units of struct io_event */
	unsigned		nr_events;
106

K
Kent Overstreet 已提交
107 108 109 110 111 112
	unsigned long		mmap_base;
	unsigned long		mmap_size;

	struct page		**ring_pages;
	long			nr_pages;

K
Kent Overstreet 已提交
113
	struct work_struct	free_work;
114

115 116 117 118 119
	/*
	 * signals when all in-flight requests are done
	 */
	struct completion *requests_done;

120
	struct {
121 122 123 124 125
		/*
		 * This counts the number of available slots in the ringbuffer,
		 * so we avoid overflowing it: it's decremented (if positive)
		 * when allocating a kiocb and incremented when the resulting
		 * io_event is pulled off the ringbuffer.
K
Kent Overstreet 已提交
126 127
		 *
		 * We batch accesses to it with a percpu version.
128 129
		 */
		atomic_t	reqs_available;
130 131 132 133 134 135 136
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t	ctx_lock;
		struct list_head active_reqs;	/* used for cancellation */
	} ____cacheline_aligned_in_smp;

K
Kent Overstreet 已提交
137 138
	struct {
		struct mutex	ring_lock;
139 140
		wait_queue_head_t wait;
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
141 142 143

	struct {
		unsigned	tail;
144
		unsigned	completed_events;
K
Kent Overstreet 已提交
145
		spinlock_t	completion_lock;
146
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
147 148

	struct page		*internal_pages[AIO_RING_PAGES];
149
	struct file		*aio_ring_file;
150 151

	unsigned		id;
152 153
};

L
Linus Torvalds 已提交
154
/*------ sysctl variables----*/
155 156 157
static DEFINE_SPINLOCK(aio_nr_lock);
unsigned long aio_nr;		/* current system wide number of aio requests */
unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
L
Linus Torvalds 已提交
158 159
/*----end sysctl variables---*/

160 161
static struct kmem_cache	*kiocb_cachep;
static struct kmem_cache	*kioctx_cachep;
L
Linus Torvalds 已提交
162

163 164 165 166 167 168 169 170 171 172 173
static struct vfsmount *aio_mnt;

static const struct file_operations aio_ring_fops;
static const struct address_space_operations aio_ctx_aops;

static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
{
	struct qstr this = QSTR_INIT("[aio]", 5);
	struct file *file;
	struct path path;
	struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
174 175
	if (IS_ERR(inode))
		return ERR_CAST(inode);
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204

	inode->i_mapping->a_ops = &aio_ctx_aops;
	inode->i_mapping->private_data = ctx;
	inode->i_size = PAGE_SIZE * nr_pages;

	path.dentry = d_alloc_pseudo(aio_mnt->mnt_sb, &this);
	if (!path.dentry) {
		iput(inode);
		return ERR_PTR(-ENOMEM);
	}
	path.mnt = mntget(aio_mnt);

	d_instantiate(path.dentry, inode);
	file = alloc_file(&path, FMODE_READ | FMODE_WRITE, &aio_ring_fops);
	if (IS_ERR(file)) {
		path_put(&path);
		return file;
	}

	file->f_flags = O_RDWR;
	return file;
}

static struct dentry *aio_mount(struct file_system_type *fs_type,
				int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname	= simple_dname,
	};
205
	return mount_pseudo(fs_type, "aio:", NULL, &ops, AIO_RING_MAGIC);
206 207
}

L
Linus Torvalds 已提交
208 209 210 211 212 213
/* aio_setup
 *	Creates the slab caches used by the aio routines, panic on
 *	failure as this is done early during the boot sequence.
 */
static int __init aio_setup(void)
{
214 215 216 217 218 219 220 221 222
	static struct file_system_type aio_fs = {
		.name		= "aio",
		.mount		= aio_mount,
		.kill_sb	= kill_anon_super,
	};
	aio_mnt = kern_mount(&aio_fs);
	if (IS_ERR(aio_mnt))
		panic("Failed to create aio fs mount.");

223 224
	kiocb_cachep = KMEM_CACHE(kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
L
Linus Torvalds 已提交
225

K
Kent Overstreet 已提交
226
	pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
L
Linus Torvalds 已提交
227 228 229

	return 0;
}
230
__initcall(aio_setup);
L
Linus Torvalds 已提交
231

232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
static void put_aio_ring_file(struct kioctx *ctx)
{
	struct file *aio_ring_file = ctx->aio_ring_file;
	if (aio_ring_file) {
		truncate_setsize(aio_ring_file->f_inode, 0);

		/* Prevent further access to the kioctx from migratepages */
		spin_lock(&aio_ring_file->f_inode->i_mapping->private_lock);
		aio_ring_file->f_inode->i_mapping->private_data = NULL;
		ctx->aio_ring_file = NULL;
		spin_unlock(&aio_ring_file->f_inode->i_mapping->private_lock);

		fput(aio_ring_file);
	}
}

L
Linus Torvalds 已提交
248 249
static void aio_free_ring(struct kioctx *ctx)
{
250
	int i;
L
Linus Torvalds 已提交
251

252 253 254 255 256
	/* Disconnect the kiotx from the ring file.  This prevents future
	 * accesses to the kioctx from page migration.
	 */
	put_aio_ring_file(ctx);

257
	for (i = 0; i < ctx->nr_pages; i++) {
258
		struct page *page;
259 260
		pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
				page_count(ctx->ring_pages[i]));
261 262 263 264 265
		page = ctx->ring_pages[i];
		if (!page)
			continue;
		ctx->ring_pages[i] = NULL;
		put_page(page);
266
	}
L
Linus Torvalds 已提交
267

268
	if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
K
Kent Overstreet 已提交
269
		kfree(ctx->ring_pages);
270 271
		ctx->ring_pages = NULL;
	}
272 273 274 275
}

static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
{
276
	vma->vm_flags |= VM_DONTEXPAND;
277 278 279 280
	vma->vm_ops = &generic_file_vm_ops;
	return 0;
}

281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
static void aio_ring_remap(struct file *file, struct vm_area_struct *vma)
{
	struct mm_struct *mm = vma->vm_mm;
	struct kioctx_table *table;
	int i;

	spin_lock(&mm->ioctx_lock);
	rcu_read_lock();
	table = rcu_dereference(mm->ioctx_table);
	for (i = 0; i < table->nr; i++) {
		struct kioctx *ctx;

		ctx = table->table[i];
		if (ctx && ctx->aio_ring_file == file) {
			ctx->user_id = ctx->mmap_base = vma->vm_start;
			break;
		}
	}

	rcu_read_unlock();
	spin_unlock(&mm->ioctx_lock);
}

304 305
static const struct file_operations aio_ring_fops = {
	.mmap = aio_ring_mmap,
306
	.mremap = aio_ring_remap,
307 308
};

309
#if IS_ENABLED(CONFIG_MIGRATION)
310 311 312
static int aio_migratepage(struct address_space *mapping, struct page *new,
			struct page *old, enum migrate_mode mode)
{
313
	struct kioctx *ctx;
314
	unsigned long flags;
315
	pgoff_t idx;
316 317
	int rc;

318 319
	rc = 0;

320
	/* mapping->private_lock here protects against the kioctx teardown.  */
321 322
	spin_lock(&mapping->private_lock);
	ctx = mapping->private_data;
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	if (!ctx) {
		rc = -EINVAL;
		goto out;
	}

	/* The ring_lock mutex.  The prevents aio_read_events() from writing
	 * to the ring's head, and prevents page migration from mucking in
	 * a partially initialized kiotx.
	 */
	if (!mutex_trylock(&ctx->ring_lock)) {
		rc = -EAGAIN;
		goto out;
	}

	idx = old->index;
	if (idx < (pgoff_t)ctx->nr_pages) {
		/* Make sure the old page hasn't already been changed */
		if (ctx->ring_pages[idx] != old)
			rc = -EAGAIN;
342 343 344 345
	} else
		rc = -EINVAL;

	if (rc != 0)
346
		goto out_unlock;
347

348 349
	/* Writeback must be complete */
	BUG_ON(PageWriteback(old));
350
	get_page(new);
351

352
	rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1);
353
	if (rc != MIGRATEPAGE_SUCCESS) {
354
		put_page(new);
355
		goto out_unlock;
356 357
	}

358 359 360
	/* Take completion_lock to prevent other writes to the ring buffer
	 * while the old page is copied to the new.  This prevents new
	 * events from being lost.
361
	 */
362 363 364 365 366
	spin_lock_irqsave(&ctx->completion_lock, flags);
	migrate_page_copy(new, old);
	BUG_ON(ctx->ring_pages[idx] != old);
	ctx->ring_pages[idx] = new;
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
367

368 369
	/* The old page is no longer accessible. */
	put_page(old);
370

371 372 373 374
out_unlock:
	mutex_unlock(&ctx->ring_lock);
out:
	spin_unlock(&mapping->private_lock);
375
	return rc;
L
Linus Torvalds 已提交
376
}
377
#endif
L
Linus Torvalds 已提交
378

379
static const struct address_space_operations aio_ctx_aops = {
380
	.set_page_dirty = __set_page_dirty_no_writeback,
381
#if IS_ENABLED(CONFIG_MIGRATION)
382
	.migratepage	= aio_migratepage,
383
#endif
384 385
};

L
Linus Torvalds 已提交
386 387 388 389
static int aio_setup_ring(struct kioctx *ctx)
{
	struct aio_ring *ring;
	unsigned nr_events = ctx->max_reqs;
Z
Zach Brown 已提交
390
	struct mm_struct *mm = current->mm;
391
	unsigned long size, unused;
L
Linus Torvalds 已提交
392
	int nr_pages;
393 394
	int i;
	struct file *file;
L
Linus Torvalds 已提交
395 396 397 398 399 400 401

	/* Compensate for the ring buffer's head/tail overlap entry */
	nr_events += 2;	/* 1 is required, 2 for good luck */

	size = sizeof(struct aio_ring);
	size += sizeof(struct io_event) * nr_events;

402
	nr_pages = PFN_UP(size);
L
Linus Torvalds 已提交
403 404 405
	if (nr_pages < 0)
		return -EINVAL;

406
	file = aio_private_file(ctx, nr_pages);
407 408
	if (IS_ERR(file)) {
		ctx->aio_ring_file = NULL;
409
		return -ENOMEM;
410 411
	}

412 413 414 415 416 417 418 419 420 421 422 423 424 425
	ctx->aio_ring_file = file;
	nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
			/ sizeof(struct io_event);

	ctx->ring_pages = ctx->internal_pages;
	if (nr_pages > AIO_RING_PAGES) {
		ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
					  GFP_KERNEL);
		if (!ctx->ring_pages) {
			put_aio_ring_file(ctx);
			return -ENOMEM;
		}
	}

426 427 428 429 430 431 432 433 434 435
	for (i = 0; i < nr_pages; i++) {
		struct page *page;
		page = find_or_create_page(file->f_inode->i_mapping,
					   i, GFP_HIGHUSER | __GFP_ZERO);
		if (!page)
			break;
		pr_debug("pid(%d) page[%d]->count=%d\n",
			 current->pid, i, page_count(page));
		SetPageUptodate(page);
		unlock_page(page);
436 437

		ctx->ring_pages[i] = page;
438
	}
439
	ctx->nr_pages = i;
L
Linus Torvalds 已提交
440

441 442
	if (unlikely(i != nr_pages)) {
		aio_free_ring(ctx);
443
		return -ENOMEM;
L
Linus Torvalds 已提交
444 445
	}

K
Kent Overstreet 已提交
446 447
	ctx->mmap_size = nr_pages * PAGE_SIZE;
	pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
448

Z
Zach Brown 已提交
449
	down_write(&mm->mmap_sem);
450 451
	ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
				       PROT_READ | PROT_WRITE,
452 453
				       MAP_SHARED, 0, &unused);
	up_write(&mm->mmap_sem);
K
Kent Overstreet 已提交
454 455
	if (IS_ERR((void *)ctx->mmap_base)) {
		ctx->mmap_size = 0;
L
Linus Torvalds 已提交
456
		aio_free_ring(ctx);
457
		return -ENOMEM;
L
Linus Torvalds 已提交
458 459
	}

K
Kent Overstreet 已提交
460
	pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
461

K
Kent Overstreet 已提交
462 463
	ctx->user_id = ctx->mmap_base;
	ctx->nr_events = nr_events; /* trusted copy */
L
Linus Torvalds 已提交
464

K
Kent Overstreet 已提交
465
	ring = kmap_atomic(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
466
	ring->nr = nr_events;	/* user copy */
467
	ring->id = ~0U;
L
Linus Torvalds 已提交
468 469 470 471 472
	ring->head = ring->tail = 0;
	ring->magic = AIO_RING_MAGIC;
	ring->compat_features = AIO_RING_COMPAT_FEATURES;
	ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
	ring->header_length = sizeof(struct aio_ring);
473
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
474
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
475 476 477 478 479 480 481 482

	return 0;
}

#define AIO_EVENTS_PER_PAGE	(PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE	((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET	(AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)

483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel)
{
	struct kioctx *ctx = req->ki_ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->ctx_lock, flags);

	if (!req->ki_list.next)
		list_add(&req->ki_list, &ctx->active_reqs);

	req->ki_cancel = cancel;

	spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
EXPORT_SYMBOL(kiocb_set_cancel_fn);

499
static int kiocb_cancel(struct kiocb *kiocb)
K
Kent Overstreet 已提交
500
{
501
	kiocb_cancel_fn *old, *cancel;
K
Kent Overstreet 已提交
502

503 504 505 506 507 508 509 510
	/*
	 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
	 * actually has a cancel function, hence the cmpxchg()
	 */

	cancel = ACCESS_ONCE(kiocb->ki_cancel);
	do {
		if (!cancel || cancel == KIOCB_CANCELLED)
K
Kent Overstreet 已提交
511
			return -EINVAL;
K
Kent Overstreet 已提交
512

513 514 515
		old = cancel;
		cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
	} while (cancel != old);
K
Kent Overstreet 已提交
516

K
Kent Overstreet 已提交
517
	return cancel(kiocb);
K
Kent Overstreet 已提交
518 519
}

K
Kent Overstreet 已提交
520
static void free_ioctx(struct work_struct *work)
K
Kent Overstreet 已提交
521
{
K
Kent Overstreet 已提交
522
	struct kioctx *ctx = container_of(work, struct kioctx, free_work);
K
Kent Overstreet 已提交
523

K
Kent Overstreet 已提交
524
	pr_debug("freeing %p\n", ctx);
K
Kent Overstreet 已提交
525

K
Kent Overstreet 已提交
526
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
527
	free_percpu(ctx->cpu);
528 529
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
K
Kent Overstreet 已提交
530 531 532
	kmem_cache_free(kioctx_cachep, ctx);
}

K
Kent Overstreet 已提交
533 534 535 536
static void free_ioctx_reqs(struct percpu_ref *ref)
{
	struct kioctx *ctx = container_of(ref, struct kioctx, reqs);

537 538 539 540
	/* At this point we know that there are no any in-flight requests */
	if (ctx->requests_done)
		complete(ctx->requests_done);

K
Kent Overstreet 已提交
541 542 543 544
	INIT_WORK(&ctx->free_work, free_ioctx);
	schedule_work(&ctx->free_work);
}

K
Kent Overstreet 已提交
545 546 547 548 549
/*
 * When this function runs, the kioctx has been removed from the "hash table"
 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
 * now it's safe to cancel any that need to be.
 */
K
Kent Overstreet 已提交
550
static void free_ioctx_users(struct percpu_ref *ref)
K
Kent Overstreet 已提交
551
{
K
Kent Overstreet 已提交
552
	struct kioctx *ctx = container_of(ref, struct kioctx, users);
K
Kent Overstreet 已提交
553 554 555 556 557 558 559 560 561
	struct kiocb *req;

	spin_lock_irq(&ctx->ctx_lock);

	while (!list_empty(&ctx->active_reqs)) {
		req = list_first_entry(&ctx->active_reqs,
				       struct kiocb, ki_list);

		list_del_init(&req->ki_list);
562
		kiocb_cancel(req);
K
Kent Overstreet 已提交
563 564 565 566
	}

	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
567 568
	percpu_ref_kill(&ctx->reqs);
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
569 570
}

571 572 573 574 575 576 577
static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
{
	unsigned i, new_nr;
	struct kioctx_table *table, *old;
	struct aio_ring *ring;

	spin_lock(&mm->ioctx_lock);
578
	table = rcu_dereference_raw(mm->ioctx_table);
579 580 581 582 583 584 585 586 587

	while (1) {
		if (table)
			for (i = 0; i < table->nr; i++)
				if (!table->table[i]) {
					ctx->id = i;
					table->table[i] = ctx;
					spin_unlock(&mm->ioctx_lock);

588 589 590 591
					/* While kioctx setup is in progress,
					 * we are protected from page migration
					 * changes ring_pages by ->ring_lock.
					 */
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
					ring = kmap_atomic(ctx->ring_pages[0]);
					ring->id = ctx->id;
					kunmap_atomic(ring);
					return 0;
				}

		new_nr = (table ? table->nr : 1) * 4;
		spin_unlock(&mm->ioctx_lock);

		table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
				new_nr, GFP_KERNEL);
		if (!table)
			return -ENOMEM;

		table->nr = new_nr;

		spin_lock(&mm->ioctx_lock);
609
		old = rcu_dereference_raw(mm->ioctx_table);
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625

		if (!old) {
			rcu_assign_pointer(mm->ioctx_table, table);
		} else if (table->nr > old->nr) {
			memcpy(table->table, old->table,
			       old->nr * sizeof(struct kioctx *));

			rcu_assign_pointer(mm->ioctx_table, table);
			kfree_rcu(old, rcu);
		} else {
			kfree(table);
			table = old;
		}
	}
}

K
Kent Overstreet 已提交
626 627 628 629 630 631 632 633 634 635
static void aio_nr_sub(unsigned nr)
{
	spin_lock(&aio_nr_lock);
	if (WARN_ON(aio_nr - nr > aio_nr))
		aio_nr = 0;
	else
		aio_nr -= nr;
	spin_unlock(&aio_nr_lock);
}

L
Linus Torvalds 已提交
636 637 638 639 640
/* ioctx_alloc
 *	Allocates and initializes an ioctx.  Returns an ERR_PTR if it failed.
 */
static struct kioctx *ioctx_alloc(unsigned nr_events)
{
Z
Zach Brown 已提交
641
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
642
	struct kioctx *ctx;
643
	int err = -ENOMEM;
L
Linus Torvalds 已提交
644

K
Kent Overstreet 已提交
645 646 647 648 649 650 651 652 653 654 655 656
	/*
	 * We keep track of the number of available ringbuffer slots, to prevent
	 * overflow (reqs_available), and we also use percpu counters for this.
	 *
	 * So since up to half the slots might be on other cpu's percpu counters
	 * and unavailable, double nr_events so userspace sees what they
	 * expected: additionally, we move req_batch slots to/from percpu
	 * counters at a time, so make sure that isn't 0:
	 */
	nr_events = max(nr_events, num_possible_cpus() * 4);
	nr_events *= 2;

L
Linus Torvalds 已提交
657 658 659 660 661 662 663
	/* Prevent overflows */
	if ((nr_events > (0x10000000U / sizeof(struct io_event))) ||
	    (nr_events > (0x10000000U / sizeof(struct kiocb)))) {
		pr_debug("ENOMEM: nr_events too high\n");
		return ERR_PTR(-EINVAL);
	}

664
	if (!nr_events || (unsigned long)nr_events > (aio_max_nr * 2UL))
L
Linus Torvalds 已提交
665 666
		return ERR_PTR(-EAGAIN);

667
	ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
668 669 670 671 672 673
	if (!ctx)
		return ERR_PTR(-ENOMEM);

	ctx->max_reqs = nr_events;

	spin_lock_init(&ctx->ctx_lock);
674
	spin_lock_init(&ctx->completion_lock);
K
Kent Overstreet 已提交
675
	mutex_init(&ctx->ring_lock);
676 677 678
	/* Protect against page migration throughout kiotx setup by keeping
	 * the ring_lock mutex held until setup is complete. */
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
679 680 681 682
	init_waitqueue_head(&ctx->wait);

	INIT_LIST_HEAD(&ctx->active_reqs);

683
	if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
684 685
		goto err;

686
	if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
687 688
		goto err;

K
Kent Overstreet 已提交
689 690
	ctx->cpu = alloc_percpu(struct kioctx_cpu);
	if (!ctx->cpu)
K
Kent Overstreet 已提交
691
		goto err;
L
Linus Torvalds 已提交
692

693 694
	err = aio_setup_ring(ctx);
	if (err < 0)
K
Kent Overstreet 已提交
695
		goto err;
K
Kent Overstreet 已提交
696

697
	atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
K
Kent Overstreet 已提交
698
	ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
699 700
	if (ctx->req_batch < 1)
		ctx->req_batch = 1;
701

L
Linus Torvalds 已提交
702
	/* limit the number of system wide aios */
703
	spin_lock(&aio_nr_lock);
704
	if (aio_nr + nr_events > (aio_max_nr * 2UL) ||
705
	    aio_nr + nr_events < aio_nr) {
706
		spin_unlock(&aio_nr_lock);
K
Kent Overstreet 已提交
707
		err = -EAGAIN;
G
Gu Zheng 已提交
708
		goto err_ctx;
709 710
	}
	aio_nr += ctx->max_reqs;
711
	spin_unlock(&aio_nr_lock);
L
Linus Torvalds 已提交
712

713 714
	percpu_ref_get(&ctx->users);	/* io_setup() will drop this ref */
	percpu_ref_get(&ctx->reqs);	/* free_ioctx_users() will drop this */
K
Kent Overstreet 已提交
715

716 717
	err = ioctx_add_table(ctx, mm);
	if (err)
K
Kent Overstreet 已提交
718
		goto err_cleanup;
719

720 721 722
	/* Release the ring_lock mutex now that all setup is complete. */
	mutex_unlock(&ctx->ring_lock);

K
Kent Overstreet 已提交
723
	pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
K
Kent Overstreet 已提交
724
		 ctx, ctx->user_id, mm, ctx->nr_events);
L
Linus Torvalds 已提交
725 726
	return ctx;

K
Kent Overstreet 已提交
727 728
err_cleanup:
	aio_nr_sub(ctx->max_reqs);
G
Gu Zheng 已提交
729 730
err_ctx:
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
731
err:
732
	mutex_unlock(&ctx->ring_lock);
K
Kent Overstreet 已提交
733
	free_percpu(ctx->cpu);
734 735
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
L
Linus Torvalds 已提交
736
	kmem_cache_free(kioctx_cachep, ctx);
K
Kent Overstreet 已提交
737
	pr_debug("error allocating ioctx %d\n", err);
738
	return ERR_PTR(err);
L
Linus Torvalds 已提交
739 740
}

K
Kent Overstreet 已提交
741 742 743 744 745
/* kill_ioctx
 *	Cancels all outstanding aio requests on an aio context.  Used
 *	when the processes owning a context have all exited to encourage
 *	the rapid destruction of the kioctx.
 */
746
static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
747
		struct completion *requests_done)
K
Kent Overstreet 已提交
748
{
749
	struct kioctx_table *table;
750

751 752
	if (atomic_xchg(&ctx->dead, 1))
		return -EINVAL;
753 754


755
	spin_lock(&mm->ioctx_lock);
756
	table = rcu_dereference_raw(mm->ioctx_table);
757 758 759
	WARN_ON(ctx != table->table[ctx->id]);
	table->table[ctx->id] = NULL;
	spin_unlock(&mm->ioctx_lock);
760

761 762
	/* percpu_ref_kill() will do the necessary call_rcu() */
	wake_up_all(&ctx->wait);
763

764 765 766 767 768 769 770 771
	/*
	 * It'd be more correct to do this in free_ioctx(), after all
	 * the outstanding kiocbs have finished - but by then io_destroy
	 * has already returned, so io_setup() could potentially return
	 * -EAGAIN with no ioctxs actually in use (as far as userspace
	 *  could tell).
	 */
	aio_nr_sub(ctx->max_reqs);
772

773 774
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
775

776 777 778
	ctx->requests_done = requests_done;
	percpu_ref_kill(&ctx->users);
	return 0;
L
Linus Torvalds 已提交
779 780
}

K
Kent Overstreet 已提交
781 782 783 784 785 786 787
/*
 * exit_aio: called when the last user of mm goes away.  At this point, there is
 * no way for any new requests to be submited or any of the io_* syscalls to be
 * called on the context.
 *
 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
 * them.
L
Linus Torvalds 已提交
788
 */
789
void exit_aio(struct mm_struct *mm)
L
Linus Torvalds 已提交
790
{
791 792
	struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
	int i;
793

794 795
	if (!table)
		return;
796

797 798
	for (i = 0; i < table->nr; ++i) {
		struct kioctx *ctx = table->table[i];
799 800
		struct completion requests_done =
			COMPLETION_INITIALIZER_ONSTACK(requests_done);
J
Jens Axboe 已提交
801

802 803
		if (!ctx)
			continue;
804
		/*
805 806 807 808 809
		 * We don't need to bother with munmap() here - exit_mmap(mm)
		 * is coming and it'll unmap everything. And we simply can't,
		 * this is not necessarily our ->mm.
		 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
		 * that it needs to unmap the area, just set it to 0.
810
		 */
K
Kent Overstreet 已提交
811
		ctx->mmap_size = 0;
812
		kill_ioctx(mm, ctx, &requests_done);
K
Kent Overstreet 已提交
813

814 815
		/* Wait until all IO for the context are done. */
		wait_for_completion(&requests_done);
L
Linus Torvalds 已提交
816
	}
817 818 819

	RCU_INIT_POINTER(mm->ioctx_table, NULL);
	kfree(table);
L
Linus Torvalds 已提交
820 821
}

K
Kent Overstreet 已提交
822 823 824
static void put_reqs_available(struct kioctx *ctx, unsigned nr)
{
	struct kioctx_cpu *kcpu;
825
	unsigned long flags;
K
Kent Overstreet 已提交
826

827
	local_irq_save(flags);
828
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
829
	kcpu->reqs_available += nr;
830

K
Kent Overstreet 已提交
831 832 833 834 835
	while (kcpu->reqs_available >= ctx->req_batch * 2) {
		kcpu->reqs_available -= ctx->req_batch;
		atomic_add(ctx->req_batch, &ctx->reqs_available);
	}

836
	local_irq_restore(flags);
K
Kent Overstreet 已提交
837 838 839 840 841 842
}

static bool get_reqs_available(struct kioctx *ctx)
{
	struct kioctx_cpu *kcpu;
	bool ret = false;
843
	unsigned long flags;
K
Kent Overstreet 已提交
844

845
	local_irq_save(flags);
846
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864
	if (!kcpu->reqs_available) {
		int old, avail = atomic_read(&ctx->reqs_available);

		do {
			if (avail < ctx->req_batch)
				goto out;

			old = avail;
			avail = atomic_cmpxchg(&ctx->reqs_available,
					       avail, avail - ctx->req_batch);
		} while (avail != old);

		kcpu->reqs_available += ctx->req_batch;
	}

	ret = true;
	kcpu->reqs_available--;
out:
865
	local_irq_restore(flags);
K
Kent Overstreet 已提交
866 867 868
	return ret;
}

869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
/* refill_reqs_available
 *	Updates the reqs_available reference counts used for tracking the
 *	number of free slots in the completion ring.  This can be called
 *	from aio_complete() (to optimistically update reqs_available) or
 *	from aio_get_req() (the we're out of events case).  It must be
 *	called holding ctx->completion_lock.
 */
static void refill_reqs_available(struct kioctx *ctx, unsigned head,
                                  unsigned tail)
{
	unsigned events_in_ring, completed;

	/* Clamp head since userland can write to it. */
	head %= ctx->nr_events;
	if (head <= tail)
		events_in_ring = tail - head;
	else
		events_in_ring = ctx->nr_events - (head - tail);

	completed = ctx->completed_events;
	if (events_in_ring < completed)
		completed -= events_in_ring;
	else
		completed = 0;

	if (!completed)
		return;

	ctx->completed_events -= completed;
	put_reqs_available(ctx, completed);
}

/* user_refill_reqs_available
 *	Called to refill reqs_available when aio_get_req() encounters an
 *	out of space in the completion ring.
 */
static void user_refill_reqs_available(struct kioctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	if (ctx->completed_events) {
		struct aio_ring *ring;
		unsigned head;

		/* Access of ring->head may race with aio_read_events_ring()
		 * here, but that's okay since whether we read the old version
		 * or the new version, and either will be valid.  The important
		 * part is that head cannot pass tail since we prevent
		 * aio_complete() from updating tail by holding
		 * ctx->completion_lock.  Even if head is invalid, the check
		 * against ctx->completed_events below will make sure we do the
		 * safe/right thing.
		 */
		ring = kmap_atomic(ctx->ring_pages[0]);
		head = ring->head;
		kunmap_atomic(ring);

		refill_reqs_available(ctx, head, ctx->tail);
	}

	spin_unlock_irq(&ctx->completion_lock);
}

L
Linus Torvalds 已提交
931
/* aio_get_req
K
Kent Overstreet 已提交
932 933
 *	Allocate a slot for an aio request.
 * Returns NULL if no requests are free.
L
Linus Torvalds 已提交
934
 */
K
Kent Overstreet 已提交
935
static inline struct kiocb *aio_get_req(struct kioctx *ctx)
L
Linus Torvalds 已提交
936
{
K
Kent Overstreet 已提交
937 938
	struct kiocb *req;

939 940 941 942 943
	if (!get_reqs_available(ctx)) {
		user_refill_reqs_available(ctx);
		if (!get_reqs_available(ctx))
			return NULL;
	}
K
Kent Overstreet 已提交
944

945
	req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
L
Linus Torvalds 已提交
946
	if (unlikely(!req))
K
Kent Overstreet 已提交
947
		goto out_put;
L
Linus Torvalds 已提交
948

K
Kent Overstreet 已提交
949 950
	percpu_ref_get(&ctx->reqs);

L
Linus Torvalds 已提交
951
	req->ki_ctx = ctx;
J
Jeff Moyer 已提交
952
	return req;
K
Kent Overstreet 已提交
953
out_put:
K
Kent Overstreet 已提交
954
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
955
	return NULL;
L
Linus Torvalds 已提交
956 957
}

958
static void kiocb_free(struct kiocb *req)
L
Linus Torvalds 已提交
959
{
960 961
	if (req->ki_filp)
		fput(req->ki_filp);
962 963
	if (req->ki_eventfd != NULL)
		eventfd_ctx_put(req->ki_eventfd);
L
Linus Torvalds 已提交
964 965 966
	kmem_cache_free(kiocb_cachep, req);
}

A
Adrian Bunk 已提交
967
static struct kioctx *lookup_ioctx(unsigned long ctx_id)
L
Linus Torvalds 已提交
968
{
969
	struct aio_ring __user *ring  = (void __user *)ctx_id;
J
Jens Axboe 已提交
970
	struct mm_struct *mm = current->mm;
971
	struct kioctx *ctx, *ret = NULL;
972 973 974 975 976
	struct kioctx_table *table;
	unsigned id;

	if (get_user(id, &ring->id))
		return NULL;
L
Linus Torvalds 已提交
977

J
Jens Axboe 已提交
978
	rcu_read_lock();
979
	table = rcu_dereference(mm->ioctx_table);
J
Jens Axboe 已提交
980

981 982
	if (!table || id >= table->nr)
		goto out;
L
Linus Torvalds 已提交
983

984
	ctx = table->table[id];
985
	if (ctx && ctx->user_id == ctx_id) {
986 987 988 989
		percpu_ref_get(&ctx->users);
		ret = ctx;
	}
out:
J
Jens Axboe 已提交
990
	rcu_read_unlock();
991
	return ret;
L
Linus Torvalds 已提交
992 993 994 995 996
}

/* aio_complete
 *	Called when the io request on the given iocb is complete.
 */
997
void aio_complete(struct kiocb *iocb, long res, long res2)
L
Linus Torvalds 已提交
998 999 1000
{
	struct kioctx	*ctx = iocb->ki_ctx;
	struct aio_ring	*ring;
K
Kent Overstreet 已提交
1001
	struct io_event	*ev_page, *event;
1002
	unsigned tail, pos, head;
L
Linus Torvalds 已提交
1003 1004
	unsigned long	flags;

1005 1006 1007 1008 1009 1010
	/*
	 * Special case handling for sync iocbs:
	 *  - events go directly into the iocb for fast handling
	 *  - the sync task with the iocb in its stack holds the single iocb
	 *    ref, no other paths have a way to get another ref
	 *  - the sync task helpfully left a reference to itself in the iocb
L
Linus Torvalds 已提交
1011
	 */
1012
	BUG_ON(is_sync_kiocb(iocb));
L
Linus Torvalds 已提交
1013

1014 1015 1016 1017 1018 1019 1020
	if (iocb->ki_list.next) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->ctx_lock, flags);
		list_del(&iocb->ki_list);
		spin_unlock_irqrestore(&ctx->ctx_lock, flags);
	}
1021

1022 1023
	/*
	 * Add a completion event to the ring buffer. Must be done holding
1024
	 * ctx->completion_lock to prevent other code from messing with the tail
1025 1026 1027 1028
	 * pointer since we might be called from irq context.
	 */
	spin_lock_irqsave(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1029
	tail = ctx->tail;
K
Kent Overstreet 已提交
1030 1031
	pos = tail + AIO_EVENTS_OFFSET;

K
Kent Overstreet 已提交
1032
	if (++tail >= ctx->nr_events)
1033
		tail = 0;
L
Linus Torvalds 已提交
1034

K
Kent Overstreet 已提交
1035
	ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1036 1037
	event = ev_page + pos % AIO_EVENTS_PER_PAGE;

L
Linus Torvalds 已提交
1038 1039 1040 1041 1042
	event->obj = (u64)(unsigned long)iocb->ki_obj.user;
	event->data = iocb->ki_user_data;
	event->res = res;
	event->res2 = res2;

K
Kent Overstreet 已提交
1043
	kunmap_atomic(ev_page);
K
Kent Overstreet 已提交
1044
	flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1045 1046

	pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
K
Kent Overstreet 已提交
1047 1048
		 ctx, tail, iocb, iocb->ki_obj.user, iocb->ki_user_data,
		 res, res2);
L
Linus Torvalds 已提交
1049 1050 1051 1052 1053 1054

	/* after flagging the request as done, we
	 * must never even look at it again
	 */
	smp_wmb();	/* make event visible before updating tail */

K
Kent Overstreet 已提交
1055
	ctx->tail = tail;
L
Linus Torvalds 已提交
1056

K
Kent Overstreet 已提交
1057
	ring = kmap_atomic(ctx->ring_pages[0]);
1058
	head = ring->head;
K
Kent Overstreet 已提交
1059
	ring->tail = tail;
1060
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1061
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
1062

1063 1064 1065
	ctx->completed_events++;
	if (ctx->completed_events > 1)
		refill_reqs_available(ctx, head, tail);
1066 1067
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1068
	pr_debug("added to ring %p at [%u]\n", iocb, tail);
D
Davide Libenzi 已提交
1069 1070 1071 1072 1073 1074

	/*
	 * Check if the user asked us to deliver the result through an
	 * eventfd. The eventfd_signal() function is safe to be called
	 * from IRQ context.
	 */
1075
	if (iocb->ki_eventfd != NULL)
D
Davide Libenzi 已提交
1076 1077
		eventfd_signal(iocb->ki_eventfd, 1);

L
Linus Torvalds 已提交
1078
	/* everything turned out well, dispose of the aiocb. */
K
Kent Overstreet 已提交
1079
	kiocb_free(iocb);
L
Linus Torvalds 已提交
1080

1081 1082 1083 1084 1085 1086 1087 1088
	/*
	 * We have to order our ring_info tail store above and test
	 * of the wait list below outside the wait lock.  This is
	 * like in wake_up_bit() where clearing a bit has to be
	 * ordered with the unlocked test.
	 */
	smp_mb();

L
Linus Torvalds 已提交
1089 1090 1091
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);

K
Kent Overstreet 已提交
1092
	percpu_ref_put(&ctx->reqs);
L
Linus Torvalds 已提交
1093
}
1094
EXPORT_SYMBOL(aio_complete);
L
Linus Torvalds 已提交
1095

G
Gu Zheng 已提交
1096
/* aio_read_events_ring
1097 1098
 *	Pull an event off of the ioctx's event ring.  Returns the number of
 *	events fetched
L
Linus Torvalds 已提交
1099
 */
1100 1101
static long aio_read_events_ring(struct kioctx *ctx,
				 struct io_event __user *event, long nr)
L
Linus Torvalds 已提交
1102 1103
{
	struct aio_ring *ring;
1104
	unsigned head, tail, pos;
1105 1106 1107
	long ret = 0;
	int copy_ret;

1108 1109 1110 1111 1112 1113 1114
	/*
	 * The mutex can block and wake us up and that will cause
	 * wait_event_interruptible_hrtimeout() to schedule without sleeping
	 * and repeat. This should be rare enough that it doesn't cause
	 * peformance issues. See the comment in read_events() for more detail.
	 */
	sched_annotate_sleep();
K
Kent Overstreet 已提交
1115
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
1116

1117
	/* Access to ->ring_pages here is protected by ctx->ring_lock. */
K
Kent Overstreet 已提交
1118
	ring = kmap_atomic(ctx->ring_pages[0]);
1119
	head = ring->head;
1120
	tail = ring->tail;
1121 1122
	kunmap_atomic(ring);

1123 1124 1125 1126 1127 1128
	/*
	 * Ensure that once we've read the current tail pointer, that
	 * we also see the events that were stored up to the tail.
	 */
	smp_rmb();

1129
	pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
L
Linus Torvalds 已提交
1130

1131
	if (head == tail)
L
Linus Torvalds 已提交
1132 1133
		goto out;

1134 1135 1136
	head %= ctx->nr_events;
	tail %= ctx->nr_events;

1137 1138 1139 1140 1141
	while (ret < nr) {
		long avail;
		struct io_event *ev;
		struct page *page;

1142 1143
		avail = (head <= tail ?  tail : ctx->nr_events) - head;
		if (head == tail)
1144 1145 1146 1147 1148 1149 1150
			break;

		avail = min(avail, nr - ret);
		avail = min_t(long, avail, AIO_EVENTS_PER_PAGE -
			    ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE));

		pos = head + AIO_EVENTS_OFFSET;
K
Kent Overstreet 已提交
1151
		page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
		pos %= AIO_EVENTS_PER_PAGE;

		ev = kmap(page);
		copy_ret = copy_to_user(event + ret, ev + pos,
					sizeof(*ev) * avail);
		kunmap(page);

		if (unlikely(copy_ret)) {
			ret = -EFAULT;
			goto out;
		}

		ret += avail;
		head += avail;
K
Kent Overstreet 已提交
1166
		head %= ctx->nr_events;
L
Linus Torvalds 已提交
1167 1168
	}

K
Kent Overstreet 已提交
1169
	ring = kmap_atomic(ctx->ring_pages[0]);
1170
	ring->head = head;
1171
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1172
	flush_dcache_page(ctx->ring_pages[0]);
1173

1174
	pr_debug("%li  h%u t%u\n", ret, head, tail);
1175
out:
K
Kent Overstreet 已提交
1176
	mutex_unlock(&ctx->ring_lock);
1177

L
Linus Torvalds 已提交
1178 1179 1180
	return ret;
}

1181 1182
static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
			    struct io_event __user *event, long *i)
L
Linus Torvalds 已提交
1183
{
1184
	long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
L
Linus Torvalds 已提交
1185

1186 1187
	if (ret > 0)
		*i += ret;
L
Linus Torvalds 已提交
1188

1189 1190
	if (unlikely(atomic_read(&ctx->dead)))
		ret = -EINVAL;
L
Linus Torvalds 已提交
1191

1192 1193
	if (!*i)
		*i = ret;
L
Linus Torvalds 已提交
1194

1195
	return ret < 0 || *i >= min_nr;
L
Linus Torvalds 已提交
1196 1197
}

1198
static long read_events(struct kioctx *ctx, long min_nr, long nr,
L
Linus Torvalds 已提交
1199 1200 1201
			struct io_event __user *event,
			struct timespec __user *timeout)
{
1202 1203
	ktime_t until = { .tv64 = KTIME_MAX };
	long ret = 0;
L
Linus Torvalds 已提交
1204 1205 1206

	if (timeout) {
		struct timespec	ts;
1207

L
Linus Torvalds 已提交
1208
		if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
1209
			return -EFAULT;
L
Linus Torvalds 已提交
1210

1211
		until = timespec_to_ktime(ts);
L
Linus Torvalds 已提交
1212 1213
	}

1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	/*
	 * Note that aio_read_events() is being called as the conditional - i.e.
	 * we're calling it after prepare_to_wait() has set task state to
	 * TASK_INTERRUPTIBLE.
	 *
	 * But aio_read_events() can block, and if it blocks it's going to flip
	 * the task state back to TASK_RUNNING.
	 *
	 * This should be ok, provided it doesn't flip the state back to
	 * TASK_RUNNING and return 0 too much - that causes us to spin. That
	 * will only happen if the mutex_lock() call blocks, and we then find
	 * the ringbuffer empty. So in practice we should be ok, but it's
	 * something to be aware of when touching this code.
	 */
1228 1229 1230 1231 1232 1233
	if (until.tv64 == 0)
		aio_read_events(ctx, min_nr, nr, event, &ret);
	else
		wait_event_interruptible_hrtimeout(ctx->wait,
				aio_read_events(ctx, min_nr, nr, event, &ret),
				until);
L
Linus Torvalds 已提交
1234

1235 1236
	if (!ret && signal_pending(current))
		ret = -EINTR;
L
Linus Torvalds 已提交
1237

1238
	return ret;
L
Linus Torvalds 已提交
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
}

/* sys_io_setup:
 *	Create an aio_context capable of receiving at least nr_events.
 *	ctxp must not point to an aio_context that already exists, and
 *	must be initialized to 0 prior to the call.  On successful
 *	creation of the aio_context, *ctxp is filled in with the resulting 
 *	handle.  May fail with -EINVAL if *ctxp is not initialized,
 *	if the specified nr_events exceeds internal limits.  May fail 
 *	with -EAGAIN if the specified nr_events exceeds the user's limit 
 *	of available events.  May fail with -ENOMEM if insufficient kernel
 *	resources are available.  May fail with -EFAULT if an invalid
 *	pointer is passed for ctxp.  Will fail with -ENOSYS if not
 *	implemented.
 */
1254
SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
L
Linus Torvalds 已提交
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
{
	struct kioctx *ioctx = NULL;
	unsigned long ctx;
	long ret;

	ret = get_user(ctx, ctxp);
	if (unlikely(ret))
		goto out;

	ret = -EINVAL;
1265 1266 1267
	if (unlikely(ctx || nr_events == 0)) {
		pr_debug("EINVAL: io_setup: ctx %lu nr_events %u\n",
		         ctx, nr_events);
L
Linus Torvalds 已提交
1268 1269 1270 1271 1272 1273 1274
		goto out;
	}

	ioctx = ioctx_alloc(nr_events);
	ret = PTR_ERR(ioctx);
	if (!IS_ERR(ioctx)) {
		ret = put_user(ioctx->user_id, ctxp);
1275
		if (ret)
1276
			kill_ioctx(current->mm, ioctx, NULL);
K
Kent Overstreet 已提交
1277
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1278 1279 1280 1281 1282 1283 1284 1285 1286
	}

out:
	return ret;
}

/* sys_io_destroy:
 *	Destroy the aio_context specified.  May cancel any outstanding 
 *	AIOs and block on completion.  Will fail with -ENOSYS if not
1287
 *	implemented.  May fail with -EINVAL if the context pointed to
L
Linus Torvalds 已提交
1288 1289
 *	is invalid.
 */
1290
SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
L
Linus Torvalds 已提交
1291 1292 1293
{
	struct kioctx *ioctx = lookup_ioctx(ctx);
	if (likely(NULL != ioctx)) {
1294 1295
		struct completion requests_done =
			COMPLETION_INITIALIZER_ONSTACK(requests_done);
1296
		int ret;
1297 1298 1299 1300 1301

		/* Pass requests_done to kill_ioctx() where it can be set
		 * in a thread-safe way. If we try to set it here then we have
		 * a race condition if two io_destroy() called simultaneously.
		 */
1302
		ret = kill_ioctx(current->mm, ioctx, &requests_done);
K
Kent Overstreet 已提交
1303
		percpu_ref_put(&ioctx->users);
1304 1305 1306 1307 1308

		/* Wait until all IO for the context are done. Otherwise kernel
		 * keep using user-space buffers even if user thinks the context
		 * is destroyed.
		 */
1309 1310
		if (!ret)
			wait_for_completion(&requests_done);
1311

1312
		return ret;
L
Linus Torvalds 已提交
1313 1314 1315 1316 1317
	}
	pr_debug("EINVAL: io_destroy: invalid context id\n");
	return -EINVAL;
}

K
Kent Overstreet 已提交
1318 1319
typedef ssize_t (aio_rw_op)(struct kiocb *, const struct iovec *,
			    unsigned long, loff_t);
1320
typedef ssize_t (rw_iter_op)(struct kiocb *, struct iov_iter *);
K
Kent Overstreet 已提交
1321

1322 1323 1324
static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
				     int rw, char __user *buf,
				     unsigned long *nr_segs,
C
Christoph Hellwig 已提交
1325
				     size_t *len,
1326 1327
				     struct iovec **iovec,
				     bool compat)
B
Badari Pulavarty 已提交
1328 1329 1330
{
	ssize_t ret;

C
Christoph Hellwig 已提交
1331
	*nr_segs = *len;
K
Kent Overstreet 已提交
1332

1333 1334
#ifdef CONFIG_COMPAT
	if (compat)
K
Kent Overstreet 已提交
1335
		ret = compat_rw_copy_check_uvector(rw,
1336
				(struct compat_iovec __user *)buf,
1337
				*nr_segs, UIO_FASTIOV, *iovec, iovec);
1338 1339
	else
#endif
K
Kent Overstreet 已提交
1340
		ret = rw_copy_check_uvector(rw,
1341
				(struct iovec __user *)buf,
1342
				*nr_segs, UIO_FASTIOV, *iovec, iovec);
B
Badari Pulavarty 已提交
1343
	if (ret < 0)
K
Kent Overstreet 已提交
1344
		return ret;
1345

C
Christoph Hellwig 已提交
1346 1347
	/* len now reflect bytes instead of segs */
	*len = ret;
K
Kent Overstreet 已提交
1348
	return 0;
B
Badari Pulavarty 已提交
1349 1350
}

1351 1352 1353
static ssize_t aio_setup_single_vector(struct kiocb *kiocb,
				       int rw, char __user *buf,
				       unsigned long *nr_segs,
C
Christoph Hellwig 已提交
1354
				       size_t len,
1355
				       struct iovec *iovec)
B
Badari Pulavarty 已提交
1356
{
C
Christoph Hellwig 已提交
1357
	if (unlikely(!access_ok(!rw, buf, len)))
K
Kent Overstreet 已提交
1358
		return -EFAULT;
1359

1360
	iovec->iov_base = buf;
C
Christoph Hellwig 已提交
1361
	iovec->iov_len = len;
1362
	*nr_segs = 1;
B
Badari Pulavarty 已提交
1363 1364 1365
	return 0;
}

L
Linus Torvalds 已提交
1366
/*
G
Gu Zheng 已提交
1367 1368
 * aio_run_iocb:
 *	Performs the initial checks and io submission.
L
Linus Torvalds 已提交
1369
 */
1370
static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
C
Christoph Hellwig 已提交
1371
			    char __user *buf, size_t len, bool compat)
L
Linus Torvalds 已提交
1372
{
K
Kent Overstreet 已提交
1373 1374
	struct file *file = req->ki_filp;
	ssize_t ret;
1375
	unsigned long nr_segs;
K
Kent Overstreet 已提交
1376 1377 1378
	int rw;
	fmode_t mode;
	aio_rw_op *rw_op;
1379
	rw_iter_op *iter_op;
1380
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1381
	struct iov_iter iter;
L
Linus Torvalds 已提交
1382

1383
	switch (opcode) {
L
Linus Torvalds 已提交
1384
	case IOCB_CMD_PREAD:
B
Badari Pulavarty 已提交
1385
	case IOCB_CMD_PREADV:
K
Kent Overstreet 已提交
1386 1387 1388
		mode	= FMODE_READ;
		rw	= READ;
		rw_op	= file->f_op->aio_read;
1389
		iter_op	= file->f_op->read_iter;
K
Kent Overstreet 已提交
1390 1391 1392
		goto rw_common;

	case IOCB_CMD_PWRITE:
B
Badari Pulavarty 已提交
1393
	case IOCB_CMD_PWRITEV:
K
Kent Overstreet 已提交
1394 1395 1396
		mode	= FMODE_WRITE;
		rw	= WRITE;
		rw_op	= file->f_op->aio_write;
1397
		iter_op	= file->f_op->write_iter;
K
Kent Overstreet 已提交
1398 1399 1400 1401 1402
		goto rw_common;
rw_common:
		if (unlikely(!(file->f_mode & mode)))
			return -EBADF;

1403
		if (!rw_op && !iter_op)
K
Kent Overstreet 已提交
1404 1405
			return -EINVAL;

C
Christoph Hellwig 已提交
1406 1407 1408 1409 1410 1411
		if (opcode == IOCB_CMD_PREADV || opcode == IOCB_CMD_PWRITEV)
			ret = aio_setup_vectored_rw(req, rw, buf, &nr_segs,
						&len, &iovec, compat);
		else
			ret = aio_setup_single_vector(req, rw, buf, &nr_segs,
						  len, iovec);
1412
		if (!ret)
C
Christoph Hellwig 已提交
1413
			ret = rw_verify_area(rw, file, &req->ki_pos, len);
1414
		if (ret < 0) {
1415
			if (iovec != inline_vecs)
1416
				kfree(iovec);
K
Kent Overstreet 已提交
1417
			return ret;
1418
		}
K
Kent Overstreet 已提交
1419

C
Christoph Hellwig 已提交
1420
		len = ret;
K
Kent Overstreet 已提交
1421

K
Kent Overstreet 已提交
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
		/* XXX: move/kill - rw_verify_area()? */
		/* This matches the pread()/pwrite() logic */
		if (req->ki_pos < 0) {
			ret = -EINVAL;
			break;
		}

		if (rw == WRITE)
			file_start_write(file);

1432
		if (iter_op) {
C
Christoph Hellwig 已提交
1433
			iov_iter_init(&iter, rw, iovec, nr_segs, len);
1434 1435 1436 1437
			ret = iter_op(req, &iter);
		} else {
			ret = rw_op(req, iovec, nr_segs, req->ki_pos);
		}
K
Kent Overstreet 已提交
1438 1439 1440

		if (rw == WRITE)
			file_end_write(file);
L
Linus Torvalds 已提交
1441
		break;
K
Kent Overstreet 已提交
1442

L
Linus Torvalds 已提交
1443
	case IOCB_CMD_FDSYNC:
K
Kent Overstreet 已提交
1444 1445 1446 1447
		if (!file->f_op->aio_fsync)
			return -EINVAL;

		ret = file->f_op->aio_fsync(req, 1);
L
Linus Torvalds 已提交
1448
		break;
K
Kent Overstreet 已提交
1449

L
Linus Torvalds 已提交
1450
	case IOCB_CMD_FSYNC:
K
Kent Overstreet 已提交
1451 1452 1453 1454
		if (!file->f_op->aio_fsync)
			return -EINVAL;

		ret = file->f_op->aio_fsync(req, 0);
L
Linus Torvalds 已提交
1455
		break;
K
Kent Overstreet 已提交
1456

L
Linus Torvalds 已提交
1457
	default:
K
Kent Overstreet 已提交
1458
		pr_debug("EINVAL: no operation provided\n");
K
Kent Overstreet 已提交
1459
		return -EINVAL;
L
Linus Torvalds 已提交
1460 1461
	}

1462
	if (iovec != inline_vecs)
1463 1464
		kfree(iovec);

K
Kent Overstreet 已提交
1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
	if (ret != -EIOCBQUEUED) {
		/*
		 * There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		if (unlikely(ret == -ERESTARTSYS || ret == -ERESTARTNOINTR ||
			     ret == -ERESTARTNOHAND ||
			     ret == -ERESTART_RESTARTBLOCK))
			ret = -EINTR;
		aio_complete(req, ret, 0);
	}
L
Linus Torvalds 已提交
1476 1477 1478 1479

	return 0;
}

A
Adrian Bunk 已提交
1480
static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
K
Kent Overstreet 已提交
1481
			 struct iocb *iocb, bool compat)
L
Linus Torvalds 已提交
1482 1483 1484 1485 1486
{
	struct kiocb *req;
	ssize_t ret;

	/* enforce forwards compatibility on users */
1487
	if (unlikely(iocb->aio_reserved1 || iocb->aio_reserved2)) {
K
Kent Overstreet 已提交
1488
		pr_debug("EINVAL: reserve field set\n");
L
Linus Torvalds 已提交
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
		return -EINVAL;
	}

	/* prevent overflows */
	if (unlikely(
	    (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
	    (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
	    ((ssize_t)iocb->aio_nbytes < 0)
	   )) {
		pr_debug("EINVAL: io_submit: overflow check\n");
		return -EINVAL;
	}

K
Kent Overstreet 已提交
1502
	req = aio_get_req(ctx);
1503
	if (unlikely(!req))
L
Linus Torvalds 已提交
1504
		return -EAGAIN;
1505 1506 1507 1508 1509

	req->ki_filp = fget(iocb->aio_fildes);
	if (unlikely(!req->ki_filp)) {
		ret = -EBADF;
		goto out_put_req;
L
Linus Torvalds 已提交
1510
	}
1511

1512 1513 1514 1515 1516 1517 1518
	if (iocb->aio_flags & IOCB_FLAG_RESFD) {
		/*
		 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
		 * instance of the file* now. The file descriptor must be
		 * an eventfd() fd, and will be signaled for each completed
		 * event using the eventfd_signal() function.
		 */
1519
		req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd);
1520
		if (IS_ERR(req->ki_eventfd)) {
1521
			ret = PTR_ERR(req->ki_eventfd);
1522
			req->ki_eventfd = NULL;
1523 1524 1525
			goto out_put_req;
		}
	}
L
Linus Torvalds 已提交
1526

K
Kent Overstreet 已提交
1527
	ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
L
Linus Torvalds 已提交
1528
	if (unlikely(ret)) {
K
Kent Overstreet 已提交
1529
		pr_debug("EFAULT: aio_key\n");
L
Linus Torvalds 已提交
1530 1531 1532 1533 1534 1535 1536
		goto out_put_req;
	}

	req->ki_obj.user = user_iocb;
	req->ki_user_data = iocb->aio_data;
	req->ki_pos = iocb->aio_offset;

1537 1538
	ret = aio_run_iocb(req, iocb->aio_lio_opcode,
			   (char __user *)(unsigned long)iocb->aio_buf,
C
Christoph Hellwig 已提交
1539
			   iocb->aio_nbytes,
1540
			   compat);
Z
Zach Brown 已提交
1541
	if (ret)
1542
		goto out_put_req;
Z
Zach Brown 已提交
1543

L
Linus Torvalds 已提交
1544 1545
	return 0;
out_put_req:
K
Kent Overstreet 已提交
1546
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1547
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
1548
	kiocb_free(req);
L
Linus Torvalds 已提交
1549 1550 1551
	return ret;
}

1552 1553
long do_io_submit(aio_context_t ctx_id, long nr,
		  struct iocb __user *__user *iocbpp, bool compat)
L
Linus Torvalds 已提交
1554 1555 1556
{
	struct kioctx *ctx;
	long ret = 0;
J
Jeff Moyer 已提交
1557
	int i = 0;
S
Shaohua Li 已提交
1558
	struct blk_plug plug;
L
Linus Torvalds 已提交
1559 1560 1561 1562

	if (unlikely(nr < 0))
		return -EINVAL;

1563 1564 1565
	if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
		nr = LONG_MAX/sizeof(*iocbpp);

L
Linus Torvalds 已提交
1566 1567 1568 1569 1570
	if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx)) {
K
Kent Overstreet 已提交
1571
		pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1572 1573 1574
		return -EINVAL;
	}

S
Shaohua Li 已提交
1575 1576
	blk_start_plug(&plug);

L
Linus Torvalds 已提交
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
	/*
	 * AKPM: should this return a partial result if some of the IOs were
	 * successfully submitted?
	 */
	for (i=0; i<nr; i++) {
		struct iocb __user *user_iocb;
		struct iocb tmp;

		if (unlikely(__get_user(user_iocb, iocbpp + i))) {
			ret = -EFAULT;
			break;
		}

		if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
			ret = -EFAULT;
			break;
		}

K
Kent Overstreet 已提交
1595
		ret = io_submit_one(ctx, user_iocb, &tmp, compat);
L
Linus Torvalds 已提交
1596 1597 1598
		if (ret)
			break;
	}
S
Shaohua Li 已提交
1599
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
1600

K
Kent Overstreet 已提交
1601
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1602 1603 1604
	return i ? i : ret;
}

1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
/* sys_io_submit:
 *	Queue the nr iocbs pointed to by iocbpp for processing.  Returns
 *	the number of iocbs queued.  May return -EINVAL if the aio_context
 *	specified by ctx_id is invalid, if nr is < 0, if the iocb at
 *	*iocbpp[0] is not properly initialized, if the operation specified
 *	is invalid for the file descriptor in the iocb.  May fail with
 *	-EFAULT if any of the data structures point to invalid data.  May
 *	fail with -EBADF if the file descriptor specified in the first
 *	iocb is invalid.  May fail with -EAGAIN if insufficient resources
 *	are available to queue any iocbs.  Will return 0 if nr is 0.  Will
 *	fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
		struct iocb __user * __user *, iocbpp)
{
	return do_io_submit(ctx_id, nr, iocbpp, 0);
}

L
Linus Torvalds 已提交
1623 1624 1625
/* lookup_kiocb
 *	Finds a given iocb for cancellation.
 */
1626 1627
static struct kiocb *lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb,
				  u32 key)
L
Linus Torvalds 已提交
1628 1629
{
	struct list_head *pos;
1630 1631 1632

	assert_spin_locked(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1633 1634 1635
	if (key != KIOCB_KEY)
		return NULL;

L
Linus Torvalds 已提交
1636 1637 1638
	/* TODO: use a hash or array, this sucks. */
	list_for_each(pos, &ctx->active_reqs) {
		struct kiocb *kiocb = list_kiocb(pos);
K
Kent Overstreet 已提交
1639
		if (kiocb->ki_obj.user == iocb)
L
Linus Torvalds 已提交
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
			return kiocb;
	}
	return NULL;
}

/* sys_io_cancel:
 *	Attempts to cancel an iocb previously passed to io_submit.  If
 *	the operation is successfully cancelled, the resulting event is
 *	copied into the memory pointed to by result without being placed
 *	into the completion queue and 0 is returned.  May fail with
 *	-EFAULT if any of the data structures pointed to are invalid.
 *	May fail with -EINVAL if aio_context specified by ctx_id is
 *	invalid.  May fail with -EAGAIN if the iocb specified was not
 *	cancelled.  Will fail with -ENOSYS if not implemented.
 */
1655 1656
SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
		struct io_event __user *, result)
L
Linus Torvalds 已提交
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
{
	struct kioctx *ctx;
	struct kiocb *kiocb;
	u32 key;
	int ret;

	ret = get_user(key, &iocb->aio_key);
	if (unlikely(ret))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx))
		return -EINVAL;

	spin_lock_irq(&ctx->ctx_lock);
K
Kent Overstreet 已提交
1672

L
Linus Torvalds 已提交
1673
	kiocb = lookup_kiocb(ctx, iocb, key);
K
Kent Overstreet 已提交
1674
	if (kiocb)
1675
		ret = kiocb_cancel(kiocb);
K
Kent Overstreet 已提交
1676 1677 1678
	else
		ret = -EINVAL;

L
Linus Torvalds 已提交
1679 1680
	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1681
	if (!ret) {
1682 1683 1684 1685
		/*
		 * The result argument is no longer used - the io_event is
		 * always delivered via the ring buffer. -EINPROGRESS indicates
		 * cancellation is progress:
K
Kent Overstreet 已提交
1686
		 */
1687
		ret = -EINPROGRESS;
K
Kent Overstreet 已提交
1688
	}
L
Linus Torvalds 已提交
1689

K
Kent Overstreet 已提交
1690
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1691 1692 1693 1694 1695 1696

	return ret;
}

/* io_getevents:
 *	Attempts to read at least min_nr events and up to nr events from
1697 1698 1699 1700 1701 1702 1703 1704
 *	the completion queue for the aio_context specified by ctx_id. If
 *	it succeeds, the number of read events is returned. May fail with
 *	-EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
 *	out of range, if timeout is out of range.  May fail with -EFAULT
 *	if any of the memory specified is invalid.  May return 0 or
 *	< min_nr if the timeout specified by timeout has elapsed
 *	before sufficient events are available, where timeout == NULL
 *	specifies an infinite timeout. Note that the timeout pointed to by
J
Jeff Moyer 已提交
1705
 *	timeout is relative.  Will fail with -ENOSYS if not implemented.
L
Linus Torvalds 已提交
1706
 */
1707 1708 1709 1710 1711
SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
		long, min_nr,
		long, nr,
		struct io_event __user *, events,
		struct timespec __user *, timeout)
L
Linus Torvalds 已提交
1712 1713 1714 1715 1716
{
	struct kioctx *ioctx = lookup_ioctx(ctx_id);
	long ret = -EINVAL;

	if (likely(ioctx)) {
N
Namhyung Kim 已提交
1717
		if (likely(min_nr <= nr && min_nr >= 0))
L
Linus Torvalds 已提交
1718
			ret = read_events(ioctx, min_nr, nr, events, timeout);
K
Kent Overstreet 已提交
1719
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1720 1721 1722
	}
	return ret;
}