aio.c 43.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 *	An async IO implementation for Linux
 *	Written by Benjamin LaHaise <bcrl@kvack.org>
 *
 *	Implements an efficient asynchronous io interface.
 *
 *	Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved.
 *
 *	See ../COPYING for licensing terms.
 */
K
Kent Overstreet 已提交
11 12
#define pr_fmt(fmt) "%s: " fmt, __func__

L
Linus Torvalds 已提交
13 14 15 16 17
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/aio_abi.h>
18
#include <linux/export.h>
L
Linus Torvalds 已提交
19
#include <linux/syscalls.h>
20
#include <linux/backing-dev.h>
21
#include <linux/uio.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
28
#include <linux/mmu_context.h>
K
Kent Overstreet 已提交
29
#include <linux/percpu.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/aio.h>
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
36
#include <linux/eventfd.h>
J
Jeff Moyer 已提交
37
#include <linux/blkdev.h>
38
#include <linux/compat.h>
39 40
#include <linux/migrate.h>
#include <linux/ramfs.h>
K
Kent Overstreet 已提交
41
#include <linux/percpu-refcount.h>
42
#include <linux/mount.h>
L
Linus Torvalds 已提交
43 44 45 46

#include <asm/kmap_types.h>
#include <asm/uaccess.h>

A
Al Viro 已提交
47 48
#include "internal.h"

49 50 51 52 53 54
#define AIO_RING_MAGIC			0xa10a10a1
#define AIO_RING_COMPAT_FEATURES	1
#define AIO_RING_INCOMPAT_FEATURES	0
struct aio_ring {
	unsigned	id;	/* kernel internal index number */
	unsigned	nr;	/* number of io_events */
55 56
	unsigned	head;	/* Written to by userland or under ring_lock
				 * mutex by aio_read_events_ring(). */
57 58 59 60 61 62 63 64 65 66 67 68 69
	unsigned	tail;

	unsigned	magic;
	unsigned	compat_features;
	unsigned	incompat_features;
	unsigned	header_length;	/* size of aio_ring */


	struct io_event		io_events[0];
}; /* 128 bytes + ring size */

#define AIO_RING_PAGES	8

70 71 72 73 74 75
struct kioctx_table {
	struct rcu_head	rcu;
	unsigned	nr;
	struct kioctx	*table[];
};

K
Kent Overstreet 已提交
76 77 78 79
struct kioctx_cpu {
	unsigned		reqs_available;
};

80
struct kioctx {
K
Kent Overstreet 已提交
81
	struct percpu_ref	users;
K
Kent Overstreet 已提交
82
	atomic_t		dead;
83

K
Kent Overstreet 已提交
84 85
	struct percpu_ref	reqs;

86 87
	unsigned long		user_id;

K
Kent Overstreet 已提交
88 89 90 91 92 93 94
	struct __percpu kioctx_cpu *cpu;

	/*
	 * For percpu reqs_available, number of slots we move to/from global
	 * counter at a time:
	 */
	unsigned		req_batch;
95 96 97 98
	/*
	 * This is what userspace passed to io_setup(), it's not used for
	 * anything but counting against the global max_reqs quota.
	 *
K
Kent Overstreet 已提交
99
	 * The real limit is nr_events - 1, which will be larger (see
100 101
	 * aio_setup_ring())
	 */
102 103
	unsigned		max_reqs;

K
Kent Overstreet 已提交
104 105
	/* Size of ringbuffer, in units of struct io_event */
	unsigned		nr_events;
106

K
Kent Overstreet 已提交
107 108 109 110 111 112
	unsigned long		mmap_base;
	unsigned long		mmap_size;

	struct page		**ring_pages;
	long			nr_pages;

K
Kent Overstreet 已提交
113
	struct work_struct	free_work;
114

115 116 117 118 119
	/*
	 * signals when all in-flight requests are done
	 */
	struct completion *requests_done;

120
	struct {
121 122 123 124 125
		/*
		 * This counts the number of available slots in the ringbuffer,
		 * so we avoid overflowing it: it's decremented (if positive)
		 * when allocating a kiocb and incremented when the resulting
		 * io_event is pulled off the ringbuffer.
K
Kent Overstreet 已提交
126 127
		 *
		 * We batch accesses to it with a percpu version.
128 129
		 */
		atomic_t	reqs_available;
130 131 132 133 134 135 136
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t	ctx_lock;
		struct list_head active_reqs;	/* used for cancellation */
	} ____cacheline_aligned_in_smp;

K
Kent Overstreet 已提交
137 138
	struct {
		struct mutex	ring_lock;
139 140
		wait_queue_head_t wait;
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
141 142 143

	struct {
		unsigned	tail;
144
		unsigned	completed_events;
K
Kent Overstreet 已提交
145
		spinlock_t	completion_lock;
146
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
147 148

	struct page		*internal_pages[AIO_RING_PAGES];
149
	struct file		*aio_ring_file;
150 151

	unsigned		id;
152 153
};

154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
/*
 * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
 * cancelled or completed (this makes a certain amount of sense because
 * successful cancellation - io_cancel() - does deliver the completion to
 * userspace).
 *
 * And since most things don't implement kiocb cancellation and we'd really like
 * kiocb completion to be lockless when possible, we use ki_cancel to
 * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
 * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
 */
#define KIOCB_CANCELLED		((void *) (~0ULL))

struct aio_kiocb {
	struct kiocb		common;

	struct kioctx		*ki_ctx;
	kiocb_cancel_fn		*ki_cancel;

	struct iocb __user	*ki_user_iocb;	/* user's aiocb */
	__u64			ki_user_data;	/* user's data for completion */

	struct list_head	ki_list;	/* the aio core uses this
						 * for cancellation */

	/*
	 * If the aio_resfd field of the userspace iocb is not zero,
	 * this is the underlying eventfd context to deliver events to.
	 */
	struct eventfd_ctx	*ki_eventfd;
};

L
Linus Torvalds 已提交
186
/*------ sysctl variables----*/
187 188 189
static DEFINE_SPINLOCK(aio_nr_lock);
unsigned long aio_nr;		/* current system wide number of aio requests */
unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
L
Linus Torvalds 已提交
190 191
/*----end sysctl variables---*/

192 193
static struct kmem_cache	*kiocb_cachep;
static struct kmem_cache	*kioctx_cachep;
L
Linus Torvalds 已提交
194

195 196 197 198 199 200 201 202 203 204 205
static struct vfsmount *aio_mnt;

static const struct file_operations aio_ring_fops;
static const struct address_space_operations aio_ctx_aops;

static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
{
	struct qstr this = QSTR_INIT("[aio]", 5);
	struct file *file;
	struct path path;
	struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
206 207
	if (IS_ERR(inode))
		return ERR_CAST(inode);
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236

	inode->i_mapping->a_ops = &aio_ctx_aops;
	inode->i_mapping->private_data = ctx;
	inode->i_size = PAGE_SIZE * nr_pages;

	path.dentry = d_alloc_pseudo(aio_mnt->mnt_sb, &this);
	if (!path.dentry) {
		iput(inode);
		return ERR_PTR(-ENOMEM);
	}
	path.mnt = mntget(aio_mnt);

	d_instantiate(path.dentry, inode);
	file = alloc_file(&path, FMODE_READ | FMODE_WRITE, &aio_ring_fops);
	if (IS_ERR(file)) {
		path_put(&path);
		return file;
	}

	file->f_flags = O_RDWR;
	return file;
}

static struct dentry *aio_mount(struct file_system_type *fs_type,
				int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname	= simple_dname,
	};
237
	return mount_pseudo(fs_type, "aio:", NULL, &ops, AIO_RING_MAGIC);
238 239
}

L
Linus Torvalds 已提交
240 241 242 243 244 245
/* aio_setup
 *	Creates the slab caches used by the aio routines, panic on
 *	failure as this is done early during the boot sequence.
 */
static int __init aio_setup(void)
{
246 247 248 249 250 251 252 253 254
	static struct file_system_type aio_fs = {
		.name		= "aio",
		.mount		= aio_mount,
		.kill_sb	= kill_anon_super,
	};
	aio_mnt = kern_mount(&aio_fs);
	if (IS_ERR(aio_mnt))
		panic("Failed to create aio fs mount.");

255
	kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
256
	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
L
Linus Torvalds 已提交
257

K
Kent Overstreet 已提交
258
	pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
L
Linus Torvalds 已提交
259 260 261

	return 0;
}
262
__initcall(aio_setup);
L
Linus Torvalds 已提交
263

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
static void put_aio_ring_file(struct kioctx *ctx)
{
	struct file *aio_ring_file = ctx->aio_ring_file;
	if (aio_ring_file) {
		truncate_setsize(aio_ring_file->f_inode, 0);

		/* Prevent further access to the kioctx from migratepages */
		spin_lock(&aio_ring_file->f_inode->i_mapping->private_lock);
		aio_ring_file->f_inode->i_mapping->private_data = NULL;
		ctx->aio_ring_file = NULL;
		spin_unlock(&aio_ring_file->f_inode->i_mapping->private_lock);

		fput(aio_ring_file);
	}
}

L
Linus Torvalds 已提交
280 281
static void aio_free_ring(struct kioctx *ctx)
{
282
	int i;
L
Linus Torvalds 已提交
283

284 285 286 287 288
	/* Disconnect the kiotx from the ring file.  This prevents future
	 * accesses to the kioctx from page migration.
	 */
	put_aio_ring_file(ctx);

289
	for (i = 0; i < ctx->nr_pages; i++) {
290
		struct page *page;
291 292
		pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
				page_count(ctx->ring_pages[i]));
293 294 295 296 297
		page = ctx->ring_pages[i];
		if (!page)
			continue;
		ctx->ring_pages[i] = NULL;
		put_page(page);
298
	}
L
Linus Torvalds 已提交
299

300
	if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
K
Kent Overstreet 已提交
301
		kfree(ctx->ring_pages);
302 303
		ctx->ring_pages = NULL;
	}
304 305 306 307
}

static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
{
308
	vma->vm_flags |= VM_DONTEXPAND;
309 310 311 312
	vma->vm_ops = &generic_file_vm_ops;
	return 0;
}

313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
static void aio_ring_remap(struct file *file, struct vm_area_struct *vma)
{
	struct mm_struct *mm = vma->vm_mm;
	struct kioctx_table *table;
	int i;

	spin_lock(&mm->ioctx_lock);
	rcu_read_lock();
	table = rcu_dereference(mm->ioctx_table);
	for (i = 0; i < table->nr; i++) {
		struct kioctx *ctx;

		ctx = table->table[i];
		if (ctx && ctx->aio_ring_file == file) {
			ctx->user_id = ctx->mmap_base = vma->vm_start;
			break;
		}
	}

	rcu_read_unlock();
	spin_unlock(&mm->ioctx_lock);
}

336 337
static const struct file_operations aio_ring_fops = {
	.mmap = aio_ring_mmap,
338
	.mremap = aio_ring_remap,
339 340
};

341
#if IS_ENABLED(CONFIG_MIGRATION)
342 343 344
static int aio_migratepage(struct address_space *mapping, struct page *new,
			struct page *old, enum migrate_mode mode)
{
345
	struct kioctx *ctx;
346
	unsigned long flags;
347
	pgoff_t idx;
348 349
	int rc;

350 351
	rc = 0;

352
	/* mapping->private_lock here protects against the kioctx teardown.  */
353 354
	spin_lock(&mapping->private_lock);
	ctx = mapping->private_data;
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
	if (!ctx) {
		rc = -EINVAL;
		goto out;
	}

	/* The ring_lock mutex.  The prevents aio_read_events() from writing
	 * to the ring's head, and prevents page migration from mucking in
	 * a partially initialized kiotx.
	 */
	if (!mutex_trylock(&ctx->ring_lock)) {
		rc = -EAGAIN;
		goto out;
	}

	idx = old->index;
	if (idx < (pgoff_t)ctx->nr_pages) {
		/* Make sure the old page hasn't already been changed */
		if (ctx->ring_pages[idx] != old)
			rc = -EAGAIN;
374 375 376 377
	} else
		rc = -EINVAL;

	if (rc != 0)
378
		goto out_unlock;
379

380 381
	/* Writeback must be complete */
	BUG_ON(PageWriteback(old));
382
	get_page(new);
383

384
	rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1);
385
	if (rc != MIGRATEPAGE_SUCCESS) {
386
		put_page(new);
387
		goto out_unlock;
388 389
	}

390 391 392
	/* Take completion_lock to prevent other writes to the ring buffer
	 * while the old page is copied to the new.  This prevents new
	 * events from being lost.
393
	 */
394 395 396 397 398
	spin_lock_irqsave(&ctx->completion_lock, flags);
	migrate_page_copy(new, old);
	BUG_ON(ctx->ring_pages[idx] != old);
	ctx->ring_pages[idx] = new;
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
399

400 401
	/* The old page is no longer accessible. */
	put_page(old);
402

403 404 405 406
out_unlock:
	mutex_unlock(&ctx->ring_lock);
out:
	spin_unlock(&mapping->private_lock);
407
	return rc;
L
Linus Torvalds 已提交
408
}
409
#endif
L
Linus Torvalds 已提交
410

411
static const struct address_space_operations aio_ctx_aops = {
412
	.set_page_dirty = __set_page_dirty_no_writeback,
413
#if IS_ENABLED(CONFIG_MIGRATION)
414
	.migratepage	= aio_migratepage,
415
#endif
416 417
};

L
Linus Torvalds 已提交
418 419 420 421
static int aio_setup_ring(struct kioctx *ctx)
{
	struct aio_ring *ring;
	unsigned nr_events = ctx->max_reqs;
Z
Zach Brown 已提交
422
	struct mm_struct *mm = current->mm;
423
	unsigned long size, unused;
L
Linus Torvalds 已提交
424
	int nr_pages;
425 426
	int i;
	struct file *file;
L
Linus Torvalds 已提交
427 428 429 430 431 432 433

	/* Compensate for the ring buffer's head/tail overlap entry */
	nr_events += 2;	/* 1 is required, 2 for good luck */

	size = sizeof(struct aio_ring);
	size += sizeof(struct io_event) * nr_events;

434
	nr_pages = PFN_UP(size);
L
Linus Torvalds 已提交
435 436 437
	if (nr_pages < 0)
		return -EINVAL;

438
	file = aio_private_file(ctx, nr_pages);
439 440
	if (IS_ERR(file)) {
		ctx->aio_ring_file = NULL;
441
		return -ENOMEM;
442 443
	}

444 445 446 447 448 449 450 451 452 453 454 455 456 457
	ctx->aio_ring_file = file;
	nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
			/ sizeof(struct io_event);

	ctx->ring_pages = ctx->internal_pages;
	if (nr_pages > AIO_RING_PAGES) {
		ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
					  GFP_KERNEL);
		if (!ctx->ring_pages) {
			put_aio_ring_file(ctx);
			return -ENOMEM;
		}
	}

458 459 460 461 462 463 464 465 466 467
	for (i = 0; i < nr_pages; i++) {
		struct page *page;
		page = find_or_create_page(file->f_inode->i_mapping,
					   i, GFP_HIGHUSER | __GFP_ZERO);
		if (!page)
			break;
		pr_debug("pid(%d) page[%d]->count=%d\n",
			 current->pid, i, page_count(page));
		SetPageUptodate(page);
		unlock_page(page);
468 469

		ctx->ring_pages[i] = page;
470
	}
471
	ctx->nr_pages = i;
L
Linus Torvalds 已提交
472

473 474
	if (unlikely(i != nr_pages)) {
		aio_free_ring(ctx);
475
		return -ENOMEM;
L
Linus Torvalds 已提交
476 477
	}

K
Kent Overstreet 已提交
478 479
	ctx->mmap_size = nr_pages * PAGE_SIZE;
	pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
480

Z
Zach Brown 已提交
481
	down_write(&mm->mmap_sem);
482 483
	ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
				       PROT_READ | PROT_WRITE,
484 485
				       MAP_SHARED, 0, &unused);
	up_write(&mm->mmap_sem);
K
Kent Overstreet 已提交
486 487
	if (IS_ERR((void *)ctx->mmap_base)) {
		ctx->mmap_size = 0;
L
Linus Torvalds 已提交
488
		aio_free_ring(ctx);
489
		return -ENOMEM;
L
Linus Torvalds 已提交
490 491
	}

K
Kent Overstreet 已提交
492
	pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
493

K
Kent Overstreet 已提交
494 495
	ctx->user_id = ctx->mmap_base;
	ctx->nr_events = nr_events; /* trusted copy */
L
Linus Torvalds 已提交
496

K
Kent Overstreet 已提交
497
	ring = kmap_atomic(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
498
	ring->nr = nr_events;	/* user copy */
499
	ring->id = ~0U;
L
Linus Torvalds 已提交
500 501 502 503 504
	ring->head = ring->tail = 0;
	ring->magic = AIO_RING_MAGIC;
	ring->compat_features = AIO_RING_COMPAT_FEATURES;
	ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
	ring->header_length = sizeof(struct aio_ring);
505
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
506
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
507 508 509 510 511 512 513 514

	return 0;
}

#define AIO_EVENTS_PER_PAGE	(PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE	((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET	(AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)

515
void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
516
{
517
	struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common);
518 519 520 521 522 523 524 525 526 527 528 529 530 531
	struct kioctx *ctx = req->ki_ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->ctx_lock, flags);

	if (!req->ki_list.next)
		list_add(&req->ki_list, &ctx->active_reqs);

	req->ki_cancel = cancel;

	spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
EXPORT_SYMBOL(kiocb_set_cancel_fn);

532
static int kiocb_cancel(struct aio_kiocb *kiocb)
K
Kent Overstreet 已提交
533
{
534
	kiocb_cancel_fn *old, *cancel;
K
Kent Overstreet 已提交
535

536 537 538 539 540 541 542 543
	/*
	 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
	 * actually has a cancel function, hence the cmpxchg()
	 */

	cancel = ACCESS_ONCE(kiocb->ki_cancel);
	do {
		if (!cancel || cancel == KIOCB_CANCELLED)
K
Kent Overstreet 已提交
544
			return -EINVAL;
K
Kent Overstreet 已提交
545

546 547 548
		old = cancel;
		cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
	} while (cancel != old);
K
Kent Overstreet 已提交
549

550
	return cancel(&kiocb->common);
K
Kent Overstreet 已提交
551 552
}

K
Kent Overstreet 已提交
553
static void free_ioctx(struct work_struct *work)
K
Kent Overstreet 已提交
554
{
K
Kent Overstreet 已提交
555
	struct kioctx *ctx = container_of(work, struct kioctx, free_work);
K
Kent Overstreet 已提交
556

K
Kent Overstreet 已提交
557
	pr_debug("freeing %p\n", ctx);
K
Kent Overstreet 已提交
558

K
Kent Overstreet 已提交
559
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
560
	free_percpu(ctx->cpu);
561 562
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
K
Kent Overstreet 已提交
563 564 565
	kmem_cache_free(kioctx_cachep, ctx);
}

K
Kent Overstreet 已提交
566 567 568 569
static void free_ioctx_reqs(struct percpu_ref *ref)
{
	struct kioctx *ctx = container_of(ref, struct kioctx, reqs);

570 571 572 573
	/* At this point we know that there are no any in-flight requests */
	if (ctx->requests_done)
		complete(ctx->requests_done);

K
Kent Overstreet 已提交
574 575 576 577
	INIT_WORK(&ctx->free_work, free_ioctx);
	schedule_work(&ctx->free_work);
}

K
Kent Overstreet 已提交
578 579 580 581 582
/*
 * When this function runs, the kioctx has been removed from the "hash table"
 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
 * now it's safe to cancel any that need to be.
 */
K
Kent Overstreet 已提交
583
static void free_ioctx_users(struct percpu_ref *ref)
K
Kent Overstreet 已提交
584
{
K
Kent Overstreet 已提交
585
	struct kioctx *ctx = container_of(ref, struct kioctx, users);
586
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
587 588 589 590 591

	spin_lock_irq(&ctx->ctx_lock);

	while (!list_empty(&ctx->active_reqs)) {
		req = list_first_entry(&ctx->active_reqs,
592
				       struct aio_kiocb, ki_list);
K
Kent Overstreet 已提交
593 594

		list_del_init(&req->ki_list);
595
		kiocb_cancel(req);
K
Kent Overstreet 已提交
596 597 598 599
	}

	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
600 601
	percpu_ref_kill(&ctx->reqs);
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
602 603
}

604 605 606 607 608 609 610
static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
{
	unsigned i, new_nr;
	struct kioctx_table *table, *old;
	struct aio_ring *ring;

	spin_lock(&mm->ioctx_lock);
611
	table = rcu_dereference_raw(mm->ioctx_table);
612 613 614 615 616 617 618 619 620

	while (1) {
		if (table)
			for (i = 0; i < table->nr; i++)
				if (!table->table[i]) {
					ctx->id = i;
					table->table[i] = ctx;
					spin_unlock(&mm->ioctx_lock);

621 622 623 624
					/* While kioctx setup is in progress,
					 * we are protected from page migration
					 * changes ring_pages by ->ring_lock.
					 */
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
					ring = kmap_atomic(ctx->ring_pages[0]);
					ring->id = ctx->id;
					kunmap_atomic(ring);
					return 0;
				}

		new_nr = (table ? table->nr : 1) * 4;
		spin_unlock(&mm->ioctx_lock);

		table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
				new_nr, GFP_KERNEL);
		if (!table)
			return -ENOMEM;

		table->nr = new_nr;

		spin_lock(&mm->ioctx_lock);
642
		old = rcu_dereference_raw(mm->ioctx_table);
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658

		if (!old) {
			rcu_assign_pointer(mm->ioctx_table, table);
		} else if (table->nr > old->nr) {
			memcpy(table->table, old->table,
			       old->nr * sizeof(struct kioctx *));

			rcu_assign_pointer(mm->ioctx_table, table);
			kfree_rcu(old, rcu);
		} else {
			kfree(table);
			table = old;
		}
	}
}

K
Kent Overstreet 已提交
659 660 661 662 663 664 665 666 667 668
static void aio_nr_sub(unsigned nr)
{
	spin_lock(&aio_nr_lock);
	if (WARN_ON(aio_nr - nr > aio_nr))
		aio_nr = 0;
	else
		aio_nr -= nr;
	spin_unlock(&aio_nr_lock);
}

L
Linus Torvalds 已提交
669 670 671 672 673
/* ioctx_alloc
 *	Allocates and initializes an ioctx.  Returns an ERR_PTR if it failed.
 */
static struct kioctx *ioctx_alloc(unsigned nr_events)
{
Z
Zach Brown 已提交
674
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
675
	struct kioctx *ctx;
676
	int err = -ENOMEM;
L
Linus Torvalds 已提交
677

K
Kent Overstreet 已提交
678 679 680 681 682 683 684 685 686 687 688 689
	/*
	 * We keep track of the number of available ringbuffer slots, to prevent
	 * overflow (reqs_available), and we also use percpu counters for this.
	 *
	 * So since up to half the slots might be on other cpu's percpu counters
	 * and unavailable, double nr_events so userspace sees what they
	 * expected: additionally, we move req_batch slots to/from percpu
	 * counters at a time, so make sure that isn't 0:
	 */
	nr_events = max(nr_events, num_possible_cpus() * 4);
	nr_events *= 2;

L
Linus Torvalds 已提交
690 691 692 693 694 695 696
	/* Prevent overflows */
	if ((nr_events > (0x10000000U / sizeof(struct io_event))) ||
	    (nr_events > (0x10000000U / sizeof(struct kiocb)))) {
		pr_debug("ENOMEM: nr_events too high\n");
		return ERR_PTR(-EINVAL);
	}

697
	if (!nr_events || (unsigned long)nr_events > (aio_max_nr * 2UL))
L
Linus Torvalds 已提交
698 699
		return ERR_PTR(-EAGAIN);

700
	ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
701 702 703 704 705 706
	if (!ctx)
		return ERR_PTR(-ENOMEM);

	ctx->max_reqs = nr_events;

	spin_lock_init(&ctx->ctx_lock);
707
	spin_lock_init(&ctx->completion_lock);
K
Kent Overstreet 已提交
708
	mutex_init(&ctx->ring_lock);
709 710 711
	/* Protect against page migration throughout kiotx setup by keeping
	 * the ring_lock mutex held until setup is complete. */
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
712 713 714 715
	init_waitqueue_head(&ctx->wait);

	INIT_LIST_HEAD(&ctx->active_reqs);

716
	if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
717 718
		goto err;

719
	if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
720 721
		goto err;

K
Kent Overstreet 已提交
722 723
	ctx->cpu = alloc_percpu(struct kioctx_cpu);
	if (!ctx->cpu)
K
Kent Overstreet 已提交
724
		goto err;
L
Linus Torvalds 已提交
725

726 727
	err = aio_setup_ring(ctx);
	if (err < 0)
K
Kent Overstreet 已提交
728
		goto err;
K
Kent Overstreet 已提交
729

730
	atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
K
Kent Overstreet 已提交
731
	ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
732 733
	if (ctx->req_batch < 1)
		ctx->req_batch = 1;
734

L
Linus Torvalds 已提交
735
	/* limit the number of system wide aios */
736
	spin_lock(&aio_nr_lock);
737
	if (aio_nr + nr_events > (aio_max_nr * 2UL) ||
738
	    aio_nr + nr_events < aio_nr) {
739
		spin_unlock(&aio_nr_lock);
K
Kent Overstreet 已提交
740
		err = -EAGAIN;
G
Gu Zheng 已提交
741
		goto err_ctx;
742 743
	}
	aio_nr += ctx->max_reqs;
744
	spin_unlock(&aio_nr_lock);
L
Linus Torvalds 已提交
745

746 747
	percpu_ref_get(&ctx->users);	/* io_setup() will drop this ref */
	percpu_ref_get(&ctx->reqs);	/* free_ioctx_users() will drop this */
K
Kent Overstreet 已提交
748

749 750
	err = ioctx_add_table(ctx, mm);
	if (err)
K
Kent Overstreet 已提交
751
		goto err_cleanup;
752

753 754 755
	/* Release the ring_lock mutex now that all setup is complete. */
	mutex_unlock(&ctx->ring_lock);

K
Kent Overstreet 已提交
756
	pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
K
Kent Overstreet 已提交
757
		 ctx, ctx->user_id, mm, ctx->nr_events);
L
Linus Torvalds 已提交
758 759
	return ctx;

K
Kent Overstreet 已提交
760 761
err_cleanup:
	aio_nr_sub(ctx->max_reqs);
G
Gu Zheng 已提交
762 763
err_ctx:
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
764
err:
765
	mutex_unlock(&ctx->ring_lock);
K
Kent Overstreet 已提交
766
	free_percpu(ctx->cpu);
767 768
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
L
Linus Torvalds 已提交
769
	kmem_cache_free(kioctx_cachep, ctx);
K
Kent Overstreet 已提交
770
	pr_debug("error allocating ioctx %d\n", err);
771
	return ERR_PTR(err);
L
Linus Torvalds 已提交
772 773
}

K
Kent Overstreet 已提交
774 775 776 777 778
/* kill_ioctx
 *	Cancels all outstanding aio requests on an aio context.  Used
 *	when the processes owning a context have all exited to encourage
 *	the rapid destruction of the kioctx.
 */
779
static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
780
		struct completion *requests_done)
K
Kent Overstreet 已提交
781
{
782
	struct kioctx_table *table;
783

784 785
	if (atomic_xchg(&ctx->dead, 1))
		return -EINVAL;
786 787


788
	spin_lock(&mm->ioctx_lock);
789
	table = rcu_dereference_raw(mm->ioctx_table);
790 791 792
	WARN_ON(ctx != table->table[ctx->id]);
	table->table[ctx->id] = NULL;
	spin_unlock(&mm->ioctx_lock);
793

794 795
	/* percpu_ref_kill() will do the necessary call_rcu() */
	wake_up_all(&ctx->wait);
796

797 798 799 800 801 802 803 804
	/*
	 * It'd be more correct to do this in free_ioctx(), after all
	 * the outstanding kiocbs have finished - but by then io_destroy
	 * has already returned, so io_setup() could potentially return
	 * -EAGAIN with no ioctxs actually in use (as far as userspace
	 *  could tell).
	 */
	aio_nr_sub(ctx->max_reqs);
805

806 807
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
808

809 810 811
	ctx->requests_done = requests_done;
	percpu_ref_kill(&ctx->users);
	return 0;
L
Linus Torvalds 已提交
812 813
}

K
Kent Overstreet 已提交
814 815 816 817 818 819 820
/*
 * exit_aio: called when the last user of mm goes away.  At this point, there is
 * no way for any new requests to be submited or any of the io_* syscalls to be
 * called on the context.
 *
 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
 * them.
L
Linus Torvalds 已提交
821
 */
822
void exit_aio(struct mm_struct *mm)
L
Linus Torvalds 已提交
823
{
824 825
	struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
	int i;
826

827 828
	if (!table)
		return;
829

830 831
	for (i = 0; i < table->nr; ++i) {
		struct kioctx *ctx = table->table[i];
832 833
		struct completion requests_done =
			COMPLETION_INITIALIZER_ONSTACK(requests_done);
J
Jens Axboe 已提交
834

835 836
		if (!ctx)
			continue;
837
		/*
838 839 840 841 842
		 * We don't need to bother with munmap() here - exit_mmap(mm)
		 * is coming and it'll unmap everything. And we simply can't,
		 * this is not necessarily our ->mm.
		 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
		 * that it needs to unmap the area, just set it to 0.
843
		 */
K
Kent Overstreet 已提交
844
		ctx->mmap_size = 0;
845
		kill_ioctx(mm, ctx, &requests_done);
K
Kent Overstreet 已提交
846

847 848
		/* Wait until all IO for the context are done. */
		wait_for_completion(&requests_done);
L
Linus Torvalds 已提交
849
	}
850 851 852

	RCU_INIT_POINTER(mm->ioctx_table, NULL);
	kfree(table);
L
Linus Torvalds 已提交
853 854
}

K
Kent Overstreet 已提交
855 856 857
static void put_reqs_available(struct kioctx *ctx, unsigned nr)
{
	struct kioctx_cpu *kcpu;
858
	unsigned long flags;
K
Kent Overstreet 已提交
859

860
	local_irq_save(flags);
861
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
862
	kcpu->reqs_available += nr;
863

K
Kent Overstreet 已提交
864 865 866 867 868
	while (kcpu->reqs_available >= ctx->req_batch * 2) {
		kcpu->reqs_available -= ctx->req_batch;
		atomic_add(ctx->req_batch, &ctx->reqs_available);
	}

869
	local_irq_restore(flags);
K
Kent Overstreet 已提交
870 871 872 873 874 875
}

static bool get_reqs_available(struct kioctx *ctx)
{
	struct kioctx_cpu *kcpu;
	bool ret = false;
876
	unsigned long flags;
K
Kent Overstreet 已提交
877

878
	local_irq_save(flags);
879
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
	if (!kcpu->reqs_available) {
		int old, avail = atomic_read(&ctx->reqs_available);

		do {
			if (avail < ctx->req_batch)
				goto out;

			old = avail;
			avail = atomic_cmpxchg(&ctx->reqs_available,
					       avail, avail - ctx->req_batch);
		} while (avail != old);

		kcpu->reqs_available += ctx->req_batch;
	}

	ret = true;
	kcpu->reqs_available--;
out:
898
	local_irq_restore(flags);
K
Kent Overstreet 已提交
899 900 901
	return ret;
}

902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
/* refill_reqs_available
 *	Updates the reqs_available reference counts used for tracking the
 *	number of free slots in the completion ring.  This can be called
 *	from aio_complete() (to optimistically update reqs_available) or
 *	from aio_get_req() (the we're out of events case).  It must be
 *	called holding ctx->completion_lock.
 */
static void refill_reqs_available(struct kioctx *ctx, unsigned head,
                                  unsigned tail)
{
	unsigned events_in_ring, completed;

	/* Clamp head since userland can write to it. */
	head %= ctx->nr_events;
	if (head <= tail)
		events_in_ring = tail - head;
	else
		events_in_ring = ctx->nr_events - (head - tail);

	completed = ctx->completed_events;
	if (events_in_ring < completed)
		completed -= events_in_ring;
	else
		completed = 0;

	if (!completed)
		return;

	ctx->completed_events -= completed;
	put_reqs_available(ctx, completed);
}

/* user_refill_reqs_available
 *	Called to refill reqs_available when aio_get_req() encounters an
 *	out of space in the completion ring.
 */
static void user_refill_reqs_available(struct kioctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	if (ctx->completed_events) {
		struct aio_ring *ring;
		unsigned head;

		/* Access of ring->head may race with aio_read_events_ring()
		 * here, but that's okay since whether we read the old version
		 * or the new version, and either will be valid.  The important
		 * part is that head cannot pass tail since we prevent
		 * aio_complete() from updating tail by holding
		 * ctx->completion_lock.  Even if head is invalid, the check
		 * against ctx->completed_events below will make sure we do the
		 * safe/right thing.
		 */
		ring = kmap_atomic(ctx->ring_pages[0]);
		head = ring->head;
		kunmap_atomic(ring);

		refill_reqs_available(ctx, head, ctx->tail);
	}

	spin_unlock_irq(&ctx->completion_lock);
}

L
Linus Torvalds 已提交
964
/* aio_get_req
K
Kent Overstreet 已提交
965 966
 *	Allocate a slot for an aio request.
 * Returns NULL if no requests are free.
L
Linus Torvalds 已提交
967
 */
968
static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
L
Linus Torvalds 已提交
969
{
970
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
971

972 973 974 975 976
	if (!get_reqs_available(ctx)) {
		user_refill_reqs_available(ctx);
		if (!get_reqs_available(ctx))
			return NULL;
	}
K
Kent Overstreet 已提交
977

978
	req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
L
Linus Torvalds 已提交
979
	if (unlikely(!req))
K
Kent Overstreet 已提交
980
		goto out_put;
L
Linus Torvalds 已提交
981

K
Kent Overstreet 已提交
982 983
	percpu_ref_get(&ctx->reqs);

L
Linus Torvalds 已提交
984
	req->ki_ctx = ctx;
J
Jeff Moyer 已提交
985
	return req;
K
Kent Overstreet 已提交
986
out_put:
K
Kent Overstreet 已提交
987
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
988
	return NULL;
L
Linus Torvalds 已提交
989 990
}

991
static void kiocb_free(struct aio_kiocb *req)
L
Linus Torvalds 已提交
992
{
993 994
	if (req->common.ki_filp)
		fput(req->common.ki_filp);
995 996
	if (req->ki_eventfd != NULL)
		eventfd_ctx_put(req->ki_eventfd);
L
Linus Torvalds 已提交
997 998 999
	kmem_cache_free(kiocb_cachep, req);
}

A
Adrian Bunk 已提交
1000
static struct kioctx *lookup_ioctx(unsigned long ctx_id)
L
Linus Torvalds 已提交
1001
{
1002
	struct aio_ring __user *ring  = (void __user *)ctx_id;
J
Jens Axboe 已提交
1003
	struct mm_struct *mm = current->mm;
1004
	struct kioctx *ctx, *ret = NULL;
1005 1006 1007 1008 1009
	struct kioctx_table *table;
	unsigned id;

	if (get_user(id, &ring->id))
		return NULL;
L
Linus Torvalds 已提交
1010

J
Jens Axboe 已提交
1011
	rcu_read_lock();
1012
	table = rcu_dereference(mm->ioctx_table);
J
Jens Axboe 已提交
1013

1014 1015
	if (!table || id >= table->nr)
		goto out;
L
Linus Torvalds 已提交
1016

1017
	ctx = table->table[id];
1018
	if (ctx && ctx->user_id == ctx_id) {
1019 1020 1021 1022
		percpu_ref_get(&ctx->users);
		ret = ctx;
	}
out:
J
Jens Axboe 已提交
1023
	rcu_read_unlock();
1024
	return ret;
L
Linus Torvalds 已提交
1025 1026 1027 1028 1029
}

/* aio_complete
 *	Called when the io request on the given iocb is complete.
 */
1030
static void aio_complete(struct kiocb *kiocb, long res, long res2)
L
Linus Torvalds 已提交
1031
{
1032
	struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, common);
L
Linus Torvalds 已提交
1033 1034
	struct kioctx	*ctx = iocb->ki_ctx;
	struct aio_ring	*ring;
K
Kent Overstreet 已提交
1035
	struct io_event	*ev_page, *event;
1036
	unsigned tail, pos, head;
L
Linus Torvalds 已提交
1037 1038
	unsigned long	flags;

1039 1040 1041 1042 1043 1044
	/*
	 * Special case handling for sync iocbs:
	 *  - events go directly into the iocb for fast handling
	 *  - the sync task with the iocb in its stack holds the single iocb
	 *    ref, no other paths have a way to get another ref
	 *  - the sync task helpfully left a reference to itself in the iocb
L
Linus Torvalds 已提交
1045
	 */
1046
	BUG_ON(is_sync_kiocb(kiocb));
L
Linus Torvalds 已提交
1047

1048 1049 1050 1051 1052 1053 1054
	if (iocb->ki_list.next) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->ctx_lock, flags);
		list_del(&iocb->ki_list);
		spin_unlock_irqrestore(&ctx->ctx_lock, flags);
	}
1055

1056 1057
	/*
	 * Add a completion event to the ring buffer. Must be done holding
1058
	 * ctx->completion_lock to prevent other code from messing with the tail
1059 1060 1061 1062
	 * pointer since we might be called from irq context.
	 */
	spin_lock_irqsave(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1063
	tail = ctx->tail;
K
Kent Overstreet 已提交
1064 1065
	pos = tail + AIO_EVENTS_OFFSET;

K
Kent Overstreet 已提交
1066
	if (++tail >= ctx->nr_events)
1067
		tail = 0;
L
Linus Torvalds 已提交
1068

K
Kent Overstreet 已提交
1069
	ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1070 1071
	event = ev_page + pos % AIO_EVENTS_PER_PAGE;

1072
	event->obj = (u64)(unsigned long)iocb->ki_user_iocb;
L
Linus Torvalds 已提交
1073 1074 1075 1076
	event->data = iocb->ki_user_data;
	event->res = res;
	event->res2 = res2;

K
Kent Overstreet 已提交
1077
	kunmap_atomic(ev_page);
K
Kent Overstreet 已提交
1078
	flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1079 1080

	pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
1081
		 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data,
K
Kent Overstreet 已提交
1082
		 res, res2);
L
Linus Torvalds 已提交
1083 1084 1085 1086 1087 1088

	/* after flagging the request as done, we
	 * must never even look at it again
	 */
	smp_wmb();	/* make event visible before updating tail */

K
Kent Overstreet 已提交
1089
	ctx->tail = tail;
L
Linus Torvalds 已提交
1090

K
Kent Overstreet 已提交
1091
	ring = kmap_atomic(ctx->ring_pages[0]);
1092
	head = ring->head;
K
Kent Overstreet 已提交
1093
	ring->tail = tail;
1094
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1095
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
1096

1097 1098 1099
	ctx->completed_events++;
	if (ctx->completed_events > 1)
		refill_reqs_available(ctx, head, tail);
1100 1101
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1102
	pr_debug("added to ring %p at [%u]\n", iocb, tail);
D
Davide Libenzi 已提交
1103 1104 1105 1106 1107 1108

	/*
	 * Check if the user asked us to deliver the result through an
	 * eventfd. The eventfd_signal() function is safe to be called
	 * from IRQ context.
	 */
1109
	if (iocb->ki_eventfd != NULL)
D
Davide Libenzi 已提交
1110 1111
		eventfd_signal(iocb->ki_eventfd, 1);

L
Linus Torvalds 已提交
1112
	/* everything turned out well, dispose of the aiocb. */
K
Kent Overstreet 已提交
1113
	kiocb_free(iocb);
L
Linus Torvalds 已提交
1114

1115 1116 1117 1118 1119 1120 1121 1122
	/*
	 * We have to order our ring_info tail store above and test
	 * of the wait list below outside the wait lock.  This is
	 * like in wake_up_bit() where clearing a bit has to be
	 * ordered with the unlocked test.
	 */
	smp_mb();

L
Linus Torvalds 已提交
1123 1124 1125
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);

K
Kent Overstreet 已提交
1126
	percpu_ref_put(&ctx->reqs);
L
Linus Torvalds 已提交
1127 1128
}

G
Gu Zheng 已提交
1129
/* aio_read_events_ring
1130 1131
 *	Pull an event off of the ioctx's event ring.  Returns the number of
 *	events fetched
L
Linus Torvalds 已提交
1132
 */
1133 1134
static long aio_read_events_ring(struct kioctx *ctx,
				 struct io_event __user *event, long nr)
L
Linus Torvalds 已提交
1135 1136
{
	struct aio_ring *ring;
1137
	unsigned head, tail, pos;
1138 1139 1140
	long ret = 0;
	int copy_ret;

1141 1142 1143 1144 1145 1146 1147
	/*
	 * The mutex can block and wake us up and that will cause
	 * wait_event_interruptible_hrtimeout() to schedule without sleeping
	 * and repeat. This should be rare enough that it doesn't cause
	 * peformance issues. See the comment in read_events() for more detail.
	 */
	sched_annotate_sleep();
K
Kent Overstreet 已提交
1148
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
1149

1150
	/* Access to ->ring_pages here is protected by ctx->ring_lock. */
K
Kent Overstreet 已提交
1151
	ring = kmap_atomic(ctx->ring_pages[0]);
1152
	head = ring->head;
1153
	tail = ring->tail;
1154 1155
	kunmap_atomic(ring);

1156 1157 1158 1159 1160 1161
	/*
	 * Ensure that once we've read the current tail pointer, that
	 * we also see the events that were stored up to the tail.
	 */
	smp_rmb();

1162
	pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
L
Linus Torvalds 已提交
1163

1164
	if (head == tail)
L
Linus Torvalds 已提交
1165 1166
		goto out;

1167 1168 1169
	head %= ctx->nr_events;
	tail %= ctx->nr_events;

1170 1171 1172 1173 1174
	while (ret < nr) {
		long avail;
		struct io_event *ev;
		struct page *page;

1175 1176
		avail = (head <= tail ?  tail : ctx->nr_events) - head;
		if (head == tail)
1177 1178 1179 1180 1181 1182 1183
			break;

		avail = min(avail, nr - ret);
		avail = min_t(long, avail, AIO_EVENTS_PER_PAGE -
			    ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE));

		pos = head + AIO_EVENTS_OFFSET;
K
Kent Overstreet 已提交
1184
		page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
		pos %= AIO_EVENTS_PER_PAGE;

		ev = kmap(page);
		copy_ret = copy_to_user(event + ret, ev + pos,
					sizeof(*ev) * avail);
		kunmap(page);

		if (unlikely(copy_ret)) {
			ret = -EFAULT;
			goto out;
		}

		ret += avail;
		head += avail;
K
Kent Overstreet 已提交
1199
		head %= ctx->nr_events;
L
Linus Torvalds 已提交
1200 1201
	}

K
Kent Overstreet 已提交
1202
	ring = kmap_atomic(ctx->ring_pages[0]);
1203
	ring->head = head;
1204
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1205
	flush_dcache_page(ctx->ring_pages[0]);
1206

1207
	pr_debug("%li  h%u t%u\n", ret, head, tail);
1208
out:
K
Kent Overstreet 已提交
1209
	mutex_unlock(&ctx->ring_lock);
1210

L
Linus Torvalds 已提交
1211 1212 1213
	return ret;
}

1214 1215
static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
			    struct io_event __user *event, long *i)
L
Linus Torvalds 已提交
1216
{
1217
	long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
L
Linus Torvalds 已提交
1218

1219 1220
	if (ret > 0)
		*i += ret;
L
Linus Torvalds 已提交
1221

1222 1223
	if (unlikely(atomic_read(&ctx->dead)))
		ret = -EINVAL;
L
Linus Torvalds 已提交
1224

1225 1226
	if (!*i)
		*i = ret;
L
Linus Torvalds 已提交
1227

1228
	return ret < 0 || *i >= min_nr;
L
Linus Torvalds 已提交
1229 1230
}

1231
static long read_events(struct kioctx *ctx, long min_nr, long nr,
L
Linus Torvalds 已提交
1232 1233 1234
			struct io_event __user *event,
			struct timespec __user *timeout)
{
1235 1236
	ktime_t until = { .tv64 = KTIME_MAX };
	long ret = 0;
L
Linus Torvalds 已提交
1237 1238 1239

	if (timeout) {
		struct timespec	ts;
1240

L
Linus Torvalds 已提交
1241
		if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
1242
			return -EFAULT;
L
Linus Torvalds 已提交
1243

1244
		until = timespec_to_ktime(ts);
L
Linus Torvalds 已提交
1245 1246
	}

1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
	/*
	 * Note that aio_read_events() is being called as the conditional - i.e.
	 * we're calling it after prepare_to_wait() has set task state to
	 * TASK_INTERRUPTIBLE.
	 *
	 * But aio_read_events() can block, and if it blocks it's going to flip
	 * the task state back to TASK_RUNNING.
	 *
	 * This should be ok, provided it doesn't flip the state back to
	 * TASK_RUNNING and return 0 too much - that causes us to spin. That
	 * will only happen if the mutex_lock() call blocks, and we then find
	 * the ringbuffer empty. So in practice we should be ok, but it's
	 * something to be aware of when touching this code.
	 */
1261 1262 1263 1264 1265 1266
	if (until.tv64 == 0)
		aio_read_events(ctx, min_nr, nr, event, &ret);
	else
		wait_event_interruptible_hrtimeout(ctx->wait,
				aio_read_events(ctx, min_nr, nr, event, &ret),
				until);
L
Linus Torvalds 已提交
1267

1268 1269
	if (!ret && signal_pending(current))
		ret = -EINTR;
L
Linus Torvalds 已提交
1270

1271
	return ret;
L
Linus Torvalds 已提交
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
}

/* sys_io_setup:
 *	Create an aio_context capable of receiving at least nr_events.
 *	ctxp must not point to an aio_context that already exists, and
 *	must be initialized to 0 prior to the call.  On successful
 *	creation of the aio_context, *ctxp is filled in with the resulting 
 *	handle.  May fail with -EINVAL if *ctxp is not initialized,
 *	if the specified nr_events exceeds internal limits.  May fail 
 *	with -EAGAIN if the specified nr_events exceeds the user's limit 
 *	of available events.  May fail with -ENOMEM if insufficient kernel
 *	resources are available.  May fail with -EFAULT if an invalid
 *	pointer is passed for ctxp.  Will fail with -ENOSYS if not
 *	implemented.
 */
1287
SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
L
Linus Torvalds 已提交
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
{
	struct kioctx *ioctx = NULL;
	unsigned long ctx;
	long ret;

	ret = get_user(ctx, ctxp);
	if (unlikely(ret))
		goto out;

	ret = -EINVAL;
1298
	if (unlikely(ctx || nr_events == 0)) {
1299
		pr_debug("EINVAL: ctx %lu nr_events %u\n",
1300
		         ctx, nr_events);
L
Linus Torvalds 已提交
1301 1302 1303 1304 1305 1306 1307
		goto out;
	}

	ioctx = ioctx_alloc(nr_events);
	ret = PTR_ERR(ioctx);
	if (!IS_ERR(ioctx)) {
		ret = put_user(ioctx->user_id, ctxp);
1308
		if (ret)
1309
			kill_ioctx(current->mm, ioctx, NULL);
K
Kent Overstreet 已提交
1310
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1311 1312 1313 1314 1315 1316 1317 1318 1319
	}

out:
	return ret;
}

/* sys_io_destroy:
 *	Destroy the aio_context specified.  May cancel any outstanding 
 *	AIOs and block on completion.  Will fail with -ENOSYS if not
1320
 *	implemented.  May fail with -EINVAL if the context pointed to
L
Linus Torvalds 已提交
1321 1322
 *	is invalid.
 */
1323
SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
L
Linus Torvalds 已提交
1324 1325 1326
{
	struct kioctx *ioctx = lookup_ioctx(ctx);
	if (likely(NULL != ioctx)) {
1327 1328
		struct completion requests_done =
			COMPLETION_INITIALIZER_ONSTACK(requests_done);
1329
		int ret;
1330 1331 1332 1333 1334

		/* Pass requests_done to kill_ioctx() where it can be set
		 * in a thread-safe way. If we try to set it here then we have
		 * a race condition if two io_destroy() called simultaneously.
		 */
1335
		ret = kill_ioctx(current->mm, ioctx, &requests_done);
K
Kent Overstreet 已提交
1336
		percpu_ref_put(&ioctx->users);
1337 1338 1339 1340 1341

		/* Wait until all IO for the context are done. Otherwise kernel
		 * keep using user-space buffers even if user thinks the context
		 * is destroyed.
		 */
1342 1343
		if (!ret)
			wait_for_completion(&requests_done);
1344

1345
		return ret;
L
Linus Torvalds 已提交
1346
	}
1347
	pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1348 1349 1350
	return -EINVAL;
}

K
Kent Overstreet 已提交
1351 1352
typedef ssize_t (aio_rw_op)(struct kiocb *, const struct iovec *,
			    unsigned long, loff_t);
1353
typedef ssize_t (rw_iter_op)(struct kiocb *, struct iov_iter *);
K
Kent Overstreet 已提交
1354

1355 1356 1357
static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
				     int rw, char __user *buf,
				     unsigned long *nr_segs,
C
Christoph Hellwig 已提交
1358
				     size_t *len,
1359
				     struct iovec **iovec,
1360 1361
				     bool compat,
				     struct iov_iter *iter)
B
Badari Pulavarty 已提交
1362 1363 1364
{
	ssize_t ret;

C
Christoph Hellwig 已提交
1365
	*nr_segs = *len;
K
Kent Overstreet 已提交
1366

1367 1368
#ifdef CONFIG_COMPAT
	if (compat)
K
Kent Overstreet 已提交
1369
		ret = compat_rw_copy_check_uvector(rw,
1370
				(struct compat_iovec __user *)buf,
1371
				*nr_segs, UIO_FASTIOV, *iovec, iovec);
1372 1373
	else
#endif
K
Kent Overstreet 已提交
1374
		ret = rw_copy_check_uvector(rw,
1375
				(struct iovec __user *)buf,
1376
				*nr_segs, UIO_FASTIOV, *iovec, iovec);
B
Badari Pulavarty 已提交
1377
	if (ret < 0)
K
Kent Overstreet 已提交
1378
		return ret;
1379

C
Christoph Hellwig 已提交
1380 1381
	/* len now reflect bytes instead of segs */
	*len = ret;
1382
	iov_iter_init(iter, rw, *iovec, *nr_segs, *len);
K
Kent Overstreet 已提交
1383
	return 0;
B
Badari Pulavarty 已提交
1384 1385
}

1386 1387 1388
static ssize_t aio_setup_single_vector(struct kiocb *kiocb,
				       int rw, char __user *buf,
				       unsigned long *nr_segs,
C
Christoph Hellwig 已提交
1389
				       size_t len,
1390 1391
				       struct iovec *iovec,
				       struct iov_iter *iter)
B
Badari Pulavarty 已提交
1392
{
1393 1394
	if (len > MAX_RW_COUNT)
		len = MAX_RW_COUNT;
C
Christoph Hellwig 已提交
1395
	if (unlikely(!access_ok(!rw, buf, len)))
K
Kent Overstreet 已提交
1396
		return -EFAULT;
1397

1398
	iovec->iov_base = buf;
C
Christoph Hellwig 已提交
1399
	iovec->iov_len = len;
1400
	*nr_segs = 1;
1401
	iov_iter_init(iter, rw, iovec, *nr_segs, len);
B
Badari Pulavarty 已提交
1402 1403 1404
	return 0;
}

L
Linus Torvalds 已提交
1405
/*
G
Gu Zheng 已提交
1406 1407
 * aio_run_iocb:
 *	Performs the initial checks and io submission.
L
Linus Torvalds 已提交
1408
 */
1409
static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
C
Christoph Hellwig 已提交
1410
			    char __user *buf, size_t len, bool compat)
L
Linus Torvalds 已提交
1411
{
K
Kent Overstreet 已提交
1412 1413
	struct file *file = req->ki_filp;
	ssize_t ret;
1414
	unsigned long nr_segs;
K
Kent Overstreet 已提交
1415 1416 1417
	int rw;
	fmode_t mode;
	aio_rw_op *rw_op;
1418
	rw_iter_op *iter_op;
1419
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1420
	struct iov_iter iter;
L
Linus Torvalds 已提交
1421

1422
	switch (opcode) {
L
Linus Torvalds 已提交
1423
	case IOCB_CMD_PREAD:
B
Badari Pulavarty 已提交
1424
	case IOCB_CMD_PREADV:
K
Kent Overstreet 已提交
1425 1426 1427
		mode	= FMODE_READ;
		rw	= READ;
		rw_op	= file->f_op->aio_read;
1428
		iter_op	= file->f_op->read_iter;
K
Kent Overstreet 已提交
1429 1430 1431
		goto rw_common;

	case IOCB_CMD_PWRITE:
B
Badari Pulavarty 已提交
1432
	case IOCB_CMD_PWRITEV:
K
Kent Overstreet 已提交
1433 1434 1435
		mode	= FMODE_WRITE;
		rw	= WRITE;
		rw_op	= file->f_op->aio_write;
1436
		iter_op	= file->f_op->write_iter;
K
Kent Overstreet 已提交
1437 1438 1439 1440 1441
		goto rw_common;
rw_common:
		if (unlikely(!(file->f_mode & mode)))
			return -EBADF;

1442
		if (!rw_op && !iter_op)
K
Kent Overstreet 已提交
1443 1444
			return -EINVAL;

C
Christoph Hellwig 已提交
1445 1446
		if (opcode == IOCB_CMD_PREADV || opcode == IOCB_CMD_PWRITEV)
			ret = aio_setup_vectored_rw(req, rw, buf, &nr_segs,
1447
						&len, &iovec, compat, &iter);
C
Christoph Hellwig 已提交
1448 1449
		else
			ret = aio_setup_single_vector(req, rw, buf, &nr_segs,
1450
						  len, iovec, &iter);
1451
		if (!ret)
C
Christoph Hellwig 已提交
1452
			ret = rw_verify_area(rw, file, &req->ki_pos, len);
1453
		if (ret < 0) {
1454
			if (iovec != inline_vecs)
1455
				kfree(iovec);
K
Kent Overstreet 已提交
1456
			return ret;
1457
		}
K
Kent Overstreet 已提交
1458

C
Christoph Hellwig 已提交
1459
		len = ret;
K
Kent Overstreet 已提交
1460

K
Kent Overstreet 已提交
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
		/* XXX: move/kill - rw_verify_area()? */
		/* This matches the pread()/pwrite() logic */
		if (req->ki_pos < 0) {
			ret = -EINVAL;
			break;
		}

		if (rw == WRITE)
			file_start_write(file);

1471 1472 1473
		if (iter_op) {
			ret = iter_op(req, &iter);
		} else {
1474
			ret = rw_op(req, iter.iov, iter.nr_segs, req->ki_pos);
1475
		}
K
Kent Overstreet 已提交
1476 1477 1478

		if (rw == WRITE)
			file_end_write(file);
L
Linus Torvalds 已提交
1479
		break;
K
Kent Overstreet 已提交
1480

L
Linus Torvalds 已提交
1481
	case IOCB_CMD_FDSYNC:
K
Kent Overstreet 已提交
1482 1483 1484 1485
		if (!file->f_op->aio_fsync)
			return -EINVAL;

		ret = file->f_op->aio_fsync(req, 1);
L
Linus Torvalds 已提交
1486
		break;
K
Kent Overstreet 已提交
1487

L
Linus Torvalds 已提交
1488
	case IOCB_CMD_FSYNC:
K
Kent Overstreet 已提交
1489 1490 1491 1492
		if (!file->f_op->aio_fsync)
			return -EINVAL;

		ret = file->f_op->aio_fsync(req, 0);
L
Linus Torvalds 已提交
1493
		break;
K
Kent Overstreet 已提交
1494

L
Linus Torvalds 已提交
1495
	default:
K
Kent Overstreet 已提交
1496
		pr_debug("EINVAL: no operation provided\n");
K
Kent Overstreet 已提交
1497
		return -EINVAL;
L
Linus Torvalds 已提交
1498 1499
	}

1500
	if (iovec != inline_vecs)
1501 1502
		kfree(iovec);

K
Kent Overstreet 已提交
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
	if (ret != -EIOCBQUEUED) {
		/*
		 * There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		if (unlikely(ret == -ERESTARTSYS || ret == -ERESTARTNOINTR ||
			     ret == -ERESTARTNOHAND ||
			     ret == -ERESTART_RESTARTBLOCK))
			ret = -EINTR;
		aio_complete(req, ret, 0);
	}
L
Linus Torvalds 已提交
1514 1515 1516 1517

	return 0;
}

A
Adrian Bunk 已提交
1518
static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
K
Kent Overstreet 已提交
1519
			 struct iocb *iocb, bool compat)
L
Linus Torvalds 已提交
1520
{
1521
	struct aio_kiocb *req;
L
Linus Torvalds 已提交
1522 1523 1524
	ssize_t ret;

	/* enforce forwards compatibility on users */
1525
	if (unlikely(iocb->aio_reserved1 || iocb->aio_reserved2)) {
K
Kent Overstreet 已提交
1526
		pr_debug("EINVAL: reserve field set\n");
L
Linus Torvalds 已提交
1527 1528 1529 1530 1531 1532 1533 1534 1535
		return -EINVAL;
	}

	/* prevent overflows */
	if (unlikely(
	    (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
	    (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
	    ((ssize_t)iocb->aio_nbytes < 0)
	   )) {
1536
		pr_debug("EINVAL: overflow check\n");
L
Linus Torvalds 已提交
1537 1538 1539
		return -EINVAL;
	}

K
Kent Overstreet 已提交
1540
	req = aio_get_req(ctx);
1541
	if (unlikely(!req))
L
Linus Torvalds 已提交
1542
		return -EAGAIN;
1543

1544 1545
	req->common.ki_filp = fget(iocb->aio_fildes);
	if (unlikely(!req->common.ki_filp)) {
1546 1547
		ret = -EBADF;
		goto out_put_req;
L
Linus Torvalds 已提交
1548
	}
1549 1550 1551
	req->common.ki_pos = iocb->aio_offset;
	req->common.ki_complete = aio_complete;
	req->common.ki_flags = 0;
1552

1553 1554 1555 1556 1557 1558 1559
	if (iocb->aio_flags & IOCB_FLAG_RESFD) {
		/*
		 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
		 * instance of the file* now. The file descriptor must be
		 * an eventfd() fd, and will be signaled for each completed
		 * event using the eventfd_signal() function.
		 */
1560
		req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd);
1561
		if (IS_ERR(req->ki_eventfd)) {
1562
			ret = PTR_ERR(req->ki_eventfd);
1563
			req->ki_eventfd = NULL;
1564 1565
			goto out_put_req;
		}
1566 1567

		req->common.ki_flags |= IOCB_EVENTFD;
1568
	}
L
Linus Torvalds 已提交
1569

K
Kent Overstreet 已提交
1570
	ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
L
Linus Torvalds 已提交
1571
	if (unlikely(ret)) {
K
Kent Overstreet 已提交
1572
		pr_debug("EFAULT: aio_key\n");
L
Linus Torvalds 已提交
1573 1574 1575
		goto out_put_req;
	}

1576
	req->ki_user_iocb = user_iocb;
L
Linus Torvalds 已提交
1577 1578
	req->ki_user_data = iocb->aio_data;

1579
	ret = aio_run_iocb(&req->common, iocb->aio_lio_opcode,
1580
			   (char __user *)(unsigned long)iocb->aio_buf,
C
Christoph Hellwig 已提交
1581
			   iocb->aio_nbytes,
1582
			   compat);
Z
Zach Brown 已提交
1583
	if (ret)
1584
		goto out_put_req;
Z
Zach Brown 已提交
1585

L
Linus Torvalds 已提交
1586 1587
	return 0;
out_put_req:
K
Kent Overstreet 已提交
1588
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1589
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
1590
	kiocb_free(req);
L
Linus Torvalds 已提交
1591 1592 1593
	return ret;
}

1594 1595
long do_io_submit(aio_context_t ctx_id, long nr,
		  struct iocb __user *__user *iocbpp, bool compat)
L
Linus Torvalds 已提交
1596 1597 1598
{
	struct kioctx *ctx;
	long ret = 0;
J
Jeff Moyer 已提交
1599
	int i = 0;
S
Shaohua Li 已提交
1600
	struct blk_plug plug;
L
Linus Torvalds 已提交
1601 1602 1603 1604

	if (unlikely(nr < 0))
		return -EINVAL;

1605 1606 1607
	if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
		nr = LONG_MAX/sizeof(*iocbpp);

L
Linus Torvalds 已提交
1608 1609 1610 1611 1612
	if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx)) {
K
Kent Overstreet 已提交
1613
		pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1614 1615 1616
		return -EINVAL;
	}

S
Shaohua Li 已提交
1617 1618
	blk_start_plug(&plug);

L
Linus Torvalds 已提交
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
	/*
	 * AKPM: should this return a partial result if some of the IOs were
	 * successfully submitted?
	 */
	for (i=0; i<nr; i++) {
		struct iocb __user *user_iocb;
		struct iocb tmp;

		if (unlikely(__get_user(user_iocb, iocbpp + i))) {
			ret = -EFAULT;
			break;
		}

		if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
			ret = -EFAULT;
			break;
		}

K
Kent Overstreet 已提交
1637
		ret = io_submit_one(ctx, user_iocb, &tmp, compat);
L
Linus Torvalds 已提交
1638 1639 1640
		if (ret)
			break;
	}
S
Shaohua Li 已提交
1641
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
1642

K
Kent Overstreet 已提交
1643
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1644 1645 1646
	return i ? i : ret;
}

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
/* sys_io_submit:
 *	Queue the nr iocbs pointed to by iocbpp for processing.  Returns
 *	the number of iocbs queued.  May return -EINVAL if the aio_context
 *	specified by ctx_id is invalid, if nr is < 0, if the iocb at
 *	*iocbpp[0] is not properly initialized, if the operation specified
 *	is invalid for the file descriptor in the iocb.  May fail with
 *	-EFAULT if any of the data structures point to invalid data.  May
 *	fail with -EBADF if the file descriptor specified in the first
 *	iocb is invalid.  May fail with -EAGAIN if insufficient resources
 *	are available to queue any iocbs.  Will return 0 if nr is 0.  Will
 *	fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
		struct iocb __user * __user *, iocbpp)
{
	return do_io_submit(ctx_id, nr, iocbpp, 0);
}

L
Linus Torvalds 已提交
1665 1666 1667
/* lookup_kiocb
 *	Finds a given iocb for cancellation.
 */
1668 1669
static struct aio_kiocb *
lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb, u32 key)
L
Linus Torvalds 已提交
1670
{
1671
	struct aio_kiocb *kiocb;
1672 1673 1674

	assert_spin_locked(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1675 1676 1677
	if (key != KIOCB_KEY)
		return NULL;

L
Linus Torvalds 已提交
1678
	/* TODO: use a hash or array, this sucks. */
1679 1680
	list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
		if (kiocb->ki_user_iocb == iocb)
L
Linus Torvalds 已提交
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
			return kiocb;
	}
	return NULL;
}

/* sys_io_cancel:
 *	Attempts to cancel an iocb previously passed to io_submit.  If
 *	the operation is successfully cancelled, the resulting event is
 *	copied into the memory pointed to by result without being placed
 *	into the completion queue and 0 is returned.  May fail with
 *	-EFAULT if any of the data structures pointed to are invalid.
 *	May fail with -EINVAL if aio_context specified by ctx_id is
 *	invalid.  May fail with -EAGAIN if the iocb specified was not
 *	cancelled.  Will fail with -ENOSYS if not implemented.
 */
1696 1697
SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
		struct io_event __user *, result)
L
Linus Torvalds 已提交
1698 1699
{
	struct kioctx *ctx;
1700
	struct aio_kiocb *kiocb;
L
Linus Torvalds 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
	u32 key;
	int ret;

	ret = get_user(key, &iocb->aio_key);
	if (unlikely(ret))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx))
		return -EINVAL;

	spin_lock_irq(&ctx->ctx_lock);
K
Kent Overstreet 已提交
1713

L
Linus Torvalds 已提交
1714
	kiocb = lookup_kiocb(ctx, iocb, key);
K
Kent Overstreet 已提交
1715
	if (kiocb)
1716
		ret = kiocb_cancel(kiocb);
K
Kent Overstreet 已提交
1717 1718 1719
	else
		ret = -EINVAL;

L
Linus Torvalds 已提交
1720 1721
	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1722
	if (!ret) {
1723 1724 1725 1726
		/*
		 * The result argument is no longer used - the io_event is
		 * always delivered via the ring buffer. -EINPROGRESS indicates
		 * cancellation is progress:
K
Kent Overstreet 已提交
1727
		 */
1728
		ret = -EINPROGRESS;
K
Kent Overstreet 已提交
1729
	}
L
Linus Torvalds 已提交
1730

K
Kent Overstreet 已提交
1731
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1732 1733 1734 1735 1736 1737

	return ret;
}

/* io_getevents:
 *	Attempts to read at least min_nr events and up to nr events from
1738 1739 1740 1741 1742 1743 1744 1745
 *	the completion queue for the aio_context specified by ctx_id. If
 *	it succeeds, the number of read events is returned. May fail with
 *	-EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
 *	out of range, if timeout is out of range.  May fail with -EFAULT
 *	if any of the memory specified is invalid.  May return 0 or
 *	< min_nr if the timeout specified by timeout has elapsed
 *	before sufficient events are available, where timeout == NULL
 *	specifies an infinite timeout. Note that the timeout pointed to by
J
Jeff Moyer 已提交
1746
 *	timeout is relative.  Will fail with -ENOSYS if not implemented.
L
Linus Torvalds 已提交
1747
 */
1748 1749 1750 1751 1752
SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
		long, min_nr,
		long, nr,
		struct io_event __user *, events,
		struct timespec __user *, timeout)
L
Linus Torvalds 已提交
1753 1754 1755 1756 1757
{
	struct kioctx *ioctx = lookup_ioctx(ctx_id);
	long ret = -EINVAL;

	if (likely(ioctx)) {
N
Namhyung Kim 已提交
1758
		if (likely(min_nr <= nr && min_nr >= 0))
L
Linus Torvalds 已提交
1759
			ret = read_events(ioctx, min_nr, nr, events, timeout);
K
Kent Overstreet 已提交
1760
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1761 1762 1763
	}
	return ret;
}