aio.c 43.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 *	An async IO implementation for Linux
 *	Written by Benjamin LaHaise <bcrl@kvack.org>
 *
 *	Implements an efficient asynchronous io interface.
 *
 *	Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved.
 *
 *	See ../COPYING for licensing terms.
 */
K
Kent Overstreet 已提交
11 12
#define pr_fmt(fmt) "%s: " fmt, __func__

L
Linus Torvalds 已提交
13 14 15 16 17
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/aio_abi.h>
18
#include <linux/export.h>
L
Linus Torvalds 已提交
19
#include <linux/syscalls.h>
20
#include <linux/backing-dev.h>
21
#include <linux/uio.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
28
#include <linux/mmu_context.h>
K
Kent Overstreet 已提交
29
#include <linux/percpu.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/aio.h>
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
36
#include <linux/eventfd.h>
J
Jeff Moyer 已提交
37
#include <linux/blkdev.h>
38
#include <linux/compat.h>
39 40
#include <linux/migrate.h>
#include <linux/ramfs.h>
K
Kent Overstreet 已提交
41
#include <linux/percpu-refcount.h>
42
#include <linux/mount.h>
L
Linus Torvalds 已提交
43 44 45 46

#include <asm/kmap_types.h>
#include <asm/uaccess.h>

A
Al Viro 已提交
47 48
#include "internal.h"

49 50 51 52 53 54
#define AIO_RING_MAGIC			0xa10a10a1
#define AIO_RING_COMPAT_FEATURES	1
#define AIO_RING_INCOMPAT_FEATURES	0
struct aio_ring {
	unsigned	id;	/* kernel internal index number */
	unsigned	nr;	/* number of io_events */
55 56
	unsigned	head;	/* Written to by userland or under ring_lock
				 * mutex by aio_read_events_ring(). */
57 58 59 60 61 62 63 64 65 66 67 68 69
	unsigned	tail;

	unsigned	magic;
	unsigned	compat_features;
	unsigned	incompat_features;
	unsigned	header_length;	/* size of aio_ring */


	struct io_event		io_events[0];
}; /* 128 bytes + ring size */

#define AIO_RING_PAGES	8

70 71 72 73 74 75
struct kioctx_table {
	struct rcu_head	rcu;
	unsigned	nr;
	struct kioctx	*table[];
};

K
Kent Overstreet 已提交
76 77 78 79
struct kioctx_cpu {
	unsigned		reqs_available;
};

80 81 82 83 84
struct ctx_rq_wait {
	struct completion comp;
	atomic_t count;
};

85
struct kioctx {
K
Kent Overstreet 已提交
86
	struct percpu_ref	users;
K
Kent Overstreet 已提交
87
	atomic_t		dead;
88

K
Kent Overstreet 已提交
89 90
	struct percpu_ref	reqs;

91 92
	unsigned long		user_id;

K
Kent Overstreet 已提交
93 94 95 96 97 98 99
	struct __percpu kioctx_cpu *cpu;

	/*
	 * For percpu reqs_available, number of slots we move to/from global
	 * counter at a time:
	 */
	unsigned		req_batch;
100 101 102 103
	/*
	 * This is what userspace passed to io_setup(), it's not used for
	 * anything but counting against the global max_reqs quota.
	 *
K
Kent Overstreet 已提交
104
	 * The real limit is nr_events - 1, which will be larger (see
105 106
	 * aio_setup_ring())
	 */
107 108
	unsigned		max_reqs;

K
Kent Overstreet 已提交
109 110
	/* Size of ringbuffer, in units of struct io_event */
	unsigned		nr_events;
111

K
Kent Overstreet 已提交
112 113 114 115 116 117
	unsigned long		mmap_base;
	unsigned long		mmap_size;

	struct page		**ring_pages;
	long			nr_pages;

K
Kent Overstreet 已提交
118
	struct work_struct	free_work;
119

120 121 122
	/*
	 * signals when all in-flight requests are done
	 */
123
	struct ctx_rq_wait	*rq_wait;
124

125
	struct {
126 127 128 129 130
		/*
		 * This counts the number of available slots in the ringbuffer,
		 * so we avoid overflowing it: it's decremented (if positive)
		 * when allocating a kiocb and incremented when the resulting
		 * io_event is pulled off the ringbuffer.
K
Kent Overstreet 已提交
131 132
		 *
		 * We batch accesses to it with a percpu version.
133 134
		 */
		atomic_t	reqs_available;
135 136 137 138 139 140 141
	} ____cacheline_aligned_in_smp;

	struct {
		spinlock_t	ctx_lock;
		struct list_head active_reqs;	/* used for cancellation */
	} ____cacheline_aligned_in_smp;

K
Kent Overstreet 已提交
142 143
	struct {
		struct mutex	ring_lock;
144 145
		wait_queue_head_t wait;
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
146 147 148

	struct {
		unsigned	tail;
149
		unsigned	completed_events;
K
Kent Overstreet 已提交
150
		spinlock_t	completion_lock;
151
	} ____cacheline_aligned_in_smp;
K
Kent Overstreet 已提交
152 153

	struct page		*internal_pages[AIO_RING_PAGES];
154
	struct file		*aio_ring_file;
155 156

	unsigned		id;
157 158
};

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
/*
 * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
 * cancelled or completed (this makes a certain amount of sense because
 * successful cancellation - io_cancel() - does deliver the completion to
 * userspace).
 *
 * And since most things don't implement kiocb cancellation and we'd really like
 * kiocb completion to be lockless when possible, we use ki_cancel to
 * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
 * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
 */
#define KIOCB_CANCELLED		((void *) (~0ULL))

struct aio_kiocb {
	struct kiocb		common;

	struct kioctx		*ki_ctx;
	kiocb_cancel_fn		*ki_cancel;

	struct iocb __user	*ki_user_iocb;	/* user's aiocb */
	__u64			ki_user_data;	/* user's data for completion */

	struct list_head	ki_list;	/* the aio core uses this
						 * for cancellation */

	/*
	 * If the aio_resfd field of the userspace iocb is not zero,
	 * this is the underlying eventfd context to deliver events to.
	 */
	struct eventfd_ctx	*ki_eventfd;
};

L
Linus Torvalds 已提交
191
/*------ sysctl variables----*/
192 193 194
static DEFINE_SPINLOCK(aio_nr_lock);
unsigned long aio_nr;		/* current system wide number of aio requests */
unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
L
Linus Torvalds 已提交
195 196
/*----end sysctl variables---*/

197 198
static struct kmem_cache	*kiocb_cachep;
static struct kmem_cache	*kioctx_cachep;
L
Linus Torvalds 已提交
199

200 201 202 203 204 205 206 207 208 209 210
static struct vfsmount *aio_mnt;

static const struct file_operations aio_ring_fops;
static const struct address_space_operations aio_ctx_aops;

static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
{
	struct qstr this = QSTR_INIT("[aio]", 5);
	struct file *file;
	struct path path;
	struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
211 212
	if (IS_ERR(inode))
		return ERR_CAST(inode);
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241

	inode->i_mapping->a_ops = &aio_ctx_aops;
	inode->i_mapping->private_data = ctx;
	inode->i_size = PAGE_SIZE * nr_pages;

	path.dentry = d_alloc_pseudo(aio_mnt->mnt_sb, &this);
	if (!path.dentry) {
		iput(inode);
		return ERR_PTR(-ENOMEM);
	}
	path.mnt = mntget(aio_mnt);

	d_instantiate(path.dentry, inode);
	file = alloc_file(&path, FMODE_READ | FMODE_WRITE, &aio_ring_fops);
	if (IS_ERR(file)) {
		path_put(&path);
		return file;
	}

	file->f_flags = O_RDWR;
	return file;
}

static struct dentry *aio_mount(struct file_system_type *fs_type,
				int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname	= simple_dname,
	};
J
Jann Horn 已提交
242 243 244 245 246 247
	struct dentry *root = mount_pseudo(fs_type, "aio:", NULL, &ops,
					   AIO_RING_MAGIC);

	if (!IS_ERR(root))
		root->d_sb->s_iflags |= SB_I_NOEXEC;
	return root;
248 249
}

L
Linus Torvalds 已提交
250 251 252 253 254 255
/* aio_setup
 *	Creates the slab caches used by the aio routines, panic on
 *	failure as this is done early during the boot sequence.
 */
static int __init aio_setup(void)
{
256 257 258 259 260 261 262 263 264
	static struct file_system_type aio_fs = {
		.name		= "aio",
		.mount		= aio_mount,
		.kill_sb	= kill_anon_super,
	};
	aio_mnt = kern_mount(&aio_fs);
	if (IS_ERR(aio_mnt))
		panic("Failed to create aio fs mount.");

265
	kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
266
	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
L
Linus Torvalds 已提交
267

K
Kent Overstreet 已提交
268
	pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
L
Linus Torvalds 已提交
269 270 271

	return 0;
}
272
__initcall(aio_setup);
L
Linus Torvalds 已提交
273

274 275 276
static void put_aio_ring_file(struct kioctx *ctx)
{
	struct file *aio_ring_file = ctx->aio_ring_file;
277 278
	struct address_space *i_mapping;

279 280 281 282
	if (aio_ring_file) {
		truncate_setsize(aio_ring_file->f_inode, 0);

		/* Prevent further access to the kioctx from migratepages */
283 284 285
		i_mapping = aio_ring_file->f_inode->i_mapping;
		spin_lock(&i_mapping->private_lock);
		i_mapping->private_data = NULL;
286
		ctx->aio_ring_file = NULL;
287
		spin_unlock(&i_mapping->private_lock);
288 289 290 291 292

		fput(aio_ring_file);
	}
}

L
Linus Torvalds 已提交
293 294
static void aio_free_ring(struct kioctx *ctx)
{
295
	int i;
L
Linus Torvalds 已提交
296

297 298 299 300 301
	/* Disconnect the kiotx from the ring file.  This prevents future
	 * accesses to the kioctx from page migration.
	 */
	put_aio_ring_file(ctx);

302
	for (i = 0; i < ctx->nr_pages; i++) {
303
		struct page *page;
304 305
		pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
				page_count(ctx->ring_pages[i]));
306 307 308 309 310
		page = ctx->ring_pages[i];
		if (!page)
			continue;
		ctx->ring_pages[i] = NULL;
		put_page(page);
311
	}
L
Linus Torvalds 已提交
312

313
	if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
K
Kent Overstreet 已提交
314
		kfree(ctx->ring_pages);
315 316
		ctx->ring_pages = NULL;
	}
317 318
}

319
static int aio_ring_mremap(struct vm_area_struct *vma)
320
{
321
	struct file *file = vma->vm_file;
322 323
	struct mm_struct *mm = vma->vm_mm;
	struct kioctx_table *table;
A
Al Viro 已提交
324
	int i, res = -EINVAL;
325 326 327 328 329 330 331 332 333

	spin_lock(&mm->ioctx_lock);
	rcu_read_lock();
	table = rcu_dereference(mm->ioctx_table);
	for (i = 0; i < table->nr; i++) {
		struct kioctx *ctx;

		ctx = table->table[i];
		if (ctx && ctx->aio_ring_file == file) {
A
Al Viro 已提交
334 335 336 337
			if (!atomic_read(&ctx->dead)) {
				ctx->user_id = ctx->mmap_base = vma->vm_start;
				res = 0;
			}
338 339 340 341 342 343
			break;
		}
	}

	rcu_read_unlock();
	spin_unlock(&mm->ioctx_lock);
A
Al Viro 已提交
344
	return res;
345 346
}

347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
static const struct vm_operations_struct aio_ring_vm_ops = {
	.mremap		= aio_ring_mremap,
#if IS_ENABLED(CONFIG_MMU)
	.fault		= filemap_fault,
	.map_pages	= filemap_map_pages,
	.page_mkwrite	= filemap_page_mkwrite,
#endif
};

static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_flags |= VM_DONTEXPAND;
	vma->vm_ops = &aio_ring_vm_ops;
	return 0;
}

363 364 365 366
static const struct file_operations aio_ring_fops = {
	.mmap = aio_ring_mmap,
};

367
#if IS_ENABLED(CONFIG_MIGRATION)
368 369 370
static int aio_migratepage(struct address_space *mapping, struct page *new,
			struct page *old, enum migrate_mode mode)
{
371
	struct kioctx *ctx;
372
	unsigned long flags;
373
	pgoff_t idx;
374 375
	int rc;

376 377
	rc = 0;

378
	/* mapping->private_lock here protects against the kioctx teardown.  */
379 380
	spin_lock(&mapping->private_lock);
	ctx = mapping->private_data;
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
	if (!ctx) {
		rc = -EINVAL;
		goto out;
	}

	/* The ring_lock mutex.  The prevents aio_read_events() from writing
	 * to the ring's head, and prevents page migration from mucking in
	 * a partially initialized kiotx.
	 */
	if (!mutex_trylock(&ctx->ring_lock)) {
		rc = -EAGAIN;
		goto out;
	}

	idx = old->index;
	if (idx < (pgoff_t)ctx->nr_pages) {
		/* Make sure the old page hasn't already been changed */
		if (ctx->ring_pages[idx] != old)
			rc = -EAGAIN;
400 401 402 403
	} else
		rc = -EINVAL;

	if (rc != 0)
404
		goto out_unlock;
405

406 407
	/* Writeback must be complete */
	BUG_ON(PageWriteback(old));
408
	get_page(new);
409

410
	rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1);
411
	if (rc != MIGRATEPAGE_SUCCESS) {
412
		put_page(new);
413
		goto out_unlock;
414 415
	}

416 417 418
	/* Take completion_lock to prevent other writes to the ring buffer
	 * while the old page is copied to the new.  This prevents new
	 * events from being lost.
419
	 */
420 421 422 423 424
	spin_lock_irqsave(&ctx->completion_lock, flags);
	migrate_page_copy(new, old);
	BUG_ON(ctx->ring_pages[idx] != old);
	ctx->ring_pages[idx] = new;
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
425

426 427
	/* The old page is no longer accessible. */
	put_page(old);
428

429 430 431 432
out_unlock:
	mutex_unlock(&ctx->ring_lock);
out:
	spin_unlock(&mapping->private_lock);
433
	return rc;
L
Linus Torvalds 已提交
434
}
435
#endif
L
Linus Torvalds 已提交
436

437
static const struct address_space_operations aio_ctx_aops = {
438
	.set_page_dirty = __set_page_dirty_no_writeback,
439
#if IS_ENABLED(CONFIG_MIGRATION)
440
	.migratepage	= aio_migratepage,
441
#endif
442 443
};

L
Linus Torvalds 已提交
444 445 446 447
static int aio_setup_ring(struct kioctx *ctx)
{
	struct aio_ring *ring;
	unsigned nr_events = ctx->max_reqs;
Z
Zach Brown 已提交
448
	struct mm_struct *mm = current->mm;
449
	unsigned long size, unused;
L
Linus Torvalds 已提交
450
	int nr_pages;
451 452
	int i;
	struct file *file;
L
Linus Torvalds 已提交
453 454 455 456 457 458 459

	/* Compensate for the ring buffer's head/tail overlap entry */
	nr_events += 2;	/* 1 is required, 2 for good luck */

	size = sizeof(struct aio_ring);
	size += sizeof(struct io_event) * nr_events;

460
	nr_pages = PFN_UP(size);
L
Linus Torvalds 已提交
461 462 463
	if (nr_pages < 0)
		return -EINVAL;

464
	file = aio_private_file(ctx, nr_pages);
465 466
	if (IS_ERR(file)) {
		ctx->aio_ring_file = NULL;
467
		return -ENOMEM;
468 469
	}

470 471 472 473 474 475 476 477 478 479 480 481 482 483
	ctx->aio_ring_file = file;
	nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
			/ sizeof(struct io_event);

	ctx->ring_pages = ctx->internal_pages;
	if (nr_pages > AIO_RING_PAGES) {
		ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
					  GFP_KERNEL);
		if (!ctx->ring_pages) {
			put_aio_ring_file(ctx);
			return -ENOMEM;
		}
	}

484 485 486 487 488 489 490 491 492 493
	for (i = 0; i < nr_pages; i++) {
		struct page *page;
		page = find_or_create_page(file->f_inode->i_mapping,
					   i, GFP_HIGHUSER | __GFP_ZERO);
		if (!page)
			break;
		pr_debug("pid(%d) page[%d]->count=%d\n",
			 current->pid, i, page_count(page));
		SetPageUptodate(page);
		unlock_page(page);
494 495

		ctx->ring_pages[i] = page;
496
	}
497
	ctx->nr_pages = i;
L
Linus Torvalds 已提交
498

499 500
	if (unlikely(i != nr_pages)) {
		aio_free_ring(ctx);
501
		return -ENOMEM;
L
Linus Torvalds 已提交
502 503
	}

K
Kent Overstreet 已提交
504 505
	ctx->mmap_size = nr_pages * PAGE_SIZE;
	pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
506

M
Michal Hocko 已提交
507 508 509 510 511 512
	if (down_write_killable(&mm->mmap_sem)) {
		ctx->mmap_size = 0;
		aio_free_ring(ctx);
		return -EINTR;
	}

513 514
	ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
				       PROT_READ | PROT_WRITE,
515 516
				       MAP_SHARED, 0, &unused);
	up_write(&mm->mmap_sem);
K
Kent Overstreet 已提交
517 518
	if (IS_ERR((void *)ctx->mmap_base)) {
		ctx->mmap_size = 0;
L
Linus Torvalds 已提交
519
		aio_free_ring(ctx);
520
		return -ENOMEM;
L
Linus Torvalds 已提交
521 522
	}

K
Kent Overstreet 已提交
523
	pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
524

K
Kent Overstreet 已提交
525 526
	ctx->user_id = ctx->mmap_base;
	ctx->nr_events = nr_events; /* trusted copy */
L
Linus Torvalds 已提交
527

K
Kent Overstreet 已提交
528
	ring = kmap_atomic(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
529
	ring->nr = nr_events;	/* user copy */
530
	ring->id = ~0U;
L
Linus Torvalds 已提交
531 532 533 534 535
	ring->head = ring->tail = 0;
	ring->magic = AIO_RING_MAGIC;
	ring->compat_features = AIO_RING_COMPAT_FEATURES;
	ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
	ring->header_length = sizeof(struct aio_ring);
536
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
537
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
538 539 540 541 542 543 544 545

	return 0;
}

#define AIO_EVENTS_PER_PAGE	(PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE	((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET	(AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)

546
void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
547
{
548
	struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common);
549 550 551 552 553 554 555 556 557 558 559 560 561 562
	struct kioctx *ctx = req->ki_ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->ctx_lock, flags);

	if (!req->ki_list.next)
		list_add(&req->ki_list, &ctx->active_reqs);

	req->ki_cancel = cancel;

	spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
EXPORT_SYMBOL(kiocb_set_cancel_fn);

563
static int kiocb_cancel(struct aio_kiocb *kiocb)
K
Kent Overstreet 已提交
564
{
565
	kiocb_cancel_fn *old, *cancel;
K
Kent Overstreet 已提交
566

567 568 569 570 571 572 573 574
	/*
	 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
	 * actually has a cancel function, hence the cmpxchg()
	 */

	cancel = ACCESS_ONCE(kiocb->ki_cancel);
	do {
		if (!cancel || cancel == KIOCB_CANCELLED)
K
Kent Overstreet 已提交
575
			return -EINVAL;
K
Kent Overstreet 已提交
576

577 578 579
		old = cancel;
		cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
	} while (cancel != old);
K
Kent Overstreet 已提交
580

581
	return cancel(&kiocb->common);
K
Kent Overstreet 已提交
582 583
}

K
Kent Overstreet 已提交
584
static void free_ioctx(struct work_struct *work)
K
Kent Overstreet 已提交
585
{
K
Kent Overstreet 已提交
586
	struct kioctx *ctx = container_of(work, struct kioctx, free_work);
K
Kent Overstreet 已提交
587

K
Kent Overstreet 已提交
588
	pr_debug("freeing %p\n", ctx);
K
Kent Overstreet 已提交
589

K
Kent Overstreet 已提交
590
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
591
	free_percpu(ctx->cpu);
592 593
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
K
Kent Overstreet 已提交
594 595 596
	kmem_cache_free(kioctx_cachep, ctx);
}

K
Kent Overstreet 已提交
597 598 599 600
static void free_ioctx_reqs(struct percpu_ref *ref)
{
	struct kioctx *ctx = container_of(ref, struct kioctx, reqs);

601
	/* At this point we know that there are no any in-flight requests */
602 603
	if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
		complete(&ctx->rq_wait->comp);
604

K
Kent Overstreet 已提交
605 606 607 608
	INIT_WORK(&ctx->free_work, free_ioctx);
	schedule_work(&ctx->free_work);
}

K
Kent Overstreet 已提交
609 610 611 612 613
/*
 * When this function runs, the kioctx has been removed from the "hash table"
 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
 * now it's safe to cancel any that need to be.
 */
K
Kent Overstreet 已提交
614
static void free_ioctx_users(struct percpu_ref *ref)
K
Kent Overstreet 已提交
615
{
K
Kent Overstreet 已提交
616
	struct kioctx *ctx = container_of(ref, struct kioctx, users);
617
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
618 619 620 621 622

	spin_lock_irq(&ctx->ctx_lock);

	while (!list_empty(&ctx->active_reqs)) {
		req = list_first_entry(&ctx->active_reqs,
623
				       struct aio_kiocb, ki_list);
K
Kent Overstreet 已提交
624 625

		list_del_init(&req->ki_list);
626
		kiocb_cancel(req);
K
Kent Overstreet 已提交
627 628 629 630
	}

	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
631 632
	percpu_ref_kill(&ctx->reqs);
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
633 634
}

635 636 637 638 639 640 641
static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
{
	unsigned i, new_nr;
	struct kioctx_table *table, *old;
	struct aio_ring *ring;

	spin_lock(&mm->ioctx_lock);
642
	table = rcu_dereference_raw(mm->ioctx_table);
643 644 645 646 647 648 649 650 651

	while (1) {
		if (table)
			for (i = 0; i < table->nr; i++)
				if (!table->table[i]) {
					ctx->id = i;
					table->table[i] = ctx;
					spin_unlock(&mm->ioctx_lock);

652 653 654 655
					/* While kioctx setup is in progress,
					 * we are protected from page migration
					 * changes ring_pages by ->ring_lock.
					 */
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
					ring = kmap_atomic(ctx->ring_pages[0]);
					ring->id = ctx->id;
					kunmap_atomic(ring);
					return 0;
				}

		new_nr = (table ? table->nr : 1) * 4;
		spin_unlock(&mm->ioctx_lock);

		table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
				new_nr, GFP_KERNEL);
		if (!table)
			return -ENOMEM;

		table->nr = new_nr;

		spin_lock(&mm->ioctx_lock);
673
		old = rcu_dereference_raw(mm->ioctx_table);
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689

		if (!old) {
			rcu_assign_pointer(mm->ioctx_table, table);
		} else if (table->nr > old->nr) {
			memcpy(table->table, old->table,
			       old->nr * sizeof(struct kioctx *));

			rcu_assign_pointer(mm->ioctx_table, table);
			kfree_rcu(old, rcu);
		} else {
			kfree(table);
			table = old;
		}
	}
}

K
Kent Overstreet 已提交
690 691 692 693 694 695 696 697 698 699
static void aio_nr_sub(unsigned nr)
{
	spin_lock(&aio_nr_lock);
	if (WARN_ON(aio_nr - nr > aio_nr))
		aio_nr = 0;
	else
		aio_nr -= nr;
	spin_unlock(&aio_nr_lock);
}

L
Linus Torvalds 已提交
700 701 702 703 704
/* ioctx_alloc
 *	Allocates and initializes an ioctx.  Returns an ERR_PTR if it failed.
 */
static struct kioctx *ioctx_alloc(unsigned nr_events)
{
Z
Zach Brown 已提交
705
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
706
	struct kioctx *ctx;
707
	int err = -ENOMEM;
L
Linus Torvalds 已提交
708

K
Kent Overstreet 已提交
709 710 711 712 713 714 715 716 717 718 719 720
	/*
	 * We keep track of the number of available ringbuffer slots, to prevent
	 * overflow (reqs_available), and we also use percpu counters for this.
	 *
	 * So since up to half the slots might be on other cpu's percpu counters
	 * and unavailable, double nr_events so userspace sees what they
	 * expected: additionally, we move req_batch slots to/from percpu
	 * counters at a time, so make sure that isn't 0:
	 */
	nr_events = max(nr_events, num_possible_cpus() * 4);
	nr_events *= 2;

L
Linus Torvalds 已提交
721
	/* Prevent overflows */
A
Al Viro 已提交
722
	if (nr_events > (0x10000000U / sizeof(struct io_event))) {
L
Linus Torvalds 已提交
723 724 725 726
		pr_debug("ENOMEM: nr_events too high\n");
		return ERR_PTR(-EINVAL);
	}

727
	if (!nr_events || (unsigned long)nr_events > (aio_max_nr * 2UL))
L
Linus Torvalds 已提交
728 729
		return ERR_PTR(-EAGAIN);

730
	ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
731 732 733 734 735 736
	if (!ctx)
		return ERR_PTR(-ENOMEM);

	ctx->max_reqs = nr_events;

	spin_lock_init(&ctx->ctx_lock);
737
	spin_lock_init(&ctx->completion_lock);
K
Kent Overstreet 已提交
738
	mutex_init(&ctx->ring_lock);
739 740 741
	/* Protect against page migration throughout kiotx setup by keeping
	 * the ring_lock mutex held until setup is complete. */
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
742 743 744 745
	init_waitqueue_head(&ctx->wait);

	INIT_LIST_HEAD(&ctx->active_reqs);

746
	if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
747 748
		goto err;

749
	if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
750 751
		goto err;

K
Kent Overstreet 已提交
752 753
	ctx->cpu = alloc_percpu(struct kioctx_cpu);
	if (!ctx->cpu)
K
Kent Overstreet 已提交
754
		goto err;
L
Linus Torvalds 已提交
755

756 757
	err = aio_setup_ring(ctx);
	if (err < 0)
K
Kent Overstreet 已提交
758
		goto err;
K
Kent Overstreet 已提交
759

760
	atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
K
Kent Overstreet 已提交
761
	ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
762 763
	if (ctx->req_batch < 1)
		ctx->req_batch = 1;
764

L
Linus Torvalds 已提交
765
	/* limit the number of system wide aios */
766
	spin_lock(&aio_nr_lock);
767
	if (aio_nr + nr_events > (aio_max_nr * 2UL) ||
768
	    aio_nr + nr_events < aio_nr) {
769
		spin_unlock(&aio_nr_lock);
K
Kent Overstreet 已提交
770
		err = -EAGAIN;
G
Gu Zheng 已提交
771
		goto err_ctx;
772 773
	}
	aio_nr += ctx->max_reqs;
774
	spin_unlock(&aio_nr_lock);
L
Linus Torvalds 已提交
775

776 777
	percpu_ref_get(&ctx->users);	/* io_setup() will drop this ref */
	percpu_ref_get(&ctx->reqs);	/* free_ioctx_users() will drop this */
K
Kent Overstreet 已提交
778

779 780
	err = ioctx_add_table(ctx, mm);
	if (err)
K
Kent Overstreet 已提交
781
		goto err_cleanup;
782

783 784 785
	/* Release the ring_lock mutex now that all setup is complete. */
	mutex_unlock(&ctx->ring_lock);

K
Kent Overstreet 已提交
786
	pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
K
Kent Overstreet 已提交
787
		 ctx, ctx->user_id, mm, ctx->nr_events);
L
Linus Torvalds 已提交
788 789
	return ctx;

K
Kent Overstreet 已提交
790 791
err_cleanup:
	aio_nr_sub(ctx->max_reqs);
G
Gu Zheng 已提交
792
err_ctx:
793 794 795
	atomic_set(&ctx->dead, 1);
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
G
Gu Zheng 已提交
796
	aio_free_ring(ctx);
K
Kent Overstreet 已提交
797
err:
798
	mutex_unlock(&ctx->ring_lock);
K
Kent Overstreet 已提交
799
	free_percpu(ctx->cpu);
800 801
	percpu_ref_exit(&ctx->reqs);
	percpu_ref_exit(&ctx->users);
L
Linus Torvalds 已提交
802
	kmem_cache_free(kioctx_cachep, ctx);
K
Kent Overstreet 已提交
803
	pr_debug("error allocating ioctx %d\n", err);
804
	return ERR_PTR(err);
L
Linus Torvalds 已提交
805 806
}

K
Kent Overstreet 已提交
807 808 809 810 811
/* kill_ioctx
 *	Cancels all outstanding aio requests on an aio context.  Used
 *	when the processes owning a context have all exited to encourage
 *	the rapid destruction of the kioctx.
 */
812
static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
813
		      struct ctx_rq_wait *wait)
K
Kent Overstreet 已提交
814
{
815
	struct kioctx_table *table;
816

A
Al Viro 已提交
817 818 819
	spin_lock(&mm->ioctx_lock);
	if (atomic_xchg(&ctx->dead, 1)) {
		spin_unlock(&mm->ioctx_lock);
820
		return -EINVAL;
A
Al Viro 已提交
821
	}
822

823
	table = rcu_dereference_raw(mm->ioctx_table);
824 825 826
	WARN_ON(ctx != table->table[ctx->id]);
	table->table[ctx->id] = NULL;
	spin_unlock(&mm->ioctx_lock);
827

828 829
	/* percpu_ref_kill() will do the necessary call_rcu() */
	wake_up_all(&ctx->wait);
830

831 832 833 834 835 836 837 838
	/*
	 * It'd be more correct to do this in free_ioctx(), after all
	 * the outstanding kiocbs have finished - but by then io_destroy
	 * has already returned, so io_setup() could potentially return
	 * -EAGAIN with no ioctxs actually in use (as far as userspace
	 *  could tell).
	 */
	aio_nr_sub(ctx->max_reqs);
839

840 841
	if (ctx->mmap_size)
		vm_munmap(ctx->mmap_base, ctx->mmap_size);
842

843
	ctx->rq_wait = wait;
844 845
	percpu_ref_kill(&ctx->users);
	return 0;
L
Linus Torvalds 已提交
846 847
}

K
Kent Overstreet 已提交
848 849 850 851 852 853 854
/*
 * exit_aio: called when the last user of mm goes away.  At this point, there is
 * no way for any new requests to be submited or any of the io_* syscalls to be
 * called on the context.
 *
 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
 * them.
L
Linus Torvalds 已提交
855
 */
856
void exit_aio(struct mm_struct *mm)
L
Linus Torvalds 已提交
857
{
858
	struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
859 860
	struct ctx_rq_wait wait;
	int i, skipped;
861

862 863
	if (!table)
		return;
864

865 866 867 868
	atomic_set(&wait.count, table->nr);
	init_completion(&wait.comp);

	skipped = 0;
869 870
	for (i = 0; i < table->nr; ++i) {
		struct kioctx *ctx = table->table[i];
J
Jens Axboe 已提交
871

872 873
		if (!ctx) {
			skipped++;
874
			continue;
875 876
		}

877
		/*
878 879 880 881 882
		 * We don't need to bother with munmap() here - exit_mmap(mm)
		 * is coming and it'll unmap everything. And we simply can't,
		 * this is not necessarily our ->mm.
		 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
		 * that it needs to unmap the area, just set it to 0.
883
		 */
K
Kent Overstreet 已提交
884
		ctx->mmap_size = 0;
885 886
		kill_ioctx(mm, ctx, &wait);
	}
K
Kent Overstreet 已提交
887

888
	if (!atomic_sub_and_test(skipped, &wait.count)) {
889
		/* Wait until all IO for the context are done. */
890
		wait_for_completion(&wait.comp);
L
Linus Torvalds 已提交
891
	}
892 893 894

	RCU_INIT_POINTER(mm->ioctx_table, NULL);
	kfree(table);
L
Linus Torvalds 已提交
895 896
}

K
Kent Overstreet 已提交
897 898 899
static void put_reqs_available(struct kioctx *ctx, unsigned nr)
{
	struct kioctx_cpu *kcpu;
900
	unsigned long flags;
K
Kent Overstreet 已提交
901

902
	local_irq_save(flags);
903
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
904
	kcpu->reqs_available += nr;
905

K
Kent Overstreet 已提交
906 907 908 909 910
	while (kcpu->reqs_available >= ctx->req_batch * 2) {
		kcpu->reqs_available -= ctx->req_batch;
		atomic_add(ctx->req_batch, &ctx->reqs_available);
	}

911
	local_irq_restore(flags);
K
Kent Overstreet 已提交
912 913 914 915 916 917
}

static bool get_reqs_available(struct kioctx *ctx)
{
	struct kioctx_cpu *kcpu;
	bool ret = false;
918
	unsigned long flags;
K
Kent Overstreet 已提交
919

920
	local_irq_save(flags);
921
	kcpu = this_cpu_ptr(ctx->cpu);
K
Kent Overstreet 已提交
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
	if (!kcpu->reqs_available) {
		int old, avail = atomic_read(&ctx->reqs_available);

		do {
			if (avail < ctx->req_batch)
				goto out;

			old = avail;
			avail = atomic_cmpxchg(&ctx->reqs_available,
					       avail, avail - ctx->req_batch);
		} while (avail != old);

		kcpu->reqs_available += ctx->req_batch;
	}

	ret = true;
	kcpu->reqs_available--;
out:
940
	local_irq_restore(flags);
K
Kent Overstreet 已提交
941 942 943
	return ret;
}

944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
/* refill_reqs_available
 *	Updates the reqs_available reference counts used for tracking the
 *	number of free slots in the completion ring.  This can be called
 *	from aio_complete() (to optimistically update reqs_available) or
 *	from aio_get_req() (the we're out of events case).  It must be
 *	called holding ctx->completion_lock.
 */
static void refill_reqs_available(struct kioctx *ctx, unsigned head,
                                  unsigned tail)
{
	unsigned events_in_ring, completed;

	/* Clamp head since userland can write to it. */
	head %= ctx->nr_events;
	if (head <= tail)
		events_in_ring = tail - head;
	else
		events_in_ring = ctx->nr_events - (head - tail);

	completed = ctx->completed_events;
	if (events_in_ring < completed)
		completed -= events_in_ring;
	else
		completed = 0;

	if (!completed)
		return;

	ctx->completed_events -= completed;
	put_reqs_available(ctx, completed);
}

/* user_refill_reqs_available
 *	Called to refill reqs_available when aio_get_req() encounters an
 *	out of space in the completion ring.
 */
static void user_refill_reqs_available(struct kioctx *ctx)
{
	spin_lock_irq(&ctx->completion_lock);
	if (ctx->completed_events) {
		struct aio_ring *ring;
		unsigned head;

		/* Access of ring->head may race with aio_read_events_ring()
		 * here, but that's okay since whether we read the old version
		 * or the new version, and either will be valid.  The important
		 * part is that head cannot pass tail since we prevent
		 * aio_complete() from updating tail by holding
		 * ctx->completion_lock.  Even if head is invalid, the check
		 * against ctx->completed_events below will make sure we do the
		 * safe/right thing.
		 */
		ring = kmap_atomic(ctx->ring_pages[0]);
		head = ring->head;
		kunmap_atomic(ring);

		refill_reqs_available(ctx, head, ctx->tail);
	}

	spin_unlock_irq(&ctx->completion_lock);
}

L
Linus Torvalds 已提交
1006
/* aio_get_req
K
Kent Overstreet 已提交
1007 1008
 *	Allocate a slot for an aio request.
 * Returns NULL if no requests are free.
L
Linus Torvalds 已提交
1009
 */
1010
static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
L
Linus Torvalds 已提交
1011
{
1012
	struct aio_kiocb *req;
K
Kent Overstreet 已提交
1013

1014 1015 1016 1017 1018
	if (!get_reqs_available(ctx)) {
		user_refill_reqs_available(ctx);
		if (!get_reqs_available(ctx))
			return NULL;
	}
K
Kent Overstreet 已提交
1019

1020
	req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO);
L
Linus Torvalds 已提交
1021
	if (unlikely(!req))
K
Kent Overstreet 已提交
1022
		goto out_put;
L
Linus Torvalds 已提交
1023

K
Kent Overstreet 已提交
1024 1025
	percpu_ref_get(&ctx->reqs);

L
Linus Torvalds 已提交
1026
	req->ki_ctx = ctx;
J
Jeff Moyer 已提交
1027
	return req;
K
Kent Overstreet 已提交
1028
out_put:
K
Kent Overstreet 已提交
1029
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1030
	return NULL;
L
Linus Torvalds 已提交
1031 1032
}

1033
static void kiocb_free(struct aio_kiocb *req)
L
Linus Torvalds 已提交
1034
{
1035 1036
	if (req->common.ki_filp)
		fput(req->common.ki_filp);
1037 1038
	if (req->ki_eventfd != NULL)
		eventfd_ctx_put(req->ki_eventfd);
L
Linus Torvalds 已提交
1039 1040 1041
	kmem_cache_free(kiocb_cachep, req);
}

A
Adrian Bunk 已提交
1042
static struct kioctx *lookup_ioctx(unsigned long ctx_id)
L
Linus Torvalds 已提交
1043
{
1044
	struct aio_ring __user *ring  = (void __user *)ctx_id;
J
Jens Axboe 已提交
1045
	struct mm_struct *mm = current->mm;
1046
	struct kioctx *ctx, *ret = NULL;
1047 1048 1049 1050 1051
	struct kioctx_table *table;
	unsigned id;

	if (get_user(id, &ring->id))
		return NULL;
L
Linus Torvalds 已提交
1052

J
Jens Axboe 已提交
1053
	rcu_read_lock();
1054
	table = rcu_dereference(mm->ioctx_table);
J
Jens Axboe 已提交
1055

1056 1057
	if (!table || id >= table->nr)
		goto out;
L
Linus Torvalds 已提交
1058

1059
	ctx = table->table[id];
1060
	if (ctx && ctx->user_id == ctx_id) {
1061 1062 1063 1064
		percpu_ref_get(&ctx->users);
		ret = ctx;
	}
out:
J
Jens Axboe 已提交
1065
	rcu_read_unlock();
1066
	return ret;
L
Linus Torvalds 已提交
1067 1068 1069 1070 1071
}

/* aio_complete
 *	Called when the io request on the given iocb is complete.
 */
1072
static void aio_complete(struct kiocb *kiocb, long res, long res2)
L
Linus Torvalds 已提交
1073
{
1074
	struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, common);
L
Linus Torvalds 已提交
1075 1076
	struct kioctx	*ctx = iocb->ki_ctx;
	struct aio_ring	*ring;
K
Kent Overstreet 已提交
1077
	struct io_event	*ev_page, *event;
1078
	unsigned tail, pos, head;
L
Linus Torvalds 已提交
1079 1080
	unsigned long	flags;

1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091
	if (kiocb->ki_flags & IOCB_WRITE) {
		struct file *file = kiocb->ki_filp;

		/*
		 * Tell lockdep we inherited freeze protection from submission
		 * thread.
		 */
		__sb_writers_acquired(file_inode(file)->i_sb, SB_FREEZE_WRITE);
		file_end_write(file);
	}

1092 1093 1094 1095 1096 1097
	/*
	 * Special case handling for sync iocbs:
	 *  - events go directly into the iocb for fast handling
	 *  - the sync task with the iocb in its stack holds the single iocb
	 *    ref, no other paths have a way to get another ref
	 *  - the sync task helpfully left a reference to itself in the iocb
L
Linus Torvalds 已提交
1098
	 */
1099
	BUG_ON(is_sync_kiocb(kiocb));
L
Linus Torvalds 已提交
1100

1101 1102 1103 1104 1105 1106 1107
	if (iocb->ki_list.next) {
		unsigned long flags;

		spin_lock_irqsave(&ctx->ctx_lock, flags);
		list_del(&iocb->ki_list);
		spin_unlock_irqrestore(&ctx->ctx_lock, flags);
	}
1108

1109 1110
	/*
	 * Add a completion event to the ring buffer. Must be done holding
1111
	 * ctx->completion_lock to prevent other code from messing with the tail
1112 1113 1114 1115
	 * pointer since we might be called from irq context.
	 */
	spin_lock_irqsave(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1116
	tail = ctx->tail;
K
Kent Overstreet 已提交
1117 1118
	pos = tail + AIO_EVENTS_OFFSET;

K
Kent Overstreet 已提交
1119
	if (++tail >= ctx->nr_events)
1120
		tail = 0;
L
Linus Torvalds 已提交
1121

K
Kent Overstreet 已提交
1122
	ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1123 1124
	event = ev_page + pos % AIO_EVENTS_PER_PAGE;

1125
	event->obj = (u64)(unsigned long)iocb->ki_user_iocb;
L
Linus Torvalds 已提交
1126 1127 1128 1129
	event->data = iocb->ki_user_data;
	event->res = res;
	event->res2 = res2;

K
Kent Overstreet 已提交
1130
	kunmap_atomic(ev_page);
K
Kent Overstreet 已提交
1131
	flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
K
Kent Overstreet 已提交
1132 1133

	pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
1134
		 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data,
K
Kent Overstreet 已提交
1135
		 res, res2);
L
Linus Torvalds 已提交
1136 1137 1138 1139 1140 1141

	/* after flagging the request as done, we
	 * must never even look at it again
	 */
	smp_wmb();	/* make event visible before updating tail */

K
Kent Overstreet 已提交
1142
	ctx->tail = tail;
L
Linus Torvalds 已提交
1143

K
Kent Overstreet 已提交
1144
	ring = kmap_atomic(ctx->ring_pages[0]);
1145
	head = ring->head;
K
Kent Overstreet 已提交
1146
	ring->tail = tail;
1147
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1148
	flush_dcache_page(ctx->ring_pages[0]);
L
Linus Torvalds 已提交
1149

1150 1151 1152
	ctx->completed_events++;
	if (ctx->completed_events > 1)
		refill_reqs_available(ctx, head, tail);
1153 1154
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

K
Kent Overstreet 已提交
1155
	pr_debug("added to ring %p at [%u]\n", iocb, tail);
D
Davide Libenzi 已提交
1156 1157 1158 1159 1160 1161

	/*
	 * Check if the user asked us to deliver the result through an
	 * eventfd. The eventfd_signal() function is safe to be called
	 * from IRQ context.
	 */
1162
	if (iocb->ki_eventfd != NULL)
D
Davide Libenzi 已提交
1163 1164
		eventfd_signal(iocb->ki_eventfd, 1);

L
Linus Torvalds 已提交
1165
	/* everything turned out well, dispose of the aiocb. */
K
Kent Overstreet 已提交
1166
	kiocb_free(iocb);
L
Linus Torvalds 已提交
1167

1168 1169 1170 1171 1172 1173 1174 1175
	/*
	 * We have to order our ring_info tail store above and test
	 * of the wait list below outside the wait lock.  This is
	 * like in wake_up_bit() where clearing a bit has to be
	 * ordered with the unlocked test.
	 */
	smp_mb();

L
Linus Torvalds 已提交
1176 1177 1178
	if (waitqueue_active(&ctx->wait))
		wake_up(&ctx->wait);

K
Kent Overstreet 已提交
1179
	percpu_ref_put(&ctx->reqs);
L
Linus Torvalds 已提交
1180 1181
}

G
Gu Zheng 已提交
1182
/* aio_read_events_ring
1183 1184
 *	Pull an event off of the ioctx's event ring.  Returns the number of
 *	events fetched
L
Linus Torvalds 已提交
1185
 */
1186 1187
static long aio_read_events_ring(struct kioctx *ctx,
				 struct io_event __user *event, long nr)
L
Linus Torvalds 已提交
1188 1189
{
	struct aio_ring *ring;
1190
	unsigned head, tail, pos;
1191 1192 1193
	long ret = 0;
	int copy_ret;

1194 1195 1196 1197 1198 1199 1200
	/*
	 * The mutex can block and wake us up and that will cause
	 * wait_event_interruptible_hrtimeout() to schedule without sleeping
	 * and repeat. This should be rare enough that it doesn't cause
	 * peformance issues. See the comment in read_events() for more detail.
	 */
	sched_annotate_sleep();
K
Kent Overstreet 已提交
1201
	mutex_lock(&ctx->ring_lock);
L
Linus Torvalds 已提交
1202

1203
	/* Access to ->ring_pages here is protected by ctx->ring_lock. */
K
Kent Overstreet 已提交
1204
	ring = kmap_atomic(ctx->ring_pages[0]);
1205
	head = ring->head;
1206
	tail = ring->tail;
1207 1208
	kunmap_atomic(ring);

1209 1210 1211 1212 1213 1214
	/*
	 * Ensure that once we've read the current tail pointer, that
	 * we also see the events that were stored up to the tail.
	 */
	smp_rmb();

1215
	pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
L
Linus Torvalds 已提交
1216

1217
	if (head == tail)
L
Linus Torvalds 已提交
1218 1219
		goto out;

1220 1221 1222
	head %= ctx->nr_events;
	tail %= ctx->nr_events;

1223 1224 1225 1226 1227
	while (ret < nr) {
		long avail;
		struct io_event *ev;
		struct page *page;

1228 1229
		avail = (head <= tail ?  tail : ctx->nr_events) - head;
		if (head == tail)
1230 1231 1232 1233 1234 1235 1236
			break;

		avail = min(avail, nr - ret);
		avail = min_t(long, avail, AIO_EVENTS_PER_PAGE -
			    ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE));

		pos = head + AIO_EVENTS_OFFSET;
K
Kent Overstreet 已提交
1237
		page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
		pos %= AIO_EVENTS_PER_PAGE;

		ev = kmap(page);
		copy_ret = copy_to_user(event + ret, ev + pos,
					sizeof(*ev) * avail);
		kunmap(page);

		if (unlikely(copy_ret)) {
			ret = -EFAULT;
			goto out;
		}

		ret += avail;
		head += avail;
K
Kent Overstreet 已提交
1252
		head %= ctx->nr_events;
L
Linus Torvalds 已提交
1253 1254
	}

K
Kent Overstreet 已提交
1255
	ring = kmap_atomic(ctx->ring_pages[0]);
1256
	ring->head = head;
1257
	kunmap_atomic(ring);
K
Kent Overstreet 已提交
1258
	flush_dcache_page(ctx->ring_pages[0]);
1259

1260
	pr_debug("%li  h%u t%u\n", ret, head, tail);
1261
out:
K
Kent Overstreet 已提交
1262
	mutex_unlock(&ctx->ring_lock);
1263

L
Linus Torvalds 已提交
1264 1265 1266
	return ret;
}

1267 1268
static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
			    struct io_event __user *event, long *i)
L
Linus Torvalds 已提交
1269
{
1270
	long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
L
Linus Torvalds 已提交
1271

1272 1273
	if (ret > 0)
		*i += ret;
L
Linus Torvalds 已提交
1274

1275 1276
	if (unlikely(atomic_read(&ctx->dead)))
		ret = -EINVAL;
L
Linus Torvalds 已提交
1277

1278 1279
	if (!*i)
		*i = ret;
L
Linus Torvalds 已提交
1280

1281
	return ret < 0 || *i >= min_nr;
L
Linus Torvalds 已提交
1282 1283
}

1284
static long read_events(struct kioctx *ctx, long min_nr, long nr,
L
Linus Torvalds 已提交
1285 1286 1287
			struct io_event __user *event,
			struct timespec __user *timeout)
{
1288 1289
	ktime_t until = { .tv64 = KTIME_MAX };
	long ret = 0;
L
Linus Torvalds 已提交
1290 1291 1292

	if (timeout) {
		struct timespec	ts;
1293

L
Linus Torvalds 已提交
1294
		if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
1295
			return -EFAULT;
L
Linus Torvalds 已提交
1296

1297
		until = timespec_to_ktime(ts);
L
Linus Torvalds 已提交
1298 1299
	}

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
	/*
	 * Note that aio_read_events() is being called as the conditional - i.e.
	 * we're calling it after prepare_to_wait() has set task state to
	 * TASK_INTERRUPTIBLE.
	 *
	 * But aio_read_events() can block, and if it blocks it's going to flip
	 * the task state back to TASK_RUNNING.
	 *
	 * This should be ok, provided it doesn't flip the state back to
	 * TASK_RUNNING and return 0 too much - that causes us to spin. That
	 * will only happen if the mutex_lock() call blocks, and we then find
	 * the ringbuffer empty. So in practice we should be ok, but it's
	 * something to be aware of when touching this code.
	 */
1314 1315 1316 1317 1318 1319
	if (until.tv64 == 0)
		aio_read_events(ctx, min_nr, nr, event, &ret);
	else
		wait_event_interruptible_hrtimeout(ctx->wait,
				aio_read_events(ctx, min_nr, nr, event, &ret),
				until);
L
Linus Torvalds 已提交
1320

1321 1322
	if (!ret && signal_pending(current))
		ret = -EINTR;
L
Linus Torvalds 已提交
1323

1324
	return ret;
L
Linus Torvalds 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
}

/* sys_io_setup:
 *	Create an aio_context capable of receiving at least nr_events.
 *	ctxp must not point to an aio_context that already exists, and
 *	must be initialized to 0 prior to the call.  On successful
 *	creation of the aio_context, *ctxp is filled in with the resulting 
 *	handle.  May fail with -EINVAL if *ctxp is not initialized,
 *	if the specified nr_events exceeds internal limits.  May fail 
 *	with -EAGAIN if the specified nr_events exceeds the user's limit 
 *	of available events.  May fail with -ENOMEM if insufficient kernel
 *	resources are available.  May fail with -EFAULT if an invalid
 *	pointer is passed for ctxp.  Will fail with -ENOSYS if not
 *	implemented.
 */
1340
SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
{
	struct kioctx *ioctx = NULL;
	unsigned long ctx;
	long ret;

	ret = get_user(ctx, ctxp);
	if (unlikely(ret))
		goto out;

	ret = -EINVAL;
1351
	if (unlikely(ctx || nr_events == 0)) {
1352
		pr_debug("EINVAL: ctx %lu nr_events %u\n",
1353
		         ctx, nr_events);
L
Linus Torvalds 已提交
1354 1355 1356 1357 1358 1359 1360
		goto out;
	}

	ioctx = ioctx_alloc(nr_events);
	ret = PTR_ERR(ioctx);
	if (!IS_ERR(ioctx)) {
		ret = put_user(ioctx->user_id, ctxp);
1361
		if (ret)
1362
			kill_ioctx(current->mm, ioctx, NULL);
K
Kent Overstreet 已提交
1363
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1364 1365 1366 1367 1368 1369 1370 1371 1372
	}

out:
	return ret;
}

/* sys_io_destroy:
 *	Destroy the aio_context specified.  May cancel any outstanding 
 *	AIOs and block on completion.  Will fail with -ENOSYS if not
1373
 *	implemented.  May fail with -EINVAL if the context pointed to
L
Linus Torvalds 已提交
1374 1375
 *	is invalid.
 */
1376
SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
L
Linus Torvalds 已提交
1377 1378 1379
{
	struct kioctx *ioctx = lookup_ioctx(ctx);
	if (likely(NULL != ioctx)) {
1380
		struct ctx_rq_wait wait;
1381
		int ret;
1382

1383 1384 1385
		init_completion(&wait.comp);
		atomic_set(&wait.count, 1);

1386 1387 1388 1389
		/* Pass requests_done to kill_ioctx() where it can be set
		 * in a thread-safe way. If we try to set it here then we have
		 * a race condition if two io_destroy() called simultaneously.
		 */
1390
		ret = kill_ioctx(current->mm, ioctx, &wait);
K
Kent Overstreet 已提交
1391
		percpu_ref_put(&ioctx->users);
1392 1393 1394 1395 1396

		/* Wait until all IO for the context are done. Otherwise kernel
		 * keep using user-space buffers even if user thinks the context
		 * is destroyed.
		 */
1397
		if (!ret)
1398
			wait_for_completion(&wait.comp);
1399

1400
		return ret;
L
Linus Torvalds 已提交
1401
	}
1402
	pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1403 1404 1405
	return -EINVAL;
}

C
Christoph Hellwig 已提交
1406 1407
static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec,
		bool vectored, bool compat, struct iov_iter *iter)
B
Badari Pulavarty 已提交
1408
{
C
Christoph Hellwig 已提交
1409 1410 1411 1412 1413 1414 1415 1416
	void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;
	size_t len = iocb->aio_nbytes;

	if (!vectored) {
		ssize_t ret = import_single_range(rw, buf, len, *iovec, iter);
		*iovec = NULL;
		return ret;
	}
1417 1418
#ifdef CONFIG_COMPAT
	if (compat)
C
Christoph Hellwig 已提交
1419 1420
		return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec,
				iter);
1421
#endif
C
Christoph Hellwig 已提交
1422
	return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter);
B
Badari Pulavarty 已提交
1423 1424
}

C
Christoph Hellwig 已提交
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
static inline ssize_t aio_ret(struct kiocb *req, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		return ret;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		ret = -EINTR;
		/*FALLTHRU*/
	default:
		aio_complete(req, ret, 0);
		return 0;
	}
}

static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored,
		bool compat)
L
Linus Torvalds 已提交
1448
{
K
Kent Overstreet 已提交
1449
	struct file *file = req->ki_filp;
1450
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1451
	struct iov_iter iter;
C
Christoph Hellwig 已提交
1452
	ssize_t ret;
L
Linus Torvalds 已提交
1453

C
Christoph Hellwig 已提交
1454 1455 1456 1457
	if (unlikely(!(file->f_mode & FMODE_READ)))
		return -EBADF;
	if (unlikely(!file->f_op->read_iter))
		return -EINVAL;
K
Kent Overstreet 已提交
1458

C
Christoph Hellwig 已提交
1459 1460 1461 1462 1463 1464 1465 1466 1467
	ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
	if (ret)
		return ret;
	ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
	if (!ret)
		ret = aio_ret(req, file->f_op->read_iter(req, &iter));
	kfree(iovec);
	return ret;
}
K
Kent Overstreet 已提交
1468

C
Christoph Hellwig 已提交
1469 1470 1471 1472 1473 1474 1475
static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored,
		bool compat)
{
	struct file *file = req->ki_filp;
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct iov_iter iter;
	ssize_t ret;
K
Kent Overstreet 已提交
1476

C
Christoph Hellwig 已提交
1477 1478 1479
	if (unlikely(!(file->f_mode & FMODE_WRITE)))
		return -EBADF;
	if (unlikely(!file->f_op->write_iter))
K
Kent Overstreet 已提交
1480
		return -EINVAL;
L
Linus Torvalds 已提交
1481

C
Christoph Hellwig 已提交
1482 1483 1484 1485 1486
	ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
	if (ret)
		return ret;
	ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
	if (!ret) {
1487
		req->ki_flags |= IOCB_WRITE;
C
Christoph Hellwig 已提交
1488 1489
		file_start_write(file);
		ret = aio_ret(req, file->f_op->write_iter(req, &iter));
1490 1491 1492 1493 1494 1495
		/*
		 * We release freeze protection in aio_complete().  Fool lockdep
		 * by telling it the lock got released so that it doesn't
		 * complain about held lock when we return to userspace.
		 */
		__sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE);
K
Kent Overstreet 已提交
1496
	}
C
Christoph Hellwig 已提交
1497 1498
	kfree(iovec);
	return ret;
L
Linus Torvalds 已提交
1499 1500
}

A
Adrian Bunk 已提交
1501
static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
K
Kent Overstreet 已提交
1502
			 struct iocb *iocb, bool compat)
L
Linus Torvalds 已提交
1503
{
1504
	struct aio_kiocb *req;
C
Christoph Hellwig 已提交
1505
	struct file *file;
L
Linus Torvalds 已提交
1506 1507 1508
	ssize_t ret;

	/* enforce forwards compatibility on users */
1509
	if (unlikely(iocb->aio_reserved1 || iocb->aio_reserved2)) {
K
Kent Overstreet 已提交
1510
		pr_debug("EINVAL: reserve field set\n");
L
Linus Torvalds 已提交
1511 1512 1513 1514 1515 1516 1517 1518 1519
		return -EINVAL;
	}

	/* prevent overflows */
	if (unlikely(
	    (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
	    (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
	    ((ssize_t)iocb->aio_nbytes < 0)
	   )) {
1520
		pr_debug("EINVAL: overflow check\n");
L
Linus Torvalds 已提交
1521 1522 1523
		return -EINVAL;
	}

K
Kent Overstreet 已提交
1524
	req = aio_get_req(ctx);
1525
	if (unlikely(!req))
L
Linus Torvalds 已提交
1526
		return -EAGAIN;
1527

C
Christoph Hellwig 已提交
1528
	req->common.ki_filp = file = fget(iocb->aio_fildes);
1529
	if (unlikely(!req->common.ki_filp)) {
1530 1531
		ret = -EBADF;
		goto out_put_req;
L
Linus Torvalds 已提交
1532
	}
1533 1534
	req->common.ki_pos = iocb->aio_offset;
	req->common.ki_complete = aio_complete;
1535
	req->common.ki_flags = iocb_flags(req->common.ki_filp);
1536

1537 1538 1539 1540 1541 1542 1543
	if (iocb->aio_flags & IOCB_FLAG_RESFD) {
		/*
		 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
		 * instance of the file* now. The file descriptor must be
		 * an eventfd() fd, and will be signaled for each completed
		 * event using the eventfd_signal() function.
		 */
1544
		req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd);
1545
		if (IS_ERR(req->ki_eventfd)) {
1546
			ret = PTR_ERR(req->ki_eventfd);
1547
			req->ki_eventfd = NULL;
1548 1549
			goto out_put_req;
		}
1550 1551

		req->common.ki_flags |= IOCB_EVENTFD;
1552
	}
L
Linus Torvalds 已提交
1553

K
Kent Overstreet 已提交
1554
	ret = put_user(KIOCB_KEY, &user_iocb->aio_key);
L
Linus Torvalds 已提交
1555
	if (unlikely(ret)) {
K
Kent Overstreet 已提交
1556
		pr_debug("EFAULT: aio_key\n");
L
Linus Torvalds 已提交
1557 1558 1559
		goto out_put_req;
	}

1560
	req->ki_user_iocb = user_iocb;
L
Linus Torvalds 已提交
1561 1562
	req->ki_user_data = iocb->aio_data;

C
Christoph Hellwig 已提交
1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
	get_file(file);
	switch (iocb->aio_lio_opcode) {
	case IOCB_CMD_PREAD:
		ret = aio_read(&req->common, iocb, false, compat);
		break;
	case IOCB_CMD_PWRITE:
		ret = aio_write(&req->common, iocb, false, compat);
		break;
	case IOCB_CMD_PREADV:
		ret = aio_read(&req->common, iocb, true, compat);
		break;
	case IOCB_CMD_PWRITEV:
		ret = aio_write(&req->common, iocb, true, compat);
		break;
	default:
		pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode);
		ret = -EINVAL;
		break;
	}
	fput(file);
Z
Zach Brown 已提交
1583

C
Christoph Hellwig 已提交
1584 1585
	if (ret && ret != -EIOCBQUEUED)
		goto out_put_req;
L
Linus Torvalds 已提交
1586 1587
	return 0;
out_put_req:
K
Kent Overstreet 已提交
1588
	put_reqs_available(ctx, 1);
K
Kent Overstreet 已提交
1589
	percpu_ref_put(&ctx->reqs);
K
Kent Overstreet 已提交
1590
	kiocb_free(req);
L
Linus Torvalds 已提交
1591 1592 1593
	return ret;
}

1594 1595
long do_io_submit(aio_context_t ctx_id, long nr,
		  struct iocb __user *__user *iocbpp, bool compat)
L
Linus Torvalds 已提交
1596 1597 1598
{
	struct kioctx *ctx;
	long ret = 0;
J
Jeff Moyer 已提交
1599
	int i = 0;
S
Shaohua Li 已提交
1600
	struct blk_plug plug;
L
Linus Torvalds 已提交
1601 1602 1603 1604

	if (unlikely(nr < 0))
		return -EINVAL;

1605 1606 1607
	if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
		nr = LONG_MAX/sizeof(*iocbpp);

L
Linus Torvalds 已提交
1608 1609 1610 1611 1612
	if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx)) {
K
Kent Overstreet 已提交
1613
		pr_debug("EINVAL: invalid context id\n");
L
Linus Torvalds 已提交
1614 1615 1616
		return -EINVAL;
	}

S
Shaohua Li 已提交
1617 1618
	blk_start_plug(&plug);

L
Linus Torvalds 已提交
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
	/*
	 * AKPM: should this return a partial result if some of the IOs were
	 * successfully submitted?
	 */
	for (i=0; i<nr; i++) {
		struct iocb __user *user_iocb;
		struct iocb tmp;

		if (unlikely(__get_user(user_iocb, iocbpp + i))) {
			ret = -EFAULT;
			break;
		}

		if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
			ret = -EFAULT;
			break;
		}

K
Kent Overstreet 已提交
1637
		ret = io_submit_one(ctx, user_iocb, &tmp, compat);
L
Linus Torvalds 已提交
1638 1639 1640
		if (ret)
			break;
	}
S
Shaohua Li 已提交
1641
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
1642

K
Kent Overstreet 已提交
1643
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1644 1645 1646
	return i ? i : ret;
}

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
/* sys_io_submit:
 *	Queue the nr iocbs pointed to by iocbpp for processing.  Returns
 *	the number of iocbs queued.  May return -EINVAL if the aio_context
 *	specified by ctx_id is invalid, if nr is < 0, if the iocb at
 *	*iocbpp[0] is not properly initialized, if the operation specified
 *	is invalid for the file descriptor in the iocb.  May fail with
 *	-EFAULT if any of the data structures point to invalid data.  May
 *	fail with -EBADF if the file descriptor specified in the first
 *	iocb is invalid.  May fail with -EAGAIN if insufficient resources
 *	are available to queue any iocbs.  Will return 0 if nr is 0.  Will
 *	fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
		struct iocb __user * __user *, iocbpp)
{
	return do_io_submit(ctx_id, nr, iocbpp, 0);
}

L
Linus Torvalds 已提交
1665 1666 1667
/* lookup_kiocb
 *	Finds a given iocb for cancellation.
 */
1668 1669
static struct aio_kiocb *
lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb, u32 key)
L
Linus Torvalds 已提交
1670
{
1671
	struct aio_kiocb *kiocb;
1672 1673 1674

	assert_spin_locked(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1675 1676 1677
	if (key != KIOCB_KEY)
		return NULL;

L
Linus Torvalds 已提交
1678
	/* TODO: use a hash or array, this sucks. */
1679 1680
	list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
		if (kiocb->ki_user_iocb == iocb)
L
Linus Torvalds 已提交
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
			return kiocb;
	}
	return NULL;
}

/* sys_io_cancel:
 *	Attempts to cancel an iocb previously passed to io_submit.  If
 *	the operation is successfully cancelled, the resulting event is
 *	copied into the memory pointed to by result without being placed
 *	into the completion queue and 0 is returned.  May fail with
 *	-EFAULT if any of the data structures pointed to are invalid.
 *	May fail with -EINVAL if aio_context specified by ctx_id is
 *	invalid.  May fail with -EAGAIN if the iocb specified was not
 *	cancelled.  Will fail with -ENOSYS if not implemented.
 */
1696 1697
SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
		struct io_event __user *, result)
L
Linus Torvalds 已提交
1698 1699
{
	struct kioctx *ctx;
1700
	struct aio_kiocb *kiocb;
L
Linus Torvalds 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
	u32 key;
	int ret;

	ret = get_user(key, &iocb->aio_key);
	if (unlikely(ret))
		return -EFAULT;

	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx))
		return -EINVAL;

	spin_lock_irq(&ctx->ctx_lock);
K
Kent Overstreet 已提交
1713

L
Linus Torvalds 已提交
1714
	kiocb = lookup_kiocb(ctx, iocb, key);
K
Kent Overstreet 已提交
1715
	if (kiocb)
1716
		ret = kiocb_cancel(kiocb);
K
Kent Overstreet 已提交
1717 1718 1719
	else
		ret = -EINVAL;

L
Linus Torvalds 已提交
1720 1721
	spin_unlock_irq(&ctx->ctx_lock);

K
Kent Overstreet 已提交
1722
	if (!ret) {
1723 1724 1725 1726
		/*
		 * The result argument is no longer used - the io_event is
		 * always delivered via the ring buffer. -EINPROGRESS indicates
		 * cancellation is progress:
K
Kent Overstreet 已提交
1727
		 */
1728
		ret = -EINPROGRESS;
K
Kent Overstreet 已提交
1729
	}
L
Linus Torvalds 已提交
1730

K
Kent Overstreet 已提交
1731
	percpu_ref_put(&ctx->users);
L
Linus Torvalds 已提交
1732 1733 1734 1735 1736 1737

	return ret;
}

/* io_getevents:
 *	Attempts to read at least min_nr events and up to nr events from
1738 1739 1740 1741 1742 1743 1744 1745
 *	the completion queue for the aio_context specified by ctx_id. If
 *	it succeeds, the number of read events is returned. May fail with
 *	-EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
 *	out of range, if timeout is out of range.  May fail with -EFAULT
 *	if any of the memory specified is invalid.  May return 0 or
 *	< min_nr if the timeout specified by timeout has elapsed
 *	before sufficient events are available, where timeout == NULL
 *	specifies an infinite timeout. Note that the timeout pointed to by
J
Jeff Moyer 已提交
1746
 *	timeout is relative.  Will fail with -ENOSYS if not implemented.
L
Linus Torvalds 已提交
1747
 */
1748 1749 1750 1751 1752
SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
		long, min_nr,
		long, nr,
		struct io_event __user *, events,
		struct timespec __user *, timeout)
L
Linus Torvalds 已提交
1753 1754 1755 1756 1757
{
	struct kioctx *ioctx = lookup_ioctx(ctx_id);
	long ret = -EINVAL;

	if (likely(ioctx)) {
N
Namhyung Kim 已提交
1758
		if (likely(min_nr <= nr && min_nr >= 0))
L
Linus Torvalds 已提交
1759
			ret = read_events(ioctx, min_nr, nr, events, timeout);
K
Kent Overstreet 已提交
1760
		percpu_ref_put(&ioctx->users);
L
Linus Torvalds 已提交
1761 1762 1763
	}
	return ret;
}